diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index c2c78f46f..0359d2a4c 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -3,7 +3,10 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING -from sglang.srt.arg_groups.overrides import declare_resolution +from sglang.srt.arg_groups.overrides import ( + declare_resolution, + resolving_view, +) from sglang.srt.environ import envs if TYPE_CHECKING: @@ -16,18 +19,16 @@ def validate_deepseek_v4_mega_moe_token_budget( server_args: ServerArgs, ) -> None: """Ensure the DSV4 prefill budget fits MegaMoE's per-rank buffer.""" - mega_moe_enabled = server_args.moe_a2a_backend == "megamoe" - if not mega_moe_enabled or server_args.disaggregation_mode == "decode": + cfg = resolving_view(server_args) + mega_moe_enabled = cfg.moe_a2a_backend == "megamoe" + if not mega_moe_enabled or cfg.disaggregation_mode == "decode": # decode node will skip the check because decode bs is not relevant with --chunk-prefill-size return - if server_args.pp_size > 1 and server_args.enable_dynamic_chunking: + if cfg.pp_size > 1 and cfg.enable_dynamic_chunking: return - if ( - server_args.chunked_prefill_size is None - or server_args.chunked_prefill_size <= 0 - ): + if cfg.chunked_prefill_size is None or cfg.chunked_prefill_size <= 0: raise ValueError( "DeepSeekV4 with MegaMoE requires chunked prefill to be enabled. " "Set --chunked-prefill-size to a positive value; " @@ -35,40 +36,38 @@ def validate_deepseek_v4_mega_moe_token_budget( "token requirement would not have a strict prefill-forward bound." ) - if server_args.enable_prefill_cp: - token_partition_size = server_args.attn_cp_size + if cfg.enable_prefill_cp: + token_partition_size = cfg.attn_cp_size token_partition_name = "attn_cp_size" token_alignment = 1 local_chunked_prefill_size = ( - server_args.chunked_prefill_size + token_partition_size - 1 + cfg.chunked_prefill_size + token_partition_size - 1 ) // token_partition_size - elif server_args.enable_dp_attention: - token_partition_size = server_args.dp_size + elif cfg.enable_dp_attention: + token_partition_size = cfg.dp_size token_partition_name = "dp_size" token_alignment = max( - server_args.tp_size // server_args.dp_size // server_args.attn_cp_size, + cfg.tp_size // cfg.dp_size // cfg.attn_cp_size, 1, ) - local_chunked_prefill_size = ( - server_args.chunked_prefill_size // token_partition_size - ) + local_chunked_prefill_size = cfg.chunked_prefill_size // token_partition_size else: # Pure TP and PP with static chunking are handled here. token_partition_size = 1 token_partition_name = "none" # global_num_tokens will ceil_align to attn_tp_size so the validation needs to do alignment as well token_alignment = max( - server_args.tp_size // token_partition_size // server_args.attn_cp_size, + cfg.tp_size // token_partition_size // cfg.attn_cp_size, 1, ) - local_chunked_prefill_size = server_args.chunked_prefill_size + local_chunked_prefill_size = cfg.chunked_prefill_size if local_chunked_prefill_size <= 0: raise ValueError( "DeepSeekV4 with MegaMoE requires a positive effective per-rank " "chunked prefill size. " f"Current values: chunked_prefill_size=" - f"{server_args.chunked_prefill_size}, " + f"{cfg.chunked_prefill_size}, " f"token_partition={token_partition_name}, " f"token_partition_size={token_partition_size}." ) @@ -87,7 +86,7 @@ def validate_deepseek_v4_mega_moe_token_budget( "SGLANG_OPT_DEEPGEMM_MEGA_MOE_NUM_MAX_TOKENS_PER_RANK to " "cover each rank's effective prefill token budget. " f"Current values: chunked_prefill_size=" - f"{server_args.chunked_prefill_size}, " + f"{cfg.chunked_prefill_size}, " f"token_partition={token_partition_name}, " f"token_partition_size={token_partition_size}, " f"token_alignment={token_alignment}, " @@ -112,6 +111,7 @@ def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None max_running_requests fill (the speculative hook is a later writer of that field) and the validations. """ + cfg = resolving_view(server_args) from sglang.srt.utils import is_hip # FlashMLA sparse prefill (SGLANG_OPT_FLASHMLA_SPARSE_PREFILL, default on) @@ -136,36 +136,36 @@ def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None run_post_process_pass(server_args, _deepseek_v4_kv_cache_dtype) - if server_args.max_running_requests is None: + if cfg.max_running_requests is None: declare_resolution( server_args, "apply_deepseek_v4_defaults", max_running_requests=256, ) logger.warning( - f"Setting max_running_requests to {server_args.max_running_requests} for {model_arch}." + f"Setting max_running_requests to {cfg.max_running_requests} for {model_arch}." ) - if server_args.speculative_algorithm is not None: - assert server_args.speculative_algorithm in ( + if cfg.speculative_algorithm is not None: + assert cfg.speculative_algorithm in ( "EAGLE", "DSPARK", ), f"Only EAGLE and DSPARK speculative algorithms are supported for {model_arch}" - if server_args.speculative_algorithm == "EAGLE": + if cfg.speculative_algorithm == "EAGLE": assert ( - server_args.speculative_eagle_topk == 1 + cfg.speculative_eagle_topk == 1 ), f"Only EAGLE speculative algorithm with topk == 1 is supported for {model_arch}" def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: """Validate DeepSeek V4 context-parallel configuration.""" - if not server_args.enable_prefill_cp: + cfg = resolving_view(server_args) + if not cfg.enable_prefill_cp: return - if server_args.cp_strategy != "interleave": + if cfg.cp_strategy != "interleave": raise ValueError( - "DeepSeekV4 only supports interleave CP strategy, " - f"got {server_args.cp_strategy}" + "DeepSeekV4 only supports interleave CP strategy, " f"got {cfg.cp_strategy}" ) declare_resolution( @@ -196,19 +196,19 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: declare_resolution( server_args, "validate_deepseek_v4_cp", - attn_cp_size=server_args.tp_size // server_args.dp_size, + attn_cp_size=cfg.tp_size // cfg.dp_size, ) assert ( - server_args.dp_size == 1 + cfg.dp_size == 1 ), "For round-robin split mode, dp attention is not supported." assert ( - server_args.tp_size <= 8 + cfg.tp_size <= 8 ), "Context parallel only supports single machine (tp_size <= 8). Cross-machine CP has precision issues." - if server_args.moe_a2a_backend not in ("none", "deepep", "megamoe"): + if cfg.moe_a2a_backend not in ("none", "deepep", "megamoe"): raise ValueError( "DeepSeekV4 CP supports moe_a2a_backend in " "('none', 'deepep', 'megamoe'), " - f"got {server_args.moe_a2a_backend!r}." + f"got {cfg.moe_a2a_backend!r}." ) logger.warning( "Disabling SGLANG_OPT_FLASHMLA_SPARSE_PREFILL because DeepSeekV4 " @@ -217,6 +217,6 @@ def validate_deepseek_v4_cp(server_args: ServerArgs) -> None: envs.SGLANG_OPT_FLASHMLA_SPARSE_PREFILL.set(False) logger.warning( f"Enable Context Parallel for DeepSeekV4, " - f"dp_size={server_args.dp_size}, moe_dense_tp_size={server_args.moe_dense_tp_size}, " - f"attn_cp_size={server_args.attn_cp_size}, ep_size={server_args.ep_size}, tp_size={server_args.tp_size}" + f"dp_size={cfg.dp_size}, moe_dense_tp_size={cfg.moe_dense_tp_size}, " + f"attn_cp_size={cfg.attn_cp_size}, ep_size={cfg.ep_size}, tp_size={cfg.tp_size}" ) diff --git a/python/sglang/srt/arg_groups/expert_pack_hook.py b/python/sglang/srt/arg_groups/expert_pack_hook.py index b7ee90484..4908c4355 100644 --- a/python/sglang/srt/arg_groups/expert_pack_hook.py +++ b/python/sglang/srt/arg_groups/expert_pack_hook.py @@ -9,7 +9,7 @@ import os from pathlib import Path from typing import Any -from sglang.srt.arg_groups.overrides import declare_resolution +from sglang.srt.arg_groups.overrides import declare_resolution, resolving_view from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import ( Backend, @@ -27,31 +27,32 @@ logger = logging.getLogger(__name__) def handle_expert_pack(server_args: Any) -> None: """Normalize expert-pack settings and report all startup errors together.""" - if server_args.load_format != "expert_pack": + cfg = resolving_view(server_args) + if cfg.load_format != "expert_pack": return errors = [] parallelism = ( - ("tensor", "--tp-size", server_args.tp_size), - ("data", "--dp-size", server_args.dp_size), - ("expert", "--ep-size", server_args.ep_size), + ("tensor", "--tp-size", cfg.tp_size), + ("data", "--dp-size", cfg.dp_size), + ("expert", "--ep-size", cfg.ep_size), ) for label, option, size in parallelism: if size != 1: errors.append(f"{label} parallelism ({option}) must be 1, got {size}") - if server_args.enforce_shared_experts_fusion: + if cfg.enforce_shared_experts_fusion: errors.append( "--enforce-shared-experts-fusion is incompatible with expert_pack" ) - if server_args.enable_waterfill: + if cfg.enable_waterfill: errors.append("--enable-waterfill is incompatible with expert_pack") explicit_cuda_graph_backends = { - Phase.DECODE: server_args.cuda_graph_backend_decode, - Phase.PREFILL: server_args.cuda_graph_backend_prefill, + Phase.DECODE: cfg.cuda_graph_backend_decode, + Phase.PREFILL: cfg.cuda_graph_backend_prefill, } - raw_cuda_graph_config = server_args.cuda_graph_config + raw_cuda_graph_config = cfg.cuda_graph_config if isinstance(raw_cuda_graph_config, CudaGraphConfig): raw_cuda_graph_config = raw_cuda_graph_config.to_dict() for phase in Phase.ALL: @@ -69,7 +70,7 @@ def handle_expert_pack(server_args: Any) -> None: f"disabled, got {explicit_backend!r}" ) - loader_config = server_args.model_loader_extra_config or {} + loader_config = cfg.model_loader_extra_config or {} if isinstance(loader_config, str): try: loader_config = json.loads(loader_config) @@ -82,7 +83,7 @@ def handle_expert_pack(server_args: Any) -> None: # A raw GGUF path is the public input form. Preparation is performed once # here, before model-config parsing and before the loader is constructed. - raw_model_path = Path(server_args.model_path).expanduser() + raw_model_path = Path(cfg.model_path).expanduser() raw_preparation_failed = False if not errors and raw_model_path.is_file(): try: @@ -136,12 +137,12 @@ def handle_expert_pack(server_args: Any) -> None: errors.append(f"expert-pack file does not exist: {pack_path}") model_kind = None - model_path = parse_path("--model-path", server_args.model_path) + model_path = parse_path("--model-path", cfg.model_path) if not raw_preparation_failed: if model_path is None or not model_path.is_dir(): errors.append( "--model-path must be a local GGUF shard or tokenizer/config " - f"directory for expert_pack, got {server_args.model_path!r}" + f"directory for expert_pack, got {cfg.model_path!r}" ) else: try: diff --git a/python/sglang/srt/arg_groups/hisparse_hook.py b/python/sglang/srt/arg_groups/hisparse_hook.py index a5cc2661e..99b2b448c 100644 --- a/python/sglang/srt/arg_groups/hisparse_hook.py +++ b/python/sglang/srt/arg_groups/hisparse_hook.py @@ -3,6 +3,8 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING +from sglang.srt.arg_groups.overrides import resolving_view + if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs @@ -79,7 +81,8 @@ def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None: def validate_hisparse(server_args: ServerArgs) -> None: """Validate --enable-hisparse constraints (model class, radix cache, DSA backend).""" - if not server_args.enable_hisparse: + cfg = resolving_view(server_args) + if not cfg.enable_hisparse: return from sglang.srt.configs.model_config import ( @@ -96,7 +99,7 @@ def validate_hisparse(server_args: ServerArgs) -> None: ) assert ( - server_args.disable_radix_cache + cfg.disable_radix_cache ), "Hierarchical sparse attention currently requires --disable-radix-cache." # DSv4 hisparse handles its own dtype/backend pairing elsewhere; the dtype- diff --git a/python/sglang/srt/arg_groups/kimi_k3_hook.py b/python/sglang/srt/arg_groups/kimi_k3_hook.py index 9b42917f8..464df72d0 100644 --- a/python/sglang/srt/arg_groups/kimi_k3_hook.py +++ b/python/sglang/srt/arg_groups/kimi_k3_hook.py @@ -3,7 +3,10 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING -from sglang.srt.arg_groups.overrides import declare_resolution +from sglang.srt.arg_groups.overrides import ( + declare_resolution, + resolving_view, +) if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs @@ -13,15 +16,16 @@ logger = logging.getLogger(__name__) def apply_kimi_k3_spec_backend_defaults(server_args: ServerArgs) -> None: """Apply speculative backend defaults for Kimi hybrid models.""" + cfg = resolving_view(server_args) from sglang.srt.utils import is_sm100_supported - if server_args.speculative_algorithm is None: + if cfg.speculative_algorithm is None: return # Use the fused Kimi-K3/DSPARK CuTeDSL kernel for KDA target verification. # Decode is left free (its bf16-ssm SM100+ flashinfer default is fine -- the # target only verifies under spec); the verify backend is pinned directly. - if server_args.linear_attn_verify_backend is None: + if cfg.linear_attn_verify_backend is None: declare_resolution( server_args, "apply_kimi_k3_spec_backend_defaults", @@ -36,8 +40,8 @@ def apply_kimi_k3_spec_backend_defaults(server_args: ServerArgs) -> None: # dspark's draft is dense MQA; trtllm_mha avoids flashinfer's blocking # per-step host plan. DSPARK-only: other spec algos use MLA-family drafts. if ( - server_args.speculative_algorithm == "DSPARK" - and server_args.speculative_draft_attention_backend is None + cfg.speculative_algorithm == "DSPARK" + and cfg.speculative_draft_attention_backend is None and is_sm100_supported() ): declare_resolution( @@ -63,20 +67,21 @@ def disable_kimi_k3_symm_mem(server_args: ServerArgs) -> None: Gates on the arch itself: this runs from cuda-graph resolution, which is earlier than the model-specific hook block. """ + cfg = resolving_view(server_args) from sglang.srt.connector import ConnectorType from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.utils import parse_connector_type - if not server_args.enable_symm_mem: + if not cfg.enable_symm_mem: return - if parse_connector_type(server_args.model_path) == ConnectorType.INSTANCE: + if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE: return if server_args.get_model_config().hf_config.architectures[0] not in ( "KimiLinearForCausalLM", "KimiK3ForConditionalGeneration", ): return - graph = server_args.cuda_graph_config + graph = cfg.cuda_graph_config if ( graph.decode.backend == Backend.DISABLED and graph.prefill.backend == Backend.DISABLED @@ -100,14 +105,15 @@ def disable_kimi_k3_symm_mem(server_args: ServerArgs) -> None: def apply_kimi_k3_linear_attn_defaults(server_args: ServerArgs) -> None: """KDA decode-fallback default for Kimi hybrid models (spec-independent).""" + cfg = resolving_view(server_args) from sglang.srt.utils import is_sm100_supported # Preempts the generic SM100+bf16 flashinfer switch (a GDN default): on # KDA shapes the triton packed decode measures ~35% faster than # recurrent_kda across bs 1-256, and ReplaySSM requires triton. if ( - server_args.linear_attn_decode_backend is None - and server_args.mamba_ssm_dtype == "bfloat16" + cfg.linear_attn_decode_backend is None + and cfg.mamba_ssm_dtype == "bfloat16" and is_sm100_supported() ): declare_resolution( diff --git a/python/sglang/srt/arg_groups/mega_moe_hook.py b/python/sglang/srt/arg_groups/mega_moe_hook.py index 0c1806e69..99777ce15 100644 --- a/python/sglang/srt/arg_groups/mega_moe_hook.py +++ b/python/sglang/srt/arg_groups/mega_moe_hook.py @@ -7,7 +7,10 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: from sglang.srt.server_args import ServerArgs -from sglang.srt.arg_groups.overrides import declare_resolution +from sglang.srt.arg_groups.overrides import ( + declare_resolution, + resolving_view, +) logger = logging.getLogger(__name__) @@ -18,15 +21,16 @@ def handle_mega_moe(server_args: ServerArgs) -> None: def handle_moe_runner_backend_alias(server_args: ServerArgs) -> None: - if server_args.moe_runner_backend != "megamoe": + cfg = resolving_view(server_args) + if cfg.moe_runner_backend != "megamoe": return - if server_args.moe_a2a_backend not in ("none", "megamoe"): + if cfg.moe_a2a_backend not in ("none", "megamoe"): logger.warning( "--moe-runner-backend megamoe is an alias for " "--moe-a2a-backend megamoe; overriding " "--moe-a2a-backend %s.", - server_args.moe_a2a_backend, + cfg.moe_a2a_backend, ) declare_resolution( server_args, @@ -37,7 +41,8 @@ def handle_moe_runner_backend_alias(server_args: ServerArgs) -> None: def handle_w4a4_mxfp4_megamoe_env(server_args: ServerArgs) -> None: - if not server_args.enable_w4a4_mxfp4_megamoe: + cfg = resolving_view(server_args) + if not cfg.enable_w4a4_mxfp4_megamoe: return os.environ["DG_USE_FP4_ACTS"] = "1" diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index b0d783881..556479e54 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -150,6 +150,42 @@ class ResolvedView: ) +class ResolvingConfig: + """Live read view of the resolution result: the declaration stash over the + record's fields, looked up per read. + + ``ResolvedView`` snapshots the overlay when it is built, which is what a + post-process pass wants -- it reads the state at its slot. A resolver that + reads *after* declaring, or after calling something that declares, needs the + current answer instead, so this one walks the stash on every read. It falls + through to the field, which is where the raw input lives. + """ + + __slots__ = ("_server_args",) + + def __init__(self, server_args: Any): + object.__setattr__(self, "_server_args", server_args) + + def __getattr__(self, name: str) -> Any: + server_args = object.__getattribute__(self, "_server_args") + for _source, declared in reversed( + getattr(server_args, "_resolved_overrides", None) or () + ): + if name in declared: + return declared[name] + return getattr(server_args, name) + + def __setattr__(self, name: str, value: Any) -> None: + raise AttributeError( + "ResolvingConfig is read-only; resolution writes through declarations" + ) + + +def resolving_view(server_args: Any) -> ResolvingConfig: + """A live read view of what resolution has decided so far.""" + return ResolvingConfig(server_args) + + # Ordered post-process passes (the normalization stage). List order is the # end-state execution order and mirrors today's handler call sequence in # __post_init__; during the transition each pass is invoked from its legacy @@ -581,16 +617,17 @@ def _require_kimi_k3_cutedsl_dcp_support() -> None: @_register_for("KimiK3ForConditionalGeneration") def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: - if server_args.dcp_size > 1: + cfg = resolving_view(server_args) + if cfg.dcp_size > 1: overrides = {} - if server_args.enable_symm_mem: + if cfg.enable_symm_mem: logger.warning( "Kimi-K3 DCP disables --enable-symm-mem due to decode CUDA " "graph correctness issues." ) overrides["enable_symm_mem"] = False - if server_args.speculative_algorithm == "DSPARK": + if cfg.speculative_algorithm == "DSPARK": from sglang.srt.speculative.ragged_verify import ( RaggedVerifyMode, read_ragged_verify_mode, @@ -611,7 +648,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: # lacks that DCP path (TypeError: unexpected kwarg 'causal_seqs'). overrides["speculative_attention_mode"] = "decode" - prefill_backend, decode_backend = attention_backends_of(server_args) + prefill_backend, decode_backend = attention_backends_of(cfg) if decode_backend == "cutedsl_mla" or decode_backend is None: _require_kimi_k3_cutedsl_dcp_support() logger.info( @@ -630,7 +667,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: ) logger.info( "Kimi-K3 DCP with tokenspeed mla backend overrides KV cache dtype: " - f"{server_args.kv_cache_dtype!r} -> 'fp8_e4m3'." + f"{cfg.kv_cache_dtype!r} -> 'fp8_e4m3'." ) overrides.update( prefill_attention_backend="tokenspeed_mla", @@ -642,7 +679,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: f"Decode attention backend for Kimi-K3 DCP must be 'cutedsl_mla' or 'tokenspeed_mla', got {decode_backend!r}." ) - if server_args.dcp_replicate_q_proj is None: + if cfg.dcp_replicate_q_proj is None: logger.info("Kimi-K3 DCP enables replicated Q projection by default.") overrides["dcp_replicate_q_proj"] = True @@ -650,7 +687,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: dcp_comm_backend = "fi_a2a" if is_mnnvl_fabric_device() else "a2a" logger.info( "Kimi-K3 DCP selects communication backend on " - f"{device_name!r}: {server_args.dcp_comm_backend!r} -> " + f"{device_name!r}: {cfg.dcp_comm_backend!r} -> " f"{dcp_comm_backend!r}." ) overrides["dcp_comm_backend"] = dcp_comm_backend @@ -659,7 +696,7 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: if not (is_sm100_supported() and get_device_sm() in (100, 103)): return {} backends_unset = server_args.is_attention_backend_not_set() - if server_args.speculative_algorithm != "DSPARK": + if cfg.speculative_algorithm != "DSPARK": if not backends_unset: return {} logger.info( @@ -673,9 +710,9 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: # DSPARK: verify runs on the decode backend (mode=decode below), so this # picks the verify kernel -- mode=prefill routes it to flashinfer, which is # slow and syncs, while plain decode is cold under dspark. - q_len = server_args.speculative_num_draft_tokens or ( - server_args.speculative_dspark_block_size + 1 - if server_args.speculative_dspark_block_size is not None + q_len = cfg.speculative_num_draft_tokens or ( + cfg.speculative_dspark_block_size + 1 + if cfg.speculative_dspark_block_size is not None # Checkpoint auto-infer happens after overrides; K3 draft uses block 7. else 8 ) @@ -688,8 +725,8 @@ def _kimi_k3_overrides(server_args: Any, hf_config: Any) -> dict: # Explicit backend knobs keep priority, but the mode is a separate knob # that still needs declaring -- else verify stays on the prefill backend, # whose host-side plan (flashinfer by default) forces a per-step D2H. - _, backend = attention_backends_of(server_args) - if _dspark_verify_on_decode_backend(backend, q_len, server_args.kv_cache_dtype): + _, backend = attention_backends_of(cfg) + if _dspark_verify_on_decode_backend(backend, q_len, cfg.kv_cache_dtype): overrides["speculative_attention_mode"] = "decode" logger.info( "Kimi-K3 DSPARK on SM100/SM103: decode/verify attention backend " @@ -727,7 +764,8 @@ def _kimi_k3_moe_runner_overrides(server_args: Any, hf_config: Any) -> dict: # (M=bs) and the target-verify (M=bs*(gamma+1)) regimes on SM100/SM103. # SM107 uses the same packed-MXFP4 runner; leaving auto unresolved falls # back to BF16 weight materialization during model loading. - if server_args.moe_runner_backend != "auto": + cfg = resolving_view(server_args) + if cfg.moe_runner_backend != "auto": return {} if not (is_sm100_supported() and get_device_sm() in (100, 103, 107)): return {} @@ -757,6 +795,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: writers), the kv-cache/split-backend defaults, the quant/moe block (read before it by _set_default_dsa_kv_cache_dtype) and the env writes stay in the branch.""" + cfg = resolving_view(server_args) from sglang.srt.configs.model_config import is_deepseek_dsa overrides: Dict[str, Any] = {} @@ -767,39 +806,39 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: overrides["attention_backend"] = "dsa" logger.info("Use dsa attention backend for DeepSeek with DSA.") if not is_npu() and not is_xpu(): # CUDA or ROCm GPU - if server_args.enable_prefill_cp: + if cfg.enable_prefill_cp: logger.warning( "Context parallel feature is still under experiment. It has only been verified on Hopper platform." ) overrides["enable_dp_attention"] = True overrides["moe_dense_tp_size"] = 1 - if server_args.cp_strategy == "zigzag": + if cfg.cp_strategy == "zigzag": overrides["moe_a2a_backend"] = "deepep" - overrides["ep_size"] = server_args.tp_size + overrides["ep_size"] = cfg.tp_size logger.warning( "zigzag DSA CP requires moe_dense_tp_size=1, " "moe_a2a_backend=deepep, ep_size=tp_size, batch_size=1." ) else: assert ( - server_args.dp_size == 1 + cfg.dp_size == 1 ), "interleave DSA CP does not support DP attention." assert ( - server_args.tp_size <= 8 + cfg.tp_size <= 8 ), "Context parallel only supports single machine (tp_size <= 8). Cross-machine CP has precision issues." # Note(kpham-sgl): Keep attn_tp_size == 1 under DSA CP. # DSACPLayerCommunicator does not all-reduce attention-TP # partial o_proj outputs before replicated dense FFNs. - attn_cp_size = server_args.tp_size // server_args.dp_size + attn_cp_size = cfg.tp_size // cfg.dp_size overrides["attn_cp_size"] = attn_cp_size logger.warning( "Enabled DSA context parallel: " - f"strategy={server_args.cp_strategy}, dp_size={server_args.dp_size}, " + f"strategy={cfg.cp_strategy}, dp_size={cfg.dp_size}, " f"moe_dense_tp_size={overrides['moe_dense_tp_size']}, " - f"ep_size={overrides.get('ep_size', server_args.ep_size)}, tp_size={server_args.tp_size}, " + f"ep_size={overrides.get('ep_size', cfg.ep_size)}, tp_size={cfg.tp_size}, " f"attn_cp_size={attn_cp_size}, " - f"kv_cache_dtype={server_args.kv_cache_dtype}, " - f"moe_a2a_backend={overrides.get('moe_a2a_backend', server_args.moe_a2a_backend)}, " + f"kv_cache_dtype={cfg.kv_cache_dtype}, " + f"moe_a2a_backend={overrides.get('moe_a2a_backend', cfg.moe_a2a_backend)}, " f"cuda_graph_config[prefill].backend=disabled" ) @@ -826,9 +865,9 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: # DeepSeek V3/R1/V3.1 if is_sm100_supported(): if ( - server_args.attention_backend is None - and server_args.prefill_attention_backend is None - and server_args.decode_attention_backend is None + cfg.attention_backend is None + and cfg.prefill_attention_backend is None + and cfg.decode_attention_backend is None ): overrides["attention_backend"] = "trtllm_mla" logger.info( @@ -836,7 +875,7 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: ) # MLA prefill CP auto-config. Mirrors the NSA CP block above # (minus the in-seq/round-robin mode split, which MLA CP does not support) - if server_args.enable_prefill_cp and server_args.use_mla_backend(): + if cfg.enable_prefill_cp and server_args.use_mla_backend(): logger.warning( "MLA prefill context parallel is still experimental. " "Verified on Hopper with the fa3 backend." @@ -845,22 +884,22 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: # TODO(kpham-sgl) Supports moe_dense_tp_size != 1. overrides["moe_dense_tp_size"] = 1 overrides["moe_a2a_backend"] = "deepep" - overrides["ep_size"] = server_args.tp_size + overrides["ep_size"] = cfg.tp_size logger.warning( "For MLA CP, we have the following restrictions: moe_dense_tp_size == 1, moe_a2a_backend == deepep, ep_size == tp_size, batch_size == 1" ) # FIXME(kpham-sgl): Keep attn_tp_size == 1 under MLA CP. # DSACPLayerCommunicator does not all-reduce attention-TP # partial o_proj outputs before replicated dense FFNs. - attn_cp_size = server_args.tp_size // server_args.dp_size + attn_cp_size = cfg.tp_size // cfg.dp_size overrides["attn_cp_size"] = attn_cp_size logger.warning( f"Enable Context Parallel opt for MLA, " - f"Setting dp_size == {server_args.dp_size} and " + f"Setting dp_size == {cfg.dp_size} and " f"attn_cp_size == {attn_cp_size}, " f"moe_dense_tp_size == {overrides['moe_dense_tp_size']}, " f"ep_size == {overrides['ep_size']}, " - f"tp_size == {server_args.tp_size}, " + f"tp_size == {cfg.tp_size}, " f"moe_a2a_backend {overrides['moe_a2a_backend']}, " f"cuda_graph_config[prefill].backend=disabled" ) @@ -870,8 +909,9 @@ def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: # Keep in sync with MIMO_V2_MODEL_ARCHS (server_args.py / configs/hf_config.py). @_register_for("MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM") def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) overrides: Dict[str, Any] = {} - if server_args.speculative_algorithm == "EAGLE": + if cfg.speculative_algorithm == "EAGLE": logger.info("Enable multi-layer EAGLE speculative decoding for MiMoV2 model.") overrides["enable_multi_layer_eagle"] = True @@ -879,7 +919,7 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict: # slower at bs=1 decode. FP4 checkpoints use flashinfer_mxfp4 instead. if ( is_sm100_supported() - and server_args.moe_runner_backend == "auto" + and cfg.moe_runner_backend == "auto" and get_quantization_config(hf_config) == "fp8" ): overrides["moe_runner_backend"] = "flashinfer_trtllm" @@ -889,13 +929,14 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict: @_register_for("MiniMaxM2ForCausalLM") def _minimax_m2_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) overrides = {"enable_tf32_matmul": True} logger.info( "Enable TF32 matmul for MiniMaxM2ForCausalLM model to improve gate gemm performance." ) if ( is_sm100_supported() - and server_args.moe_runner_backend == "auto" + and cfg.moe_runner_backend == "auto" and server_args.get_model_config().quantization == "modelopt_fp4" ): overrides["moe_runner_backend"] = "flashinfer_trtllm_routed" @@ -909,10 +950,11 @@ def _minimax_m2_overrides(server_args: Any, hf_config: Any) -> dict: @_register_for("MiniMaxM3SparseForCausalLM", "MiniMaxM3SparseForConditionalGeneration") def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) overrides: Dict[str, Any] = {} quant_method = get_quantization_config(hf_config) - quant_resolved = server_args.quantization + quant_resolved = cfg.quantization if ( quant_resolved is None and not server_args._quantization_explicitly_unset @@ -924,16 +966,12 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict: if is_hip(): if server_args.is_attention_backend_not_set(): overrides["attention_backend"] = "triton" - if server_args.moe_runner_backend == "auto" and quant_resolved == "mxfp8": + if cfg.moe_runner_backend == "auto" and quant_resolved == "mxfp8": overrides["moe_runner_backend"] = "triton" if not envs.USE_ROCM_AITER_ROPE_BACKEND.is_set(): envs.USE_ROCM_AITER_ROPE_BACKEND.set("0") - aiter_fusion_resolved = server_args.enable_aiter_allreduce_fusion - if ( - server_args.ep_size > 1 - and server_args.moe_a2a_backend == "none" - and aiter_fusion_resolved - ): + aiter_fusion_resolved = cfg.enable_aiter_allreduce_fusion + if cfg.ep_size > 1 and cfg.moe_a2a_backend == "none" and aiter_fusion_resolved: logger.warning( "Disable --enable-aiter-allreduce-fusion for MiniMax-M3 " "standard EP on ROCm because the deferred fused all-reduce " @@ -951,7 +989,7 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict: elif is_sm100_supported(): if server_args.is_attention_backend_not_set(): if ( - server_args.kv_cache_dtype == "fp8_e4m3" + cfg.kv_cache_dtype == "fp8_e4m3" and not envs.SGLANG_DISABLE_M3_FP8_ATTN_GEMM.get() ): # fp8 attention GEMMs activate whenever possible @@ -962,42 +1000,36 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict: overrides["attention_backend"] = "trtllm_mha" else: overrides["attention_backend"] = "fa4" - backend_resolved = overrides.get( - "attention_backend", server_args.attention_backend - ) - page_resolved = server_args.page_size + backend_resolved = overrides.get("attention_backend", cfg.attention_backend) + page_resolved = cfg.page_size # fa4 (fmha_sm100) and trtllm_mha both allow the page_size == 128 # sparse block MSA needs (trtllm_mha via trtllm-gen's dynamic # tokens-per-page kernels). if page_resolved is None and backend_resolved in ("fa4", "trtllm_mha"): overrides["page_size"] = 128 page_resolved = 128 - if server_args.moe_runner_backend == "auto" and quant_resolved == "mxfp8": + if cfg.moe_runner_backend == "auto" and quant_resolved == "mxfp8": overrides["moe_runner_backend"] = "deep_gemm" - elif ( - server_args.moe_runner_backend == "auto" - and quant_resolved == "modelopt_mixed" - ): + elif cfg.moe_runner_backend == "auto" and quant_resolved == "modelopt_mixed": overrides["moe_runner_backend"] = "flashinfer_trtllm_routed" logger.info( "MiniMax-M3 on SM100: attention_backend=" - f"{overrides.get('attention_backend', server_args.attention_backend)}, page_size={page_resolved}, " - f"moe_runner_backend={overrides.get('moe_runner_backend', server_args.moe_runner_backend)}." + f"{overrides.get('attention_backend', cfg.attention_backend)}, page_size={page_resolved}, " + f"moe_runner_backend={overrides.get('moe_runner_backend', cfg.moe_runner_backend)}." ) elif is_sm90_supported(): if server_args.is_attention_backend_not_set(): overrides["attention_backend"] = "fa3" - page_resolved = server_args.page_size + page_resolved = cfg.page_size if ( page_resolved is None - and overrides.get("attention_backend", server_args.attention_backend) - == "fa3" + and overrides.get("attention_backend", cfg.attention_backend) == "fa3" ): overrides["page_size"] = 128 page_resolved = 128 logger.info( "MiniMax-M3 on Hopper: attention_backend=" - f"{overrides.get('attention_backend', server_args.attention_backend)}, page_size={page_resolved} " + f"{overrides.get('attention_backend', cfg.attention_backend)}, page_size={page_resolved} " "(MSA is SM100-only; sparse attention runs on the Triton path)." ) @@ -1008,7 +1040,7 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict: # silently dispatch the e4m3 kernel, so e5m2 stays on the widening Triton # path), log when the fp8 GEMM mode is active, and log when the # SGLANG_DISABLE_M3_FP8_ATTN_GEMM kill switch suppresses it. - if server_args.kv_cache_dtype == "fp8_e5m2": + if cfg.kv_cache_dtype == "fp8_e5m2": logger.warning( "MiniMax-M3 with kv_cache_dtype fp8_e5m2: fp8 attention GEMMs stay " "DISABLED (fmha_sm100's variant lookup would silently dispatch the " @@ -1016,9 +1048,8 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict: "Triton path. Use --kv-cache-dtype fp8_e4m3 for fp8 attention GEMMs." ) elif ( - server_args.kv_cache_dtype == "fp8_e4m3" - and overrides.get("attention_backend", server_args.attention_backend) - == "trtllm_mha" + cfg.kv_cache_dtype == "fp8_e4m3" + and overrides.get("attention_backend", cfg.attention_backend) == "trtllm_mha" and is_sm100_supported() ): if envs.SGLANG_DISABLE_M3_FP8_ATTN_GEMM.get(): @@ -1036,9 +1067,7 @@ def _minimax_m3_overrides(server_args: Any, hf_config: Any) -> dict: "force the pre-fp8 numerics." ) - moe_runner_resolved = overrides.get( - "moe_runner_backend", server_args.moe_runner_backend - ) + moe_runner_resolved = overrides.get("moe_runner_backend", cfg.moe_runner_backend) if quant_resolved is None and moe_runner_resolved in ("auto", "deep_gemm"): if moe_runner_resolved == "deep_gemm": logger.warning( @@ -1078,6 +1107,7 @@ def _exaone_overrides(server_args: Any, hf_config: Any) -> dict: @_register_for("GptOssForCausalLM") def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) overrides: Dict[str, Any] = {} # Set attention backend for GPT-OSS if server_args.is_attention_backend_not_set(): @@ -1100,14 +1130,14 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict: # Check for bf16 dtype on Intel XPU. Reads the pristine dtype request, # which equals the legacy mid-branch read: dtype had no earlier writer # for this arch. - if server_args.dtype == "auto": + if cfg.dtype == "auto": logger.warning( "GptOssForCausalLM on Intel XPU currently supports bfloat16 dtype only" ) - elif server_args.dtype not in ["bfloat16"]: + elif cfg.dtype not in ["bfloat16"]: raise NotImplementedError( f"GptOssForCausalLM on Intel XPU only supports bfloat16 dtype, " - f"but got '{server_args.dtype}'. Please use --dtype bfloat16 or remove --dtype to use auto." + f"but got '{cfg.dtype}'. Please use --dtype bfloat16 or remove --dtype to use auto." ) quantization_config = getattr(hf_config, "quantization_config", None) is_mxfp4_quant_format = ( @@ -1117,7 +1147,7 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict: if is_mxfp4_quant_format: # use bf16 for mxfp4 triton kernels overrides["dtype"] = "bfloat16" - if server_args.moe_runner_backend == "auto": + if cfg.moe_runner_backend == "auto": if is_sm100_supported() and is_mxfp4_quant_format: overrides["moe_runner_backend"] = "flashinfer_mxfp4" @@ -1155,9 +1185,9 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict: "Detected MUSA with SGLANG_DEEPEP_BF16_DISPATCH for bf16 model, using deep_gemm kernel." ) elif ( - server_args.ep_size == 1 + cfg.ep_size == 1 and is_triton_kernels_available() - and server_args.quantization is None + and cfg.quantization is None and not (is_cpu() and cpu_has_amx_support()) ): # The triton_kernels package segfaults on Blackwell (B200) @@ -1179,18 +1209,19 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict: # Keep in sync with LLAMA4_MODEL_ARCHS (server_args.py). @_register_for("Llama4ForConditionalGeneration", "Llama4ForCausalLM") def _llama4_overrides(server_args: Any, hf_config: Any) -> dict: - if server_args.device == "cpu": + cfg = resolving_view(server_args) + if cfg.device == "cpu": return {} overrides: Dict[str, Any] = {} # Auto-select attention backend for Llama4 if not specified - if server_args.attention_backend is None: + if cfg.attention_backend is None: if is_sm100_supported(): backend, platform = "trtllm_mha", "sm100" elif is_sm90_supported(): backend, platform = "fa3", "sm90" elif is_hip(): backend, platform = "aiter", "hip" - elif server_args.device == "xpu": + elif cfg.device == "xpu": backend, platform = "intel_xpu", "xpu" else: backend, platform = "triton", "other platforms" @@ -1198,8 +1229,8 @@ def _llama4_overrides(server_args: Any, hf_config: Any) -> dict: f"Use {backend} as attention backend on {platform} for Llama4 model" ) overrides["attention_backend"] = backend - if is_sm100_supported() and server_args.moe_runner_backend == "auto": - if server_args.quantization in {"fp8", "modelopt_fp8"}: + if is_sm100_supported() and cfg.moe_runner_backend == "auto": + if cfg.quantization in {"fp8", "modelopt_fp8"}: overrides["moe_runner_backend"] = "flashinfer_trtllm" logger.info( "Use flashinfer_trtllm as MoE runner backend on SM100 for Llama4" @@ -1213,6 +1244,7 @@ def _llama4_overrides(server_args: Any, hf_config: Any) -> dict: "Gemma4UnifiedForConditionalGeneration", ) def _gemma4_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) overrides: Dict[str, Any] = {} default_attention_backend = "trtllm_mha" if is_sm100_supported() else "triton" if server_args.is_attention_backend_not_set(): @@ -1223,9 +1255,9 @@ def _gemma4_overrides(server_args: Any, hf_config: Any) -> dict: # If only one split backend is set, keep the other side on a # Gemma4-compatible fallback instead of letting generic backend selection # choose an unsupported backend later. - elif server_args.attention_backend is None: + elif cfg.attention_backend is None: overrides["attention_backend"] = default_attention_backend - if is_sm100_supported() and server_args.moe_runner_backend == "auto": + if is_sm100_supported() and cfg.moe_runner_backend == "auto": if server_args.get_model_config().quantization == "modelopt_fp4": overrides["quantization"] = "modelopt_fp4" overrides["moe_runner_backend"] = "flashinfer_trtllm" @@ -1255,7 +1287,8 @@ def _moss_vl_overrides(server_args: Any, hf_config: Any) -> dict: @_register_for("MiniCPMForCausalLM", "MiniCPMSALAForCausalLM") def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict: - if server_args.enable_dp_attention: + cfg = resolving_view(server_args) + if cfg.enable_dp_attention: raise ValueError("MiniCPM does not support DP attention") has_sparse_attention = getattr(hf_config, "has_minicpm_sparse_attention", False) has_hybrid_attention = has_sparse_attention or getattr( @@ -1263,7 +1296,7 @@ def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict: ) overrides: Dict[str, Any] = {} if has_hybrid_attention: - if server_args.enable_hierarchical_cache: + if cfg.enable_hierarchical_cache: raise ValueError("MiniCPM SALA does not support hierarchical cache") overrides["disable_radix_cache"] = True if envs.SGLANG_MINICPM_FORCE_DENSE.get(): @@ -1273,29 +1306,29 @@ def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict: } # Literal keys keep the written-field set statically derivable; a loop # variable hides it from the census in test_chain_read_ratchet.py. - dense_attention = dense_backends.get(server_args.attention_backend) + dense_attention = dense_backends.get(cfg.attention_backend) if dense_attention is not None: overrides["attention_backend"] = dense_attention - dense_prefill = dense_backends.get(server_args.prefill_attention_backend) + dense_prefill = dense_backends.get(cfg.prefill_attention_backend) if dense_prefill is not None: overrides["prefill_attention_backend"] = dense_prefill - dense_decode = dense_backends.get(server_args.decode_attention_backend) + dense_decode = dense_backends.get(cfg.decode_attention_backend) if dense_decode is not None: overrides["decode_attention_backend"] = dense_decode elif has_sparse_attention: - uses_sparse_backend = server_args.is_attention_backend_not_set() or any( + uses_sparse_backend = cfg.is_attention_backend_not_set() or any( backend in ("minicpm_flashattn", "minicpm_flashinfer") for backend in ( - server_args.attention_backend, - server_args.prefill_attention_backend, - server_args.decode_attention_backend, + cfg.attention_backend, + cfg.prefill_attention_backend, + cfg.decode_attention_backend, ) ) - if uses_sparse_backend and server_args.disaggregation_mode != "null": + if uses_sparse_backend and cfg.disaggregation_mode != "null": raise ValueError( "MiniCPM sparse attention does not support PD disaggregation" ) - if server_args.is_attention_backend_not_set(): + if cfg.is_attention_backend_not_set(): overrides["attention_backend"] = ( "minicpm_flashinfer" if is_blackwell_supported() @@ -1306,7 +1339,8 @@ def _minicpm_sala_overrides(server_args: Any, hf_config: Any) -> dict: @_register_for("MiniCPMV4_6ForConditionalGeneration") def _minicpm_v4_6_overrides(server_args: Any, hf_config: Any) -> dict: - if is_sm100_supported() and server_args.attention_backend is None: + cfg = resolving_view(server_args) + if is_sm100_supported() and cfg.attention_backend is None: return {"attention_backend": "triton"} return {} @@ -1315,24 +1349,27 @@ def _minicpm_v4_6_overrides(server_args: Any, hf_config: Any) -> dict: "FalconH1ForCausalLM", "JetNemotronForCausalLM", "JetVLMForConditionalGeneration" ) def _falcon_h1_jet_overrides(server_args: Any, hf_config: Any) -> dict: - if is_sm100_supported() and server_args.attention_backend is None: + cfg = resolving_view(server_args) + if is_sm100_supported() and cfg.attention_backend is None: return {"attention_backend": "triton"} return {} @_register_for("GraniteMoeHybridForCausalLM") def _granite_moe_hybrid_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) has_mamba = any( layer_type == "mamba" for layer_type in getattr(hf_config, "layer_types", []) ) - if has_mamba and is_sm100_supported() and server_args.attention_backend is None: + if has_mamba and is_sm100_supported() and cfg.attention_backend is None: return {"attention_backend": "flashinfer"} return {} @_register_for("Lfm2ForCausalLM", "Lfm2MoeForCausalLM") def _lfm2_overrides(server_args: Any, hf_config: Any) -> dict: - if is_sm100_supported() and server_args.attention_backend is None: + cfg = resolving_view(server_args) + if is_sm100_supported() and cfg.attention_backend is None: return {"attention_backend": "flashinfer"} return {} @@ -1343,13 +1380,14 @@ def _deepseek_v4_overrides(server_args: Any, hf_config: Any) -> dict: arg_groups/deepseek_v4_hook.py). The kv-cache dtype and NPU split-backend writes, the max_running_requests fill and the validations stay in the hook at its legacy slot.""" + cfg = resolving_view(server_args) from sglang.srt.server_args import ServerArgs model_arch = hf_config.architectures[0] overrides: Dict[str, Any] = {"attention_backend": "dsv4"} page_size = 256 - if server_args.device == "npu": + if cfg.device == "npu": # NPU keeps the device-aware "dsv4" backend (the registry routes it to # the Ascend V4 subclass); only the pool geometry / dtype differ. # set_default_server_args() pins all three backends to "ascend" for @@ -1363,11 +1401,11 @@ def _deepseek_v4_overrides(server_args: Any, hf_config: Any) -> dict: f"Use dsv4 attention backend for {model_arch}, setting page_size to {page_size}." ) - if server_args.swa_full_tokens_ratio == ServerArgs.swa_full_tokens_ratio: + if cfg.swa_full_tokens_ratio == ServerArgs.swa_full_tokens_ratio: overrides["swa_full_tokens_ratio"] = 0.1 logger.info(f"Setting swa_full_tokens_ratio to 0.1 for {model_arch}.") - if server_args.moe_runner_backend == "auto": + if cfg.moe_runner_backend == "auto": model_config = server_args.get_model_config() # nvidia/DeepSeek-V4-Pro-NVFP4 uses the routed TRT-LLM runner. if model_config.nvfp4_moe_meta is not None: @@ -1377,9 +1415,9 @@ def _deepseek_v4_overrides(server_args: Any, hf_config: Any) -> dict: f"{model_arch} hybrid FP8+NVFP4 checkpoint." ) elif ( - server_args.device == "cuda" + cfg.device == "cuda" and not is_hip() - and server_args.moe_a2a_backend == "none" + and cfg.moe_a2a_backend == "none" and not envs.SGLANG_DSV4_FP4_DEQUANT.get() and model_config.is_fp4_experts and (is_sm90_supported() or is_sm100_supported() or is_sm120_supported()) @@ -1407,6 +1445,7 @@ def _inkling_overrides(server_args: Any, hf_config: Any) -> dict: prefill.backend, and an explicit --cuda-graph-backend-prefill / --disable-prefill-cuda-graph still wins. The unified-radix env write follows the MiniMax-M3 handler precedent (env is not a resolvable server-arg).""" + cfg = resolving_view(server_args) from sglang.srt.server_args import ServerArgs overrides: Dict[str, Any] = {} @@ -1415,14 +1454,14 @@ def _inkling_overrides(server_args: Any, hf_config: Any) -> dict: # cuda_graph_backend_prefill declared here lands too late (the breakable # default would already have been auto-disabled for this multimodal arch). # It is set inline before _handle_cuda_graph_config instead. - if server_args.swa_full_tokens_ratio == ServerArgs.swa_full_tokens_ratio: + if cfg.swa_full_tokens_ratio == ServerArgs.swa_full_tokens_ratio: overrides["swa_full_tokens_ratio"] = 0.1 - if server_args.mamba_full_memory_ratio == ServerArgs.mamba_full_memory_ratio: + if cfg.mamba_full_memory_ratio == ServerArgs.mamba_full_memory_ratio: overrides["mamba_full_memory_ratio"] = 0.1 # Inkling requires the extra-buffer mamba strategy (inkling.py asserts # enable_mamba_extra_buffer()); the generic "auto" resolution does not cover # Inkling, so pin it here. Yields to an explicit --mamba-scheduler-strategy. - if server_args.mamba_radix_cache_strategy == ServerArgs.mamba_radix_cache_strategy: + if cfg.mamba_radix_cache_strategy == ServerArgs.mamba_radix_cache_strategy: overrides["mamba_radix_cache_strategy"] = "extra_buffer" # Inkling attention runs only on the fa4 (Blackwell) or triton backends -- # models/inkling_common/attn.py asserts attention_backend in {fa4, triton}. @@ -1447,6 +1486,7 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict: """NemotronH quantization / MoE runner / attention backend defaults (absorbed from the retired arg_groups/nemotron_h_hook.py; the mamba radix cache handling and the triton-backend assert stay in the arch branch).""" + cfg = resolving_view(server_args) model_arch = hf_config.architectures[0] model_config = server_args.get_model_config() overrides: Dict[str, Any] = {} @@ -1457,7 +1497,7 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict: "modelopt_fp4", "modelopt_mixed", ] - quantization = server_args.quantization + quantization = cfg.quantization if is_modelopt: assert model_config.hf_config.mlp_hidden_act == "relu2" if model_config.quantization == "modelopt": @@ -1482,22 +1522,22 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict: ) if has_w4a16_moe_layers: - if server_args.moe_a2a_backend != "none": + if cfg.moe_a2a_backend != "none": raise ValueError("W4A16_NVFP4 MoE layers require --moe-a2a-backend=none.") - if server_args.moe_runner_backend not in ("auto", "marlin"): + if cfg.moe_runner_backend not in ("auto", "marlin"): raise ValueError( "W4A16_NVFP4 MoE layers require --moe-runner-backend=marlin." ) - if server_args.moe_runner_backend == "auto": + if cfg.moe_runner_backend == "auto": overrides["moe_runner_backend"] = "marlin" logger.info( "Use marlin as MoE runner backend for " f"{model_arch} with W4A16_NVFP4 MoE layers" ) elif (is_modelopt or model_config.quantization is None) and ( - server_args.moe_runner_backend == "auto" + cfg.moe_runner_backend == "auto" ): - if is_sm100_supported() and server_args.moe_a2a_backend == "none": + if is_sm100_supported() and cfg.moe_a2a_backend == "none": overrides["moe_runner_backend"] = "flashinfer_trtllm" logger.info( f"Use flashinfer_trtllm as MoE runner backend on sm100 for {model_arch}" @@ -1518,27 +1558,27 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict: else: overrides["moe_runner_backend"] = "flashinfer_cutlass" - if is_blackwell_supported() and server_args.is_attention_backend_not_set(): - if server_args.speculative_algorithm is not None: - speculative_algorithm = server_args.speculative_algorithm.upper() - if is_sm100_supported() and server_args.speculative_eagle_topk in ( + if is_blackwell_supported() and cfg.is_attention_backend_not_set(): + if cfg.speculative_algorithm is not None: + speculative_algorithm = cfg.speculative_algorithm.upper() + if is_sm100_supported() and cfg.speculative_eagle_topk in ( None, 1, ): overrides["attention_backend"] = "trtllm_mha" - if server_args.page_size is None: + if cfg.page_size is None: overrides["page_size"] = 64 - if server_args.mamba_radix_cache_strategy == "auto": + if cfg.mamba_radix_cache_strategy == "auto": overrides["mamba_radix_cache_strategy"] = "extra_buffer" if ( - server_args.speculative_draft_attention_backend is None + cfg.speculative_draft_attention_backend is None and speculative_algorithm in ("EAGLE", "NEXTN", "DSPARK") ): overrides["speculative_draft_attention_backend"] = "trtllm_mha" else: overrides["attention_backend"] = "triton" if ( - server_args.speculative_draft_attention_backend is None + cfg.speculative_draft_attention_backend is None and speculative_algorithm in ("EAGLE", "NEXTN", "DFLASH", "DSPARK") ): overrides["speculative_draft_attention_backend"] = "flashinfer" @@ -1555,7 +1595,8 @@ def _nemotron_h_overrides(server_args: Any, hf_config: Any) -> dict: "Qwen3_5ForConditionalGeneration", ) def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict: - if not is_sm100_supported() or server_args.attention_backend is not None: + cfg = resolving_view(server_args) + if not is_sm100_supported() or cfg.attention_backend is not None: return {} sm100_default_attn_backend = "triton" # trtllm_mha requires speculative_eagle_topk == 1 and page_size > 1. @@ -1572,8 +1613,8 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict: # already-written field here). if default_attn_backend == "trtllm_mha" and not ( not mamba_extra_buffer_of(resolved_view(server_args)) - and not server_args.disable_radix_cache - and server_args.speculative_algorithm is None + and not cfg.disable_radix_cache + and cfg.speculative_algorithm is None ): sm100_default_attn_backend = "trtllm_mha" return { @@ -1585,7 +1626,8 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict: @_register_for("InternS2MobiusForConditionalGeneration") def _interns2_mobius_baseline_overrides(server_args: Any, hf_config: Any) -> dict: """Select the only MoE runner validated for the 2,560-expert baseline.""" - if server_args.moe_runner_backend == "auto": + cfg = resolving_view(server_args) + if cfg.moe_runner_backend == "auto": return {"moe_runner_backend": "triton_kernel"} return {} @@ -1593,11 +1635,8 @@ def _interns2_mobius_baseline_overrides(server_args: Any, hf_config: Any) -> dic @_register_for("Qwen3VLForConditionalGeneration") def _qwen3vl_overrides(server_args: Any, hf_config: Any) -> dict: - if ( - is_hip() - and envs.SGLANG_USE_AITER_UNIFIED_ATTN.get() - and server_args.page_size is None - ): + cfg = resolving_view(server_args) + if is_hip() and envs.SGLANG_USE_AITER_UNIFIED_ATTN.get() and cfg.page_size is None: logger.info( "Setting page_size=16 for aiter unified attention on Qwen3VLForConditionalGeneration." ) @@ -1614,10 +1653,11 @@ def _qwen3vl_overrides(server_args: Any, hf_config: Any) -> dict: "Qwen3_5ForConditionalGeneration", ) def _qwen3_moe_family_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) overrides: Dict[str, Any] = {} if is_sm100_supported(): quant_method = get_quantization_config(hf_config) - quantization = server_args.quantization + quantization = cfg.quantization if ( quantization is None and not server_args._quantization_explicitly_unset @@ -1627,8 +1667,8 @@ def _qwen3_moe_family_overrides(server_args: Any, hf_config: Any) -> dict: quantization = quant_method if ( (quantization in ("fp8", "modelopt_fp4") or quantization is None) - and server_args.moe_a2a_backend == "none" - and server_args.moe_runner_backend == "auto" + and cfg.moe_a2a_backend == "none" + and cfg.moe_runner_backend == "auto" ): overrides["moe_runner_backend"] = "flashinfer_trtllm" logger.info( @@ -1640,6 +1680,7 @@ def _qwen3_moe_family_overrides(server_args: Any, hf_config: Any) -> dict: @_register_for("Glm4MoeForCausalLM") def _glm4_moe_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) overrides: Dict[str, Any] = {} if is_sm100_supported(): quantization_config = getattr(hf_config, "quantization_config", None) @@ -1648,7 +1689,7 @@ def _glm4_moe_overrides(server_args: Any, hf_config: Any) -> dict: if quantization_config is not None else None ) - quantization = server_args.quantization + quantization = cfg.quantization if ( quantization is None and not server_args._quantization_explicitly_unset @@ -1658,8 +1699,8 @@ def _glm4_moe_overrides(server_args: Any, hf_config: Any) -> dict: quantization = quant_method if ( quantization in {"modelopt_fp4", None} - and server_args.moe_a2a_backend == "none" - and server_args.moe_runner_backend == "auto" + and cfg.moe_a2a_backend == "none" + and cfg.moe_runner_backend == "auto" ): overrides["moe_runner_backend"] = "flashinfer_trtllm" logger.info( @@ -1674,13 +1715,14 @@ def _glm4_moe_overrides(server_args: Any, hf_config: Any) -> dict: @_register_for("Olmo2ForCausalLM") def _olmo2_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) overrides: Dict[str, Any] = {} # FIXME: https://github.com/sgl-project/sglang/pull/7367 is not compatible with Olmo3 model. logger.warning( f"Disabling hybrid SWA memory for {hf_config.architectures[0]} as it is not yet supported." ) overrides["disable_hybrid_swa_memory"] = True - if server_args.attention_backend is None: + if cfg.attention_backend is None: if is_cuda() and is_sm100_supported(): overrides["attention_backend"] = "trtllm_mha" elif is_cuda() and get_device_sm() >= 80: @@ -1695,6 +1737,7 @@ def _olmo2_overrides(server_args: Any, hf_config: Any) -> dict: or "Step3p7ForConditionalGeneration" in arch ) def _step3p_overrides(server_args: Any, hf_config: Any) -> dict: + cfg = resolving_view(server_args) overrides: Dict[str, Any] = {} if server_args.is_attention_backend_not_set(): if is_blackwell_supported(): @@ -1703,12 +1746,12 @@ def _step3p_overrides(server_args: Any, hf_config: Any) -> dict: elif is_sm90_supported(): logger.info("Auto-select fa3 attention backend for Step3p7 on Hopper.") overrides["attention_backend"] = "fa3" - if server_args.speculative_algorithm == "EAGLE": + if cfg.speculative_algorithm == "EAGLE": logger.info( "Enable multi-layer EAGLE speculative decoding for Step3p5ForCausalLM model." ) overrides["enable_multi_layer_eagle"] = True - if server_args.enable_hierarchical_cache: + if cfg.enable_hierarchical_cache: logger.warning( "Reset swa_full_tokens_ratio to 1.0 for Step3p5ForCausalLM model with hierarchical cache" ) @@ -2170,7 +2213,8 @@ def _deepseek_v4_kv_cache_dtype(view: Any) -> dict: @_register_for("MuseGlimmerForConditionalGeneration", "MuseGlimmerForCausalLM") def _muse_glimmer_fp4_gemm_runner_overrides(server_args: Any, hf_config: Any) -> dict: - if is_sm120_supported() and server_args.fp4_gemm_runner_backend == "auto": + cfg = resolving_view(server_args) + if is_sm120_supported() and cfg.fp4_gemm_runner_backend == "auto": logger.info("Use marlin as FP4 GEMM runner backend on SM120 for Muse Glimmer") return {"fp4_gemm_runner_backend": "marlin"} return {} diff --git a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py index ff2257d61..7d45061ba 100644 --- a/python/sglang/srt/arg_groups/pd_disaggregation_hook.py +++ b/python/sglang/srt/arg_groups/pd_disaggregation_hook.py @@ -5,7 +5,10 @@ import logging import os from typing import TYPE_CHECKING -from sglang.srt.arg_groups.overrides import declare_resolution +from sglang.srt.arg_groups.overrides import ( + declare_resolution, + resolving_view, +) from sglang.srt.environ import envs if TYPE_CHECKING: @@ -16,10 +19,11 @@ logger = logging.getLogger(__name__) def handle_pd_disaggregation(server_args: ServerArgs) -> None: """Validate and normalize PD-disaggregation server args.""" + cfg = resolving_view(server_args) # "mooncake_tcp" is mooncake with the TCP transport forced: set MC_FORCE_TCP # so mooncake installs TcpTransport instead of RDMA, rewrite the backend to # mooncake, and skip RDMA HCA selection. Must run before backend-name checks. - if server_args.disaggregation_transfer_backend == "mooncake_tcp": + if cfg.disaggregation_transfer_backend == "mooncake_tcp": os.environ.setdefault("MC_FORCE_TCP", "1") declare_resolution( server_args, @@ -36,17 +40,17 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: "with MC_FORCE_TCP=1 (TCP transport, no RDMA)" ) - if server_args.disaggregation_mode == "prefill" and server_args.dcp_size > 1: + if cfg.disaggregation_mode == "prefill" and cfg.dcp_size > 1: logger.warning( "DCP on a PD prefill server is supported when prefill and decode " "use the same DCP layout, but it usually adds communication " "overhead without improving prefill performance." ) - if server_args.disaggregation_mode == "decode" and server_args.dcp_size > 1: + if cfg.disaggregation_mode == "decode" and cfg.dcp_size > 1: # Fake transfer moves no KV and is only used for synthetic decode # benchmarks, so it does not need the DCP relayout from Mooncake/NIXL. - if server_args.disaggregation_transfer_backend not in ( + if cfg.disaggregation_transfer_backend not in ( "mooncake", "nixl", "fake", @@ -54,36 +58,36 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: raise ValueError( "PD decode DCP requires --disaggregation-transfer-backend " "mooncake, nixl, or fake for synthetic benchmarking, got " - f"{server_args.disaggregation_transfer_backend!r}." + f"{cfg.disaggregation_transfer_backend!r}." ) - if server_args.disaggregation_decode_enable_radix_cache: + if cfg.disaggregation_decode_enable_radix_cache: raise ValueError( "PD decode DCP currently requires chunk cache; " "--disaggregation-decode-enable-radix-cache is not supported." ) - if server_args.enable_hierarchical_cache: + if cfg.enable_hierarchical_cache: raise ValueError( "PD decode DCP currently requires chunk cache; " "--enable-hierarchical-cache is not supported." ) - if server_args.disaggregation_mode == "decode": - if server_args.disaggregation_decode_enable_radix_cache: - if server_args.enable_hisparse: + if cfg.disaggregation_mode == "decode": + if cfg.disaggregation_decode_enable_radix_cache: + if cfg.enable_hisparse: raise ValueError( "--disaggregation-decode-enable-radix-cache is incompatible " "with --enable-hisparse" ) - if server_args.disaggregation_transfer_backend == "fake": + if cfg.disaggregation_transfer_backend == "fake": raise ValueError( "--disaggregation-decode-enable-radix-cache is incompatible " "with --disaggregation-transfer-backend fake" ) - if server_args.speculative_algorithm is not None: + if cfg.speculative_algorithm is not None: raise ValueError( "--disaggregation-decode-enable-radix-cache is incompatible " "with speculative decoding " - f"(--speculative-algorithm {server_args.speculative_algorithm})" + f"(--speculative-algorithm {cfg.speculative_algorithm})" ) from sglang.srt.arg_groups.overrides import resolved_view @@ -110,12 +114,10 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: # in-transfer (being-received-from-prefill) requests, on top of the # max_running_requests-derived pool. Large batches get none; small # per-worker batches reserve 2x the batch as cheap overlap headroom. - if server_args.disaggregation_decode_extra_slots is None: + if cfg.disaggregation_decode_extra_slots is None: extra_slots = 0 - if server_args.max_running_requests is not None: - per_worker = server_args.max_running_requests // max( - 1, server_args.dp_size - ) + if cfg.max_running_requests is not None: + per_worker = cfg.max_running_requests // max(1, cfg.dp_size) if per_worker <= 32: extra_slots = per_worker * 2 declare_resolution( @@ -124,23 +126,23 @@ def handle_pd_disaggregation(server_args: ServerArgs) -> None: disaggregation_decode_extra_slots=extra_slots, ) - elif server_args.disaggregation_mode == "prefill": + elif cfg.disaggregation_mode == "prefill": assert ( - server_args.disaggregation_transfer_backend != "fake" + cfg.disaggregation_transfer_backend != "fake" ), "Prefill server does not support 'fake' as the transfer backend" if envs.SGLANG_RUST_SERVER.get(): _alias_bootstrap_port_to_api_port(server_args) - if server_args.disaggregation_mode in ("prefill", "decode"): + if cfg.disaggregation_mode in ("prefill", "decode"): if ( envs.SGLANG_DISAGG_STAGING_BUFFER.get() - and server_args.disaggregation_transfer_backend not in ("mooncake", "nixl") + and cfg.disaggregation_transfer_backend not in ("mooncake", "nixl") ): raise ValueError( f"SGLANG_DISAGG_STAGING_BUFFER requires " f"disaggregation_transfer_backend='mooncake' or 'nixl', " - f"got '{server_args.disaggregation_transfer_backend}'." + f"got '{cfg.disaggregation_transfer_backend}'." ) @@ -151,31 +153,32 @@ def _alias_bootstrap_port_to_api_port(server_args: ServerArgs) -> None: field and agrees automatically. Decode is untouched: there the field names the PREFILL side's bootstrap port and must stay as the operator set it. """ + cfg = resolving_view(server_args) default_port = next( f.default for f in dataclasses.fields(server_args) if f.name == "disaggregation_bootstrap_port" ) - if server_args.disaggregation_bootstrap_port not in ( + if cfg.disaggregation_bootstrap_port not in ( default_port, - server_args.port, + cfg.port, ): raise ValueError( "SGLANG_RUST_SERVER serves the PD KV bootstrap registry on the api " "port itself; --disaggregation-bootstrap-port " - f"{server_args.disaggregation_bootstrap_port} conflicts with --port " - f"{server_args.port}. Drop --disaggregation-bootstrap-port (decode " + f"{cfg.disaggregation_bootstrap_port} conflicts with --port " + f"{cfg.port}. Drop --disaggregation-bootstrap-port (decode " "nodes and the PD router must then target the prefill api port)." ) - if server_args.disaggregation_bootstrap_port != server_args.port: + if cfg.disaggregation_bootstrap_port != cfg.port: logger.info( "SGLANG_RUST_SERVER: KV bootstrap registry is served on the api " "port; disaggregation_bootstrap_port %d -> %d", - server_args.disaggregation_bootstrap_port, - server_args.port, + cfg.disaggregation_bootstrap_port, + cfg.port, ) declare_resolution( server_args, "_alias_bootstrap_port_to_api_port", - disaggregation_bootstrap_port=server_args.port, + disaggregation_bootstrap_port=cfg.port, ) diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index 905dbcb54..05cb2cbe5 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Optional from sglang.srt.arg_groups.overrides import ( declare_direct_writes, declare_resolution, + resolving_view, ) if TYPE_CHECKING: @@ -17,7 +18,8 @@ logger = logging.getLogger(__name__) def _disable_overlap_schedule_for_cpu(server_args: ServerArgs) -> None: - if server_args.device != "cpu" or server_args.disable_overlap_schedule: + cfg = resolving_view(server_args) + if cfg.device != "cpu" or cfg.disable_overlap_schedule: return declare_resolution( @@ -71,9 +73,10 @@ def _resolve_speculative_algorithm_alias( def handle_speculative_decoding(server_args: ServerArgs) -> None: + cfg = resolving_view(server_args) if ( - server_args.speculative_draft_model_path is not None - and server_args.speculative_draft_model_revision is None + cfg.speculative_draft_model_path is not None + and cfg.speculative_draft_model_revision is None ): declare_resolution( server_args, @@ -90,11 +93,11 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None: run_post_process_pass(server_args, _speculative_moe_runner_default) - if server_args.speculative_algorithm is not None: + if cfg.speculative_algorithm is not None: declare_resolution( server_args, "handle_speculative_decoding", - speculative_algorithm=server_args.speculative_algorithm.upper(), + speculative_algorithm=cfg.speculative_algorithm.upper(), ) # Removal notice for the retired env var; raw os.getenv on purpose -- the @@ -108,7 +111,7 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None: kwargs = {} - override_config_file = server_args.decrypted_draft_config_file + override_config_file = cfg.decrypted_draft_config_file if override_config_file and override_config_file.strip(): kwargs["_configuration_file"] = override_config_file.strip() @@ -116,17 +119,17 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None: server_args, "handle_speculative_decoding", speculative_algorithm=_resolve_speculative_algorithm_alias( - server_args.speculative_algorithm, - server_args.speculative_draft_model_path, - trust_remote_code=server_args.trust_remote_code, + cfg.speculative_algorithm, + cfg.speculative_draft_model_path, + trust_remote_code=cfg.trust_remote_code, kwargs=kwargs, ), ) # Validate --speculative-draft-window-size once, regardless of algorithm. # Consumed by DFLASH (compact draft KV cache) and Llama EAGLE-3 (drafter attention SWA). - if server_args.speculative_draft_window_size is not None: - window_size = int(server_args.speculative_draft_window_size) + if cfg.speculative_draft_window_size is not None: + window_size = int(cfg.speculative_draft_window_size) if window_size <= 0: raise ValueError( f"--speculative-draft-window-size must be positive, got {window_size}." @@ -136,19 +139,19 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None: "handle_speculative_decoding", speculative_draft_window_size=window_size, ) - if server_args.speculative_algorithm not in ("EAGLE3", "DFLASH"): + if cfg.speculative_algorithm not in ("EAGLE3", "DFLASH"): logger.warning( "--speculative-draft-window-size has no effect with " "speculative_algorithm=%s (honored by Llama EAGLE-3 and DFLASH only).", - server_args.speculative_algorithm, + cfg.speculative_algorithm, ) algo = None - if server_args.speculative_algorithm is not None: + if cfg.speculative_algorithm is not None: from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.speculative.spec_registry import CustomSpecAlgo - algo = SpeculativeAlgorithm.from_string(server_args.speculative_algorithm) + algo = SpeculativeAlgorithm.from_string(cfg.speculative_algorithm) # TODO: move the per-algorithm validation below into spec module hooks. if isinstance(algo, CustomSpecAlgo) and algo.validate_server_args is not None: @@ -158,15 +161,15 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None: algo.validate_server_args, ) - if server_args.speculative_skip_dp_mlp_sync: - assert server_args.speculative_algorithm == "EAGLE", ( + if cfg.speculative_skip_dp_mlp_sync: + assert cfg.speculative_algorithm == "EAGLE", ( "--speculative-skip-dp-mlp-sync is only supported with " - f"speculative_algorithm == EAGLE, got {server_args.speculative_algorithm}." + f"speculative_algorithm == EAGLE, got {cfg.speculative_algorithm}." ) - if server_args.speculative_adaptive: + if cfg.speculative_adaptive: _maybe_disable_adaptive(server_args) - if server_args.speculative_adaptive: + if cfg.speculative_adaptive: _init_adaptive_speculative_params(server_args) if algo is not None: @@ -180,9 +183,10 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None: def _handle_dflash(server_args: ServerArgs) -> None: + cfg = resolving_view(server_args) from sglang.srt.arg_groups.overrides import resolved_view - if not (server_args.device.startswith("cuda") or server_args.device == "npu"): + if not (cfg.device.startswith("cuda") or cfg.device == "npu"): raise ValueError( "DFLASH speculative decoding only supports CUDA and NPU devices." ) @@ -192,12 +196,12 @@ def _handle_dflash(server_args: ServerArgs) -> None: "Currently DFLASH speculative decoding does not support dp attention." ) - if server_args.pp_size != 1: + if cfg.pp_size != 1: raise ValueError( "Currently DFLASH speculative decoding only supports pp_size == 1." ) - if server_args.speculative_draft_model_path is None: + if cfg.speculative_draft_model_path is None: raise ValueError( "DFLASH speculative decoding requires setting --speculative-draft-model-path." ) @@ -207,16 +211,16 @@ def _handle_dflash(server_args: ServerArgs) -> None: # RoPE reservation). Force them to 1 to avoid surprising memory behavior. # # For DFlash, the natural unit is `block_size` (verify window length). - if server_args.speculative_num_steps is None: + if cfg.speculative_num_steps is None: declare_resolution( server_args, "_handle_dflash", speculative_num_steps=1, ) - elif int(server_args.speculative_num_steps) != 1: + elif int(cfg.speculative_num_steps) != 1: logger.warning( "DFLASH only supports speculative_num_steps == 1; overriding speculative_num_steps=%s to 1.", - server_args.speculative_num_steps, + cfg.speculative_num_steps, ) declare_resolution( server_args, @@ -224,16 +228,16 @@ def _handle_dflash(server_args: ServerArgs) -> None: speculative_num_steps=1, ) - if server_args.speculative_eagle_topk is None: + if cfg.speculative_eagle_topk is None: declare_resolution( server_args, "_handle_dflash", speculative_eagle_topk=1, ) - elif int(server_args.speculative_eagle_topk) != 1: + elif int(cfg.speculative_eagle_topk) != 1: logger.warning( "DFLASH only supports speculative_eagle_topk == 1; overriding speculative_eagle_topk=%s to 1.", - server_args.speculative_eagle_topk, + cfg.speculative_eagle_topk, ) declare_resolution( server_args, @@ -241,41 +245,41 @@ def _handle_dflash(server_args: ServerArgs) -> None: speculative_eagle_topk=1, ) - if server_args.speculative_dflash_block_size is not None: - if int(server_args.speculative_dflash_block_size) <= 0: + if cfg.speculative_dflash_block_size is not None: + if int(cfg.speculative_dflash_block_size) <= 0: raise ValueError( "DFLASH requires --speculative-dflash-block-size to be positive, " - f"got {server_args.speculative_dflash_block_size}." + f"got {cfg.speculative_dflash_block_size}." ) - if server_args.speculative_num_draft_tokens is not None and int( - server_args.speculative_num_draft_tokens - ) != int(server_args.speculative_dflash_block_size): + if cfg.speculative_num_draft_tokens is not None and int( + cfg.speculative_num_draft_tokens + ) != int(cfg.speculative_dflash_block_size): raise ValueError( "Both --speculative-num-draft-tokens and --speculative-dflash-block-size are set " "but they differ. For DFLASH they must match. " - f"speculative_num_draft_tokens={server_args.speculative_num_draft_tokens}, " - f"speculative_dflash_block_size={server_args.speculative_dflash_block_size}." + f"speculative_num_draft_tokens={cfg.speculative_num_draft_tokens}, " + f"speculative_dflash_block_size={cfg.speculative_dflash_block_size}." ) declare_resolution( server_args, "_handle_dflash", - speculative_num_draft_tokens=int(server_args.speculative_dflash_block_size), + speculative_num_draft_tokens=int(cfg.speculative_dflash_block_size), ) - if server_args.speculative_num_draft_tokens is None: + if cfg.speculative_num_draft_tokens is None: from sglang.srt.speculative.dflash_utils import ( parse_dflash_draft_config, ) - model_override_args = json.loads(server_args.json_model_override_args) + model_override_args = json.loads(cfg.json_model_override_args) inferred_block_size = None try: from sglang.srt.utils.hf_transformers_utils import get_config draft_hf_config = get_config( - server_args.speculative_draft_model_path, - trust_remote_code=server_args.trust_remote_code, - revision=server_args.speculative_draft_model_revision, + cfg.speculative_draft_model_path, + trust_remote_code=cfg.trust_remote_code, + revision=cfg.speculative_draft_model_revision, model_override_args=model_override_args, ) inferred_block_size = parse_dflash_draft_config( @@ -300,18 +304,18 @@ def _handle_dflash(server_args: ServerArgs) -> None: speculative_num_draft_tokens=inferred_block_size, ) - if server_args.speculative_draft_window_size is not None: - draft_tokens = int(server_args.speculative_num_draft_tokens) - if server_args.speculative_draft_window_size < draft_tokens: + if cfg.speculative_draft_window_size is not None: + draft_tokens = int(cfg.speculative_num_draft_tokens) + if cfg.speculative_draft_window_size < draft_tokens: raise ValueError( "--speculative-draft-window-size must be >= " "--speculative-num-draft-tokens (block_size). " - f"window_size={server_args.speculative_draft_window_size}, block_size={draft_tokens}." + f"window_size={cfg.speculative_draft_window_size}, block_size={draft_tokens}." ) _resolve_dflash_draft_attention_backend(server_args) - if server_args.max_running_requests is None: + if cfg.max_running_requests is None: declare_resolution( server_args, "_handle_dflash", @@ -321,7 +325,7 @@ def _handle_dflash(server_args: ServerArgs) -> None: "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests." ) - if server_args.enable_mixed_chunk: + if cfg.enable_mixed_chunk: declare_resolution( server_args, "_handle_dflash", @@ -341,23 +345,24 @@ def _target_checkpoint_bundles_dspark_draft(server_args: ServerArgs) -> bool: def _handle_dspark(server_args: ServerArgs) -> None: - _is_npu = server_args.device.startswith("npu") - if not server_args.device.startswith(("cuda", "npu")): + cfg = resolving_view(server_args) + _is_npu = cfg.device.startswith("npu") + if not cfg.device.startswith(("cuda", "npu")): raise ValueError( "DSpark speculative decoding only supports CUDA or NPU device." ) # dp_size==1 with dp_attention is a degenerate flag under DSV4 CP; skip DP-only checks. - if server_args.enable_dp_attention and server_args.dp_size > 1: - if not server_args.enable_dp_lm_head: + if cfg.enable_dp_attention and cfg.dp_size > 1: + if not cfg.enable_dp_lm_head: raise ValueError("DSpark with dp attention requires --enable-dp-lm-head.") - if not _is_npu and server_args.moe_a2a_backend not in ("none", "megamoe"): + if not _is_npu and cfg.moe_a2a_backend not in ("none", "megamoe"): raise ValueError( "DSpark with dp attention supports moe_a2a_backend 'none' " "(built-in TP MoE) or 'megamoe', got " - f"{server_args.moe_a2a_backend!r}." + f"{cfg.moe_a2a_backend!r}." ) - if not _is_npu and server_args.moe_a2a_backend != "none": + if not _is_npu and cfg.moe_a2a_backend != "none": from sglang.srt.speculative.ragged_verify import ( RaggedVerifyMode, read_ragged_verify_mode, @@ -366,46 +371,46 @@ def _handle_dspark(server_args: ServerArgs) -> None: if read_ragged_verify_mode() is not RaggedVerifyMode.STATIC: raise ValueError( "DSpark with dp attention + " - f"moe_a2a_backend={server_args.moe_a2a_backend!r} requires " + f"moe_a2a_backend={cfg.moe_a2a_backend!r} requires " "SGLANG_RAGGED_VERIFY_MODE=static." ) - if server_args.attn_cp_size > 1: + if cfg.attn_cp_size > 1: raise ValueError( "DSpark with dp attention does not support context parallel " - f"(attn_cp_size={server_args.attn_cp_size})." + f"(attn_cp_size={cfg.attn_cp_size})." ) if ( not _is_npu - and server_args.speculative_moe_a2a_backend is not None - and server_args.speculative_moe_a2a_backend != server_args.moe_a2a_backend + and cfg.speculative_moe_a2a_backend is not None + and cfg.speculative_moe_a2a_backend != cfg.moe_a2a_backend ): raise ValueError( "DSpark ignores --speculative-moe-a2a-backend; with dp attention it " - f"must match the target moe_a2a_backend={server_args.moe_a2a_backend!r} " - f"(got {server_args.speculative_moe_a2a_backend!r})." + f"must match the target moe_a2a_backend={cfg.moe_a2a_backend!r} " + f"(got {cfg.speculative_moe_a2a_backend!r})." ) - if server_args.pp_size != 1: + if cfg.pp_size != 1: raise ValueError( "Currently DSpark speculative decoding only supports pp_size == 1." ) - if server_args.speculative_draft_model_path is None: + if cfg.speculative_draft_model_path is None: if _target_checkpoint_bundles_dspark_draft(server_args): declare_resolution( server_args, "_handle_dspark", - speculative_draft_model_path=server_args.model_path, + speculative_draft_model_path=cfg.model_path, ) declare_resolution( server_args, "_handle_dspark", - speculative_draft_model_revision=server_args.revision, + speculative_draft_model_revision=cfg.revision, ) logger.info( "DSpark draft weights are bundled in the target checkpoint; " "defaulting --speculative-draft-model-path to --model-path (%s).", - server_args.model_path, + cfg.model_path, ) else: raise ValueError( @@ -413,16 +418,16 @@ def _handle_dspark(server_args: ServerArgs) -> None: "--speculative-draft-model-path." ) - if server_args.speculative_num_steps is None: + if cfg.speculative_num_steps is None: declare_resolution( server_args, "_handle_dspark", speculative_num_steps=1, ) - elif int(server_args.speculative_num_steps) != 1: + elif int(cfg.speculative_num_steps) != 1: logger.warning( "DSpark only supports speculative_num_steps == 1; overriding speculative_num_steps=%s to 1.", - server_args.speculative_num_steps, + cfg.speculative_num_steps, ) declare_resolution( server_args, @@ -430,16 +435,16 @@ def _handle_dspark(server_args: ServerArgs) -> None: speculative_num_steps=1, ) - if server_args.speculative_eagle_topk is None: + if cfg.speculative_eagle_topk is None: declare_resolution( server_args, "_handle_dspark", speculative_eagle_topk=1, ) - elif int(server_args.speculative_eagle_topk) != 1: + elif int(cfg.speculative_eagle_topk) != 1: logger.warning( "DSpark only supports speculative_eagle_topk == 1; overriding speculative_eagle_topk=%s to 1.", - server_args.speculative_eagle_topk, + cfg.speculative_eagle_topk, ) declare_resolution( server_args, @@ -463,17 +468,17 @@ def _handle_dspark(server_args: ServerArgs) -> None: ) gamma: Optional[int] = None - if server_args.speculative_dspark_block_size is not None: - if int(server_args.speculative_dspark_block_size) <= 0: + if cfg.speculative_dspark_block_size is not None: + if int(cfg.speculative_dspark_block_size) <= 0: raise ValueError( "DSpark requires --speculative-dspark-block-size to be positive, " - f"got {server_args.speculative_dspark_block_size}." + f"got {cfg.speculative_dspark_block_size}." ) - gamma = int(server_args.speculative_dspark_block_size) + gamma = int(cfg.speculative_dspark_block_size) else: if draft_config is not None: gamma = draft_config.resolve_gamma(default=None) - if gamma is None and server_args.speculative_num_draft_tokens is None: + if gamma is None and cfg.speculative_num_draft_tokens is None: gamma = DEFAULT_DSPARK_GAMMA logger.warning( "DSpark gamma is not set; defaulting to %d.", @@ -483,13 +488,13 @@ def _handle_dspark(server_args: ServerArgs) -> None: if gamma is not None: verify_window = int(gamma) + 1 if ( - server_args.speculative_num_draft_tokens is not None - and int(server_args.speculative_num_draft_tokens) != verify_window + cfg.speculative_num_draft_tokens is not None + and int(cfg.speculative_num_draft_tokens) != verify_window ): raise ValueError( "DSpark speculative_num_draft_tokens must equal gamma + 1 " f"(= {verify_window} for gamma={gamma}), but got " - f"speculative_num_draft_tokens={server_args.speculative_num_draft_tokens}." + f"speculative_num_draft_tokens={cfg.speculative_num_draft_tokens}." ) declare_resolution( server_args, @@ -497,18 +502,18 @@ def _handle_dspark(server_args: ServerArgs) -> None: speculative_num_draft_tokens=verify_window, ) - if server_args.speculative_num_draft_tokens is None: + if cfg.speculative_num_draft_tokens is None: raise ValueError( "DSpark could not resolve speculative_num_draft_tokens; set " "--speculative-dspark-block-size (= gamma)." ) - if int(server_args.speculative_num_draft_tokens) < 2: + if int(cfg.speculative_num_draft_tokens) < 2: raise ValueError( "DSpark speculative_num_draft_tokens must be >= 2 (= gamma + 1), " - f"got {server_args.speculative_num_draft_tokens}." + f"got {cfg.speculative_num_draft_tokens}." ) - if server_args.max_running_requests is None: + if cfg.max_running_requests is None: declare_resolution( server_args, "_handle_dspark", @@ -518,7 +523,7 @@ def _handle_dspark(server_args: ServerArgs) -> None: "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests." ) - if server_args.enable_mixed_chunk: + if cfg.enable_mixed_chunk: declare_resolution( server_args, "_handle_dspark", @@ -535,7 +540,7 @@ def _handle_dspark(server_args: ServerArgs) -> None: ragged_mode = read_ragged_verify_mode() if ( - server_args.speculative_dspark_align_verify_tokens_to_graph_tier + cfg.speculative_dspark_align_verify_tokens_to_graph_tier and ragged_mode is not RaggedVerifyMode.COMPACT ): logger.warning( @@ -544,10 +549,7 @@ def _handle_dspark(server_args: ServerArgs) -> None: "a no-op.", ragged_mode.value, ) - if ( - server_args.speculative_dspark_sps_table_path - and ragged_mode is RaggedVerifyMode.STATIC - ): + if cfg.speculative_dspark_sps_table_path and ragged_mode is RaggedVerifyMode.STATIC: logger.warning( "--speculative-dspark-sps-table-path feeds the ragged-verify budget " "scheduler, which is off under SGLANG_RAGGED_VERIFY_MODE=static; it " @@ -561,6 +563,7 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None: Consumed by ModelRunner's `is_draft_worker` override (one backend for all draft modes). """ + cfg = resolving_view(server_args) from sglang.srt.utils import is_hip supported_draft_backends = ( @@ -574,7 +577,7 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None: # Use triton on ROCm (no FlashInfer), flashinfer on CUDA. fallback_backend = "triton" if is_hip() else "flashinfer" - draft_backend = server_args.speculative_draft_attention_backend + draft_backend = cfg.speculative_draft_attention_backend if draft_backend is None: from sglang.srt.arg_groups.overrides import ( attention_backends_of, @@ -589,10 +592,10 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None: from sglang.srt.utils.hf_transformers_utils import get_config draft_hf_config = get_config( - server_args.speculative_draft_model_path, - trust_remote_code=server_args.trust_remote_code, - revision=server_args.speculative_draft_model_revision, - model_override_args=json.loads(server_args.json_model_override_args), + cfg.speculative_draft_model_path, + trust_remote_code=cfg.trust_remote_code, + revision=cfg.speculative_draft_model_revision, + model_override_args=json.loads(cfg.json_model_override_args), ) draft_text_config = ( getattr(draft_hf_config, "text_config", None) or draft_hf_config @@ -635,7 +638,8 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None: def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None: - if server_args.max_running_requests is None: + cfg = resolving_view(server_args) + if cfg.max_running_requests is None: declare_resolution( server_args, "_handle_frozen_kv_mtp", @@ -645,7 +649,7 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None: "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests." ) - if server_args.enable_mixed_chunk: + if cfg.enable_mixed_chunk: declare_resolution( server_args, "_handle_frozen_kv_mtp", @@ -658,13 +662,14 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None: def _handle_eagle_family(server_args: ServerArgs) -> None: + cfg = resolving_view(server_args) from sglang.srt.arg_groups.overrides import ( attention_backends_of, resolved_view, ) if ( - server_args.speculative_algorithm == "STANDALONE" + cfg.speculative_algorithm == "STANDALONE" and resolved_view(server_args).enable_dp_attention ): # TODO: support dp attention for standalone speculative decoding @@ -672,7 +677,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: "Currently standalone speculative decoding does not support dp attention." ) - if server_args.max_running_requests is None: + if cfg.max_running_requests is None: declare_resolution( server_args, "_handle_eagle_family", @@ -690,7 +695,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: "speculative decoding." ) - if server_args.enable_mixed_chunk: + if cfg.enable_mixed_chunk: declare_resolution( server_args, "_handle_eagle_family", @@ -716,16 +721,16 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: "PixtralForConditionalGeneration", "HYV3ForCausalLM", ]: - if server_args.speculative_draft_model_path is None: + if cfg.speculative_draft_model_path is None: declare_resolution( server_args, "_handle_eagle_family", - speculative_draft_model_path=server_args.model_path, + speculative_draft_model_path=cfg.model_path, ) declare_resolution( server_args, "_handle_eagle_family", - speculative_draft_model_revision=server_args.revision, + speculative_draft_model_revision=cfg.revision, ) else: if model_arch not in [ @@ -736,13 +741,10 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: "DeepSeek MTP does not require setting speculative_draft_model_path." ) - if ( - not server_args.speculative_adaptive - and server_args.speculative_num_steps is None - ): + if not cfg.speculative_adaptive and cfg.speculative_num_steps is None: assert ( - server_args.speculative_eagle_topk is None - and server_args.speculative_num_draft_tokens is None + cfg.speculative_eagle_topk is None + and cfg.speculative_num_draft_tokens is None ) steps, topk, draft_tokens = _auto_choose_speculative_params( @@ -757,29 +759,29 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: ) if "trtllm_mha" in attention_backends_of(resolved_view(server_args)): - if server_args.speculative_eagle_topk > 1: + if cfg.speculative_eagle_topk > 1: raise ValueError( "trtllm_mha backend only supports topk = 1 for speculative decoding." ) - if server_args.speculative_use_rejection_sampling: + if cfg.speculative_use_rejection_sampling: # Resolved alias by now: NEXTN -> EAGLE, Gemma4 draft -> FROZEN_KV_MTP. # Only the EAGLE/EAGLE3 draft workers emit a target-vocab proposal that # the rejection-sampling kernel consumes; everything else (STANDALONE, # FROZEN_KV_MTP, NGRAM, DFLASH) is unsupported. - if server_args.speculative_algorithm not in ("EAGLE", "EAGLE3"): + if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3"): raise NotImplementedError( "--speculative-use-rejection-sampling is only supported for " "EAGLE / EAGLE3 / NEXTN, not " - f"speculative_algorithm={server_args.speculative_algorithm}." + f"speculative_algorithm={cfg.speculative_algorithm}." ) - if server_args.speculative_eagle_topk != 1: + if cfg.speculative_eagle_topk != 1: raise ValueError( "--speculative-use-rejection-sampling requires --speculative-eagle-topk=1." ) if ( - server_args.speculative_accept_threshold_single != 1.0 - or server_args.speculative_accept_threshold_acc != 1.0 + cfg.speculative_accept_threshold_single != 1.0 + or cfg.speculative_accept_threshold_acc != 1.0 ): raise ValueError( "--speculative-use-rejection-sampling is incompatible with " @@ -787,7 +789,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: "--speculative-accept-threshold-acc; rejection sampling ignores " "the accept thresholds." ) - if server_args.enable_deterministic_inference: + if cfg.enable_deterministic_inference: raise ValueError( "--speculative-use-rejection-sampling is incompatible with " "--enable-deterministic-inference; the sampling kernel draws " @@ -798,7 +800,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: if ( resolved_view(server_args).enable_multi_layer_eagle - and server_args.speculative_eagle_topk != 1 + and cfg.speculative_eagle_topk != 1 ): raise ValueError( "--speculative-use-rejection-sampling with multi-layer EAGLE " @@ -811,9 +813,8 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: ) if ( - server_args.speculative_eagle_topk == 1 - and server_args.speculative_num_draft_tokens - != server_args.speculative_num_steps + 1 + cfg.speculative_eagle_topk == 1 + and cfg.speculative_num_draft_tokens != cfg.speculative_num_steps + 1 ): logger.warning( "speculative_num_draft_tokens is adjusted to speculative_num_steps + 1 when speculative_eagle_topk == 1" @@ -821,7 +822,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: declare_resolution( server_args, "_handle_eagle_family", - speculative_num_draft_tokens=server_args.speculative_num_steps + 1, + speculative_num_draft_tokens=cfg.speculative_num_steps + 1, ) # topk > 1 + page_size > 1 needs the two-pass cascade draft-decode (shared prefix @@ -830,7 +831,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: _PAGE_TREE_SPEC_BACKENDS = ("flashinfer", "fa3", "triton") view = resolved_view(server_args) if ( - server_args.speculative_eagle_topk > 1 + cfg.speculative_eagle_topk > 1 and view.page_size > 1 and view.attention_backend not in _PAGE_TREE_SPEC_BACKENDS ): @@ -842,14 +843,15 @@ def _handle_eagle_family(server_args: ServerArgs) -> None: def _handle_ngram(server_args: ServerArgs) -> None: - if server_args.device not in ("cuda", "cpu"): + cfg = resolving_view(server_args) + if cfg.device not in ("cuda", "cpu"): raise ValueError( "Ngram speculative decoding only supports CUDA or CPU devices." ) _disable_overlap_schedule_for_cpu(server_args) - if server_args.max_running_requests is None: + if cfg.max_running_requests is None: declare_resolution( server_args, "_handle_ngram", @@ -867,9 +869,9 @@ def _handle_ngram(server_args: ServerArgs) -> None: declare_resolution( server_args, "_handle_ngram", - speculative_eagle_topk=server_args.speculative_ngram_max_bfs_breadth, + speculative_eagle_topk=cfg.speculative_ngram_max_bfs_breadth, ) - if server_args.speculative_num_draft_tokens is None: + if cfg.speculative_num_draft_tokens is None: declare_resolution( server_args, "_handle_ngram", @@ -879,31 +881,31 @@ def _handle_ngram(server_args: ServerArgs) -> None: "speculative_num_draft_tokens is set to 12 by default for ngram speculative decoding. " "You can override this by explicitly setting --speculative-num-draft-tokens." ) - if server_args.speculative_num_steps is None: + if cfg.speculative_num_steps is None: declare_resolution( server_args, "_handle_ngram", - speculative_num_steps=server_args.speculative_num_draft_tokens - // server_args.speculative_eagle_topk, + speculative_num_steps=cfg.speculative_num_draft_tokens + // cfg.speculative_eagle_topk, ) - if server_args.speculative_ngram_external_corpus_path is not None: - if server_args.speculative_ngram_external_sam_budget <= 0: + if cfg.speculative_ngram_external_corpus_path is not None: + if cfg.speculative_ngram_external_sam_budget <= 0: raise ValueError( "--speculative-ngram-external-sam-budget must be positive when " "--speculative-ngram-external-corpus-path is set." ) - if server_args.speculative_ngram_external_corpus_max_tokens <= 0: + if cfg.speculative_ngram_external_corpus_max_tokens <= 0: raise ValueError( "--speculative-ngram-external-corpus-max-tokens must be positive when " "--speculative-ngram-external-corpus-path is set." ) if ( - server_args.speculative_ngram_external_sam_budget - > server_args.speculative_num_draft_tokens - 1 + cfg.speculative_ngram_external_sam_budget + > cfg.speculative_num_draft_tokens - 1 ): raise ValueError( "speculative_ngram_external_sam_budget must be less than or equal to " - f"speculative_num_draft_tokens - 1 ({server_args.speculative_num_draft_tokens - 1})." + f"speculative_num_draft_tokens - 1 ({cfg.speculative_num_draft_tokens - 1})." ) logger.warning( "The mixed chunked prefill are disabled because of " @@ -914,12 +916,12 @@ def _handle_ngram(server_args: ServerArgs) -> None: view = resolved_view(server_args) if ( - server_args.speculative_eagle_topk > 1 + cfg.speculative_eagle_topk > 1 and view.page_size > 1 and view.attention_backend != "flashinfer" ): raise ValueError( - f"speculative_eagle_topk({server_args.speculative_eagle_topk}) > 1 " + f"speculative_eagle_topk({cfg.speculative_eagle_topk}) > 1 " f"with page_size({view.page_size}) > 1 is unstable " "and produces incorrect results for paged attention backends. " "This combination is only supported for the 'flashinfer' backend." @@ -950,31 +952,32 @@ def _maybe_disable_adaptive(server_args: ServerArgs) -> None: def _init_adaptive_speculative_params(server_args: ServerArgs) -> None: + cfg = resolving_view(server_args) from sglang.srt.speculative.adaptive_spec_params import ( resolve_candidate_steps_from_config, ) candidate_steps = resolve_candidate_steps_from_config( - cfg_path=server_args.speculative_adaptive_config, + cfg_path=cfg.speculative_adaptive_config, ) - if server_args.speculative_eagle_topk is None: + if cfg.speculative_eagle_topk is None: declare_resolution( server_args, "_init_adaptive_speculative_params", speculative_eagle_topk=1, ) - if server_args.speculative_num_steps is None: + if cfg.speculative_num_steps is None: declare_resolution( server_args, "_init_adaptive_speculative_params", speculative_num_steps=candidate_steps[len(candidate_steps) // 2], ) - if server_args.speculative_num_steps not in candidate_steps: + if cfg.speculative_num_steps not in candidate_steps: raise ValueError( - f"--speculative-num-steps={server_args.speculative_num_steps} " + f"--speculative-num-steps={cfg.speculative_num_steps} " f"is not in the adaptive config candidate_steps {candidate_steps}. " "Pass one of those values." ) @@ -982,7 +985,7 @@ def _init_adaptive_speculative_params(server_args: ServerArgs) -> None: declare_resolution( server_args, "_init_adaptive_speculative_params", - speculative_num_draft_tokens=server_args.speculative_num_steps + 1, + speculative_num_draft_tokens=cfg.speculative_num_steps + 1, ) @@ -992,7 +995,8 @@ def _auto_choose_speculative_params(server_args: ServerArgs, model_arch: str) -> You can tune them on your own models and prompts with scripts/playground/bench_speculative.py """ - if server_args.speculative_algorithm == "STANDALONE": + cfg = resolving_view(server_args) + if cfg.speculative_algorithm == "STANDALONE": return (3, 1, 4) if model_arch in ["LlamaForCausalLM"]: return (5, 4, 8) diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 32806be68..5ec73e541 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -589,46 +589,46 @@ class ModelConfig: context_length: Optional[int] = None, **kwargs, ): + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) quantization = ( - server_args.speculative_draft_model_quantization + cfg.speculative_draft_model_quantization if is_draft_model - else server_args.quantization + else cfg.quantization ) override_config_file = ( - server_args.decrypted_draft_config_file + cfg.decrypted_draft_config_file if is_draft_model - else server_args.decrypted_config_file + else cfg.decrypted_config_file ) return ModelConfig( - model_path=model_path or server_args.model_path, - trust_remote_code=server_args.trust_remote_code, - revision=model_revision or server_args.revision, + model_path=model_path or cfg.model_path, + trust_remote_code=cfg.trust_remote_code, + revision=model_revision or cfg.revision, context_length=( - context_length - if context_length is not None - else server_args.context_length + context_length if context_length is not None else cfg.context_length ), - model_override_args=server_args.json_model_override_args, - is_embedding=server_args.is_embedding, - enable_multimodal=server_args.enable_multimodal, - dtype=server_args.dtype, + model_override_args=cfg.json_model_override_args, + is_embedding=cfg.is_embedding, + enable_multimodal=cfg.enable_multimodal, + dtype=cfg.dtype, quantization=quantization, - model_impl=server_args.model_impl, - sampling_defaults=server_args.sampling_defaults, - quantize_and_serve=server_args.quantize_and_serve, + model_impl=cfg.model_impl, + sampling_defaults=cfg.sampling_defaults, + quantize_and_serve=cfg.quantize_and_serve, override_config_file=override_config_file, - is_multi_layer_eagle=server_args.enable_multi_layer_eagle, - language_only=server_args.language_only, - language_model_only=server_args.language_model_only, - encoder_only=server_args.encoder_only, + is_multi_layer_eagle=cfg.enable_multi_layer_eagle, + language_only=cfg.language_only, + language_model_only=cfg.language_model_only, + encoder_only=cfg.encoder_only, is_draft_model=is_draft_model, is_draft_quantization_explicit=( - is_draft_model - and server_args._speculative_draft_quantization_explicitly_set + is_draft_model and cfg._speculative_draft_quantization_explicitly_set ), - disable_hybrid_swa_memory=server_args.disable_hybrid_swa_memory, - model_config_parser=server_args.model_config_parser, - speculative_algorithm=server_args.speculative_algorithm, + disable_hybrid_swa_memory=cfg.disable_hybrid_swa_memory, + model_config_parser=cfg.model_config_parser, + speculative_algorithm=cfg.speculative_algorithm, **kwargs, ) diff --git a/python/sglang/srt/dllm/config.py b/python/sglang/srt/dllm/config.py index f0f2b9d11..a2206feb9 100644 --- a/python/sglang/srt/dllm/config.py +++ b/python/sglang/srt/dllm/config.py @@ -25,13 +25,16 @@ class DllmConfig: def from_server_args( server_args: ServerArgs, ): - if server_args.dllm_algorithm is None: + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) + if cfg.dllm_algorithm is None: return None model_config = ModelConfig.from_server_args( server_args, - model_path=server_args.model_path, - model_revision=server_args.revision, + model_path=cfg.model_path, + model_revision=cfg.revision, ) DLLM_PARAMS = { "LLaDA2MoeModelLM": {"block_size": 32, "mask_id": 156895}, @@ -48,13 +51,11 @@ class DllmConfig: raise RuntimeError(f"Unknown diffusion LLM: {arch}") max_running_requests = ( - 1 - if server_args.max_running_requests is None - else server_args.max_running_requests + 1 if cfg.max_running_requests is None else cfg.max_running_requests ) algorithm_config = {} - if server_args.dllm_algorithm_config is not None: + if cfg.dllm_algorithm_config is not None: try: import yaml except ImportError: @@ -62,17 +63,17 @@ class DllmConfig: "Please install PyYAML to use YAML config files. " "`pip install pyyaml`" ) - with open(server_args.dllm_algorithm_config, "r") as f: + with open(cfg.dllm_algorithm_config, "r") as f: algorithm_config = yaml.safe_load(f) # Parse common algorithm configurations block_size = algorithm_config.get("block_size", block_size) return DllmConfig( - algorithm=server_args.dllm_algorithm, + algorithm=cfg.dllm_algorithm, algorithm_config=algorithm_config, block_size=block_size, mask_id=mask_id, max_running_requests=max_running_requests, - first_done_first_out_mode=server_args.dllm_fdfo, + first_done_first_out_mode=cfg.dllm_fdfo, ) diff --git a/python/sglang/srt/hardware_backend/npu/utils.py b/python/sglang/srt/hardware_backend/npu/utils.py index 3a285f728..857ae8634 100644 --- a/python/sglang/srt/hardware_backend/npu/utils.py +++ b/python/sglang/srt/hardware_backend/npu/utils.py @@ -43,6 +43,9 @@ def set_default_server_args(args: "ServerArgs"): """ Set default server arguments for NPU backend. """ + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(args) # NPU only works with "ascend" attention backend for now declare_resolution( @@ -60,7 +63,7 @@ def set_default_server_args(args: "ServerArgs"): "set_default_server_args", decode_attention_backend="ascend", ) - if args.page_size is None: + if cfg.page_size is None: declare_resolution( args, "set_default_server_args", @@ -68,33 +71,33 @@ def set_default_server_args(args: "ServerArgs"): ) # NPU memory settings - decode = args.cuda_graph_config.decode + decode = cfg.cuda_graph_config.decode npu_mem = get_npu_memory_capacity() if npu_mem <= 32 * 1024: # Ascend 910B4,910B4_1 # (chunked_prefill_size 4k, max_bs 16 if tp < 4 else 64) - if args.chunked_prefill_size is None: + if cfg.chunked_prefill_size is None: declare_resolution( args, "set_default_server_args", chunked_prefill_size=4 * 1024, ) if decode.max_bs is None: - if args.tp_size < 4: + if cfg.tp_size < 4: decode.max_bs = 16 else: decode.max_bs = 64 elif npu_mem <= 64 * 1024: # Ascend 910B1,910B2,910B2C,910B3,910_9391,910_9392,910_9381,910_9382,910_9372,910_9362 # (chunked_prefill_size 8k, max_bs 64 if tp < 4 else 256) - if args.chunked_prefill_size is None: + if cfg.chunked_prefill_size is None: declare_resolution( args, "set_default_server_args", chunked_prefill_size=8 * 1024, ) if decode.max_bs is None: - if args.tp_size < 4: + if cfg.tp_size < 4: decode.max_bs = 64 else: decode.max_bs = 256 @@ -107,7 +110,7 @@ def set_default_server_args(args: "ServerArgs"): ) # handles hierarchical cache configs - if args.enable_hierarchical_cache: + if cfg.enable_hierarchical_cache: declare_resolution( args, "set_default_server_args", diff --git a/python/sglang/srt/layers/cp/base.py b/python/sglang/srt/layers/cp/base.py index 4191089ed..c57e33e97 100644 --- a/python/sglang/srt/layers/cp/base.py +++ b/python/sglang/srt/layers/cp/base.py @@ -239,18 +239,21 @@ _STRATEGY: Optional[ContextParallelStrategy] = None def init_cp_strategy(server_args: ServerArgs) -> None: """Bind the configured CP strategy for this process.""" + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) global _STRATEGY - if not getattr(server_args, "enable_prefill_cp", False): + if not cfg.enable_prefill_cp: _STRATEGY = None return - cp_size = getattr(server_args, "attn_cp_size", 1) + cp_size = cfg.attn_cp_size if cp_size <= 1: _STRATEGY = None return - kind = ContextParallelStrategyKind.from_string(server_args.cp_strategy) + kind = ContextParallelStrategyKind.from_string(cfg.cp_strategy) if kind == ContextParallelStrategyKind.ZIGZAG: from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy @@ -262,7 +265,7 @@ def init_cp_strategy(server_args: ServerArgs) -> None: else: raise ValueError( f"Unsupported cp_strategy kind {kind} for " - f"cp_strategy={server_args.cp_strategy!r}" + f"cp_strategy={cfg.cp_strategy!r}" ) diff --git a/python/sglang/srt/layers/cp/bcg.py b/python/sglang/srt/layers/cp/bcg.py index 6abbddcf0..6137ef091 100644 --- a/python/sglang/srt/layers/cp/bcg.py +++ b/python/sglang/srt/layers/cp/bcg.py @@ -42,12 +42,15 @@ if TYPE_CHECKING: def supports_prefill_cp_bcg(server_args: ServerArgs) -> bool: """Return whether the selected prefill-CP configuration supports BCG.""" + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) resolved = server_args._resolved() prefill_attention_backend, _ = server_args._resolved_attention_backends() return ( - server_args.enable_prefill_cp - and resolved.attn_cp_size == server_args.tp_size - and server_args.cp_strategy == "zigzag" + cfg.enable_prefill_cp + and resolved.attn_cp_size == cfg.tp_size + and cfg.cp_strategy == "zigzag" and prefill_attention_backend == "trtllm_mha" ) diff --git a/python/sglang/srt/lora/marlin_lora_temp/policy.py b/python/sglang/srt/lora/marlin_lora_temp/policy.py index 76259ead0..c05838fa1 100644 --- a/python/sglang/srt/lora/marlin_lora_temp/policy.py +++ b/python/sglang/srt/lora/marlin_lora_temp/policy.py @@ -14,6 +14,9 @@ def validate_experimental_sgl_marlin_server_args( server_args: Any, resolved_args: Any ) -> None: """Validate startup options before the experimental runner is constructed.""" + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) if resolved_args.ep_size > 1 and resolved_args.moe_a2a_backend != "none": raise ValueError("experimental_sgl_marlin EP requires --moe-a2a-backend none") @@ -21,16 +24,16 @@ def validate_experimental_sgl_marlin_server_args( # A provided adapter path implicitly enables LoRA later unless it was # explicitly disabled. No-LoRA delegates to the stock Marlin fused path. lora_enabled = bool(resolved_args.enable_lora) or ( - resolved_args.enable_lora is None and bool(server_args.lora_paths) + resolved_args.enable_lora is None and bool(cfg.lora_paths) ) if not lora_enabled: return - if not server_args.lora_use_virtual_experts: + if not cfg.lora_use_virtual_experts: raise ValueError( "experimental_sgl_marlin LoRA requires --lora-use-virtual-experts" ) - if server_args.lora_backend != "triton": + if cfg.lora_backend != "triton": # The temporary dense/sink kernels consume Triton SGEMM batch metadata # directly; other global backends are not adapted in this tree. raise ValueError("experimental_sgl_marlin LoRA requires --lora-backend triton") @@ -38,12 +41,12 @@ def validate_experimental_sgl_marlin_server_args( return if ( - server_args.init_expert_location != "trivial" - or server_args.ep_num_redundant_experts != 0 - or server_args.enable_eplb - or server_args.elastic_ep_backend is not None - or server_args.enable_elastic_expert_backup - or server_args.elastic_ep_rejoin + cfg.init_expert_location != "trivial" + or cfg.ep_num_redundant_experts != 0 + or cfg.enable_eplb + or cfg.elastic_ep_backend is not None + or cfg.enable_elastic_expert_backup + or cfg.elastic_ep_rejoin ): raise ValueError( "experimental_sgl_marlin EP requires trivial expert placement " diff --git a/python/sglang/srt/model_loader/expert_pack_runtime.py b/python/sglang/srt/model_loader/expert_pack_runtime.py index 173d890b0..cb7fda46b 100644 --- a/python/sglang/srt/model_loader/expert_pack_runtime.py +++ b/python/sglang/srt/model_loader/expert_pack_runtime.py @@ -196,10 +196,14 @@ def prepare_raw_kimi_server_args( server_args: Any, loader_config: dict[str, Any] ) -> None: """Resolve a raw GGUF model path into the normal loader inputs.""" - model_path = Path(server_args.model_path).expanduser() + + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) + model_path = Path(cfg.model_path).expanduser() if not model_path.is_file() or model_path.suffix.lower() != ".gguf": return - tokenizer_path = server_args.tokenizer_path + tokenizer_path = cfg.tokenizer_path if tokenizer_path and Path(tokenizer_path).expanduser() == model_path: tokenizer_path = None assets = ensure_kimi_assets( @@ -503,7 +507,11 @@ def prepare_raw_deepseek_server_args( server_args: Any, loader_config: dict[str, Any] ) -> None: """Resolve a raw DeepSeek V4 GGUF into metadata and Expert Pack inputs.""" - source = Path(server_args.model_path).expanduser().resolve(strict=True) + + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) + source = Path(cfg.model_path).expanduser().resolve(strict=True) if not source.is_file(): return repo = _repo_root() @@ -546,7 +554,11 @@ def prepare_raw_expert_pack_server_args( server_args: Any, loader_config: dict[str, Any] ) -> None: """Dispatch a raw GGUF to the model-specific expert-pack preparation path.""" - source = Path(server_args.model_path).expanduser() + + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) + source = Path(cfg.model_path).expanduser() if not source.is_file(): return name = source.name.upper() diff --git a/python/sglang/srt/parser/template_detection.py b/python/sglang/srt/parser/template_detection.py index d353aa1c6..13d5465a7 100644 --- a/python/sglang/srt/parser/template_detection.py +++ b/python/sglang/srt/parser/template_detection.py @@ -29,7 +29,7 @@ import jinja2.ext import jinja2.nodes import jinja2.sandbox -from sglang.srt.arg_groups.overrides import declare_late_resolution +from sglang.srt.arg_groups.overrides import declare_late_resolution, resolving_view logger = logging.getLogger(__name__) @@ -666,11 +666,12 @@ def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str, """The parsers the model architecture implies, for the fields still on auto.""" from sglang.srt.utils.hf_transformers_utils import get_config + cfg = resolving_view(server_args) config = get_config( - server_args.model_path, - trust_remote_code=server_args.trust_remote_code, - revision=getattr(server_args, "revision", None), - model_config_parser=getattr(server_args, "model_config_parser", "auto"), + cfg.model_path, + trust_remote_code=cfg.trust_remote_code, + revision=getattr(cfg, "revision", None), + model_config_parser=getattr(cfg, "model_config_parser", "auto"), ) architectures = getattr(config, "architectures", None) or [] arch = architectures[0] if architectures else "" @@ -708,17 +709,18 @@ def resolve_auto_parsers(server_args) -> None: the schedulers it forks, the HTTP server, and the tokenizer workers it is serialized for. """ + cfg = resolving_view(server_args) needs = tuple( attr for attr in ("reasoning_parser", "tool_call_parser") - if getattr(server_args, attr) == "auto" + if getattr(cfg, attr) == "auto" ) if not needs: return from sglang.srt.utils.hf_transformers_utils import get_tokenizer - chat_template_arg = getattr(server_args, "chat_template", None) + chat_template_arg = getattr(cfg, "chat_template", None) try: explicit_jinja_template = _load_explicit_jinja_template(chat_template_arg) except Exception as e: @@ -731,8 +733,8 @@ def resolve_auto_parsers(server_args) -> None: tokenizer = None try: tokenizer = get_tokenizer( - server_args.model_path, - trust_remote_code=server_args.trust_remote_code, + cfg.model_path, + trust_remote_code=cfg.trust_remote_code, ) except Exception as e: logger.warning(f"Failed to load tokenizer for auto-detection: {e}") diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index d8e483f7b..17a4afa49 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -48,6 +48,7 @@ from sglang.srt.arg_groups.overrides import ( mamba_extra_buffer_of, remote_instance_transfer_engine_of, resolved_view, + resolving_view, ) from sglang.srt.configs.embedding_model_spec import BCGPrefillPolicy from sglang.srt.configs.linear_attn_model_registry import get_linear_attn_spec_by_arch @@ -3839,6 +3840,8 @@ class ServerArgs: # _handle_model_specific_adjustments never runs. self._resolved_overrides = [] + cfg = resolving_view(self) + from sglang.srt.arg_groups.mega_moe_hook import handle_mega_moe handle_mega_moe(self) @@ -3850,7 +3853,7 @@ class ServerArgs: # Reject an explicitly enabled but incompatible hardware runtime before # model path resolution, downloads, or the dummy-model short circuit. self._handle_hardware_runtime_validation() - if self.model_path.lower() in ["none", "dummy"]: + if cfg.model_path.lower() in ["none", "dummy"]: return self._handle_model_source_paths() @@ -3911,8 +3914,7 @@ class ServerArgs: current_platform.apply_server_args_defaults, ) - # Get GPU memory capacity, which is a common dependency for several configuration steps. - gpu_mem = get_device_memory_capacity(self.device) + gpu_mem = get_device_memory_capacity(cfg.device) # Handle memory-related, chunked prefill, and CUDA graph batch size configurations. self._handle_gpu_memory_settings(gpu_mem) @@ -4029,12 +4031,13 @@ class ServerArgs: materialize_declarations(self) def _handle_return_hidden_states_mode(self): - if self.return_hidden_states_mode not in (None, "last", "full"): + cfg = resolving_view(self) + if cfg.return_hidden_states_mode not in (None, "last", "full"): raise ValueError( "return_hidden_states_mode must be one of: None, 'last', or 'full'." ) - if self.return_hidden_states_mode is None: - if self.enable_return_hidden_states: + if cfg.return_hidden_states_mode is None: + if cfg.enable_return_hidden_states: self._declare( "_handle_return_hidden_states_mode", return_hidden_states_mode="full", @@ -4046,7 +4049,8 @@ class ServerArgs: ) def _handle_model_capability_adjustments(self): - if parse_connector_type(self.model_path) == ConnectorType.INSTANCE: + cfg = resolving_view(self) + if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE: return from sglang.srt.arg_groups.overrides import ( _hrm_text_attention_force, @@ -4083,8 +4087,8 @@ class ServerArgs: ) # cuda_graph_config was already parsed from the legacy boolean, so # flipping the boolean alone would not stop graph capture. - self.cuda_graph_config.decode.backend = Backend.DISABLED - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED logger.warning( "HRM-Text (prefix_lm) detected: forcing --attention-backend " "triton, --chunked-prefill-size -1, --disable-radix-cache, and " @@ -4110,7 +4114,7 @@ class ServerArgs: if ( embedding_model_spec is not None and embedding_model_spec.auto_enable_embedding - and not self.is_embedding + and not cfg.is_embedding ): self._declare( "_handle_model_capability_adjustments", @@ -4151,7 +4155,7 @@ class ServerArgs: enable_tokenizer_batch_encode=True, ) requested_prefill_backend = ( - self.prefill_attention_backend or self.attention_backend + cfg.prefill_attention_backend or cfg.attention_backend ) if ( is_cuda() @@ -4167,15 +4171,15 @@ class ServerArgs: prefill_only_disable_kv_cache=True, ) self._validate_prefill_only_disable_kv_cache_args() - self.cuda_graph_config.decode.backend = Backend.DISABLED - if is_cuda() and self.cuda_graph_config.prefill.backend != Backend.DISABLED: - self.cuda_graph_config.prefill.backend = Backend.BREAKABLE + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + if is_cuda() and cfg.cuda_graph_config.prefill.backend != Backend.DISABLED: + cfg.cuda_graph_config.prefill.backend = Backend.BREAKABLE # CUDA-graph sizing has already run by this point and derives # its generic maximum from the 8K chunked-prefill default. # On the Hopper/Blackwell FA raw-K/V path, raise the unlocked # default to a full eight-way 2K embedding batch; callers can # still override this for larger aggregate prefills. - prefill_config = self.cuda_graph_config.prefill + prefill_config = cfg.cuda_graph_config.prefill # Unit-level capability tests may invoke this hook without # running the full CUDA-graph configuration parser, which is # where this internal lock set is normally initialized. @@ -4199,7 +4203,7 @@ class ServerArgs: elif not is_cuda(): # BCG is CUDA-only. Other graph backends do not support this # encoder-style prefill, so retain the eager Triton path. - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED logger.info( "EmbeddingGemma detected: disabling radix cache and chunked " "prefill; using breakable CUDA graph for CUDA prefill." @@ -4220,13 +4224,14 @@ class ServerArgs: def _handle_model_source_paths(self): """Prepare metadata for model paths backed by remote object stores.""" + cfg = resolving_view(self) self._resolve_hf_gguf_model_path() seen_paths = set() for model_path in ( - self.model_path, - self.tokenizer_path, - self.speculative_draft_model_path, + cfg.model_path, + cfg.tokenizer_path, + cfg.speculative_draft_model_path, ): if ( model_path is not None @@ -4244,20 +4249,21 @@ class ServerArgs: handle_pd_disaggregation(self) def _handle_dcp_validation(self): - if self.dcp_size < 1: + cfg = resolving_view(self) + if cfg.dcp_size < 1: raise ValueError( "Decode context parallel size (--dcp-size / " "--decode-context-parallel-size) must be >= 1, but got " - f"dcp_size={self.dcp_size}." + f"dcp_size={cfg.dcp_size}." ) - if self.dcp_comm_backend in ("a2a", "fi_a2a") and self.dcp_size <= 1: + if cfg.dcp_comm_backend in ("a2a", "fi_a2a") and cfg.dcp_size <= 1: raise ValueError( - f"--dcp-comm-backend {self.dcp_comm_backend} only affects the " + f"--dcp-comm-backend {cfg.dcp_comm_backend} only affects the " "decode context-parallel attention reduction and therefore " "requires --dcp-size / --decode-context-parallel-size > 1, but " - f"got dcp_size={self.dcp_size}." + f"got dcp_size={cfg.dcp_size}." ) - if self.dcp_comm_backend == "fi_a2a" and not is_cuda(): + if cfg.dcp_comm_backend == "fi_a2a" and not is_cuda(): raise ValueError( "--dcp-comm-backend fi_a2a delegates the exchange to FlashInfer's " "MNNVL All-to-All kernel, which requires an NVIDIA CUDA platform " @@ -4265,23 +4271,22 @@ class ServerArgs: "authoritative fabric probe runs at model-runner init; use 'a2a' " "or 'ag_rs' on clusters without MNNVL." ) - if self.dcp_replicate_q_proj: - if self.dcp_size <= 1: + if cfg.dcp_replicate_q_proj: + if cfg.dcp_size <= 1: raise ValueError("--dcp-replicate-q-proj requires --dcp-size > 1.") - if self.dcp_comm_backend not in ("a2a", "fi_a2a"): + if cfg.dcp_comm_backend not in ("a2a", "fi_a2a"): raise ValueError( "--dcp-replicate-q-proj only applies to the a2a/fi_a2a DCP " "communication backend (it removes the head-dim Q all-gather); " - f"got --dcp-comm-backend={self.dcp_comm_backend}." + f"got --dcp-comm-backend={cfg.dcp_comm_backend}." ) def _handle_load_balance_method(self): - if self.disaggregation_mode not in ("null", "prefill", "decode"): - raise ValueError( - f"Invalid disaggregation_mode={self.disaggregation_mode!r}" - ) + cfg = resolving_view(self) + if cfg.disaggregation_mode not in ("null", "prefill", "decode"): + raise ValueError(f"Invalid disaggregation_mode={cfg.disaggregation_mode!r}") - if self.load_balance_method == "auto": + if cfg.load_balance_method == "auto": # Default behavior: # - non-PD: round_robin # - PD prefill: follow_bootstrap_room @@ -4290,7 +4295,7 @@ class ServerArgs: "_handle_load_balance_method", load_balance_method=( "follow_bootstrap_room" - if self.disaggregation_mode == "prefill" + if cfg.disaggregation_mode == "prefill" else "round_robin" ), ) @@ -4298,47 +4303,48 @@ class ServerArgs: def _handle_ssl_validation(self): """Ensure SSL arguments are consistent and referenced files exist.""" - if self.ssl_keyfile and not self.ssl_certfile: + cfg = resolving_view(self) + if cfg.ssl_keyfile and not cfg.ssl_certfile: raise ValueError( "--ssl-keyfile requires --ssl-certfile to be specified as well." ) - if self.ssl_certfile and not self.ssl_keyfile: + if cfg.ssl_certfile and not cfg.ssl_keyfile: raise ValueError( "--ssl-certfile requires --ssl-keyfile to be specified as well." ) - if not self.ssl_certfile and not self.ssl_keyfile: - if self.ssl_ca_certs: + if not cfg.ssl_certfile and not cfg.ssl_keyfile: + if cfg.ssl_ca_certs: raise ValueError( "--ssl-ca-certs has no effect without --ssl-certfile and --ssl-keyfile." ) - if self.ssl_keyfile_password: + if cfg.ssl_keyfile_password: raise ValueError( "--ssl-keyfile-password has no effect without --ssl-certfile and --ssl-keyfile." ) # Validate files exist early to avoid late failures after model loading. - if self.ssl_keyfile and not os.path.isfile(self.ssl_keyfile): + if cfg.ssl_keyfile and not os.path.isfile(cfg.ssl_keyfile): raise ValueError( - f"SSL key file not found: '{self.ssl_keyfile}'. " + f"SSL key file not found: '{cfg.ssl_keyfile}'. " f"Please check the --ssl-keyfile path." ) - if self.ssl_certfile and not os.path.isfile(self.ssl_certfile): + if cfg.ssl_certfile and not os.path.isfile(cfg.ssl_certfile): raise ValueError( - f"SSL certificate file not found: '{self.ssl_certfile}'. " + f"SSL certificate file not found: '{cfg.ssl_certfile}'. " f"Please check the --ssl-certfile path." ) - if self.ssl_ca_certs and not os.path.isfile(self.ssl_ca_certs): + if cfg.ssl_ca_certs and not os.path.isfile(cfg.ssl_ca_certs): raise ValueError( - f"SSL CA certificates file not found: '{self.ssl_ca_certs}'. " + f"SSL CA certificates file not found: '{cfg.ssl_ca_certs}'. " f"Please check the --ssl-ca-certs path." ) - if self.enable_ssl_refresh and not (self.ssl_certfile and self.ssl_keyfile): + if cfg.enable_ssl_refresh and not (cfg.ssl_certfile and cfg.ssl_keyfile): raise ValueError( "--enable-ssl-refresh requires --ssl-certfile and --ssl-keyfile " "to be specified." ) - if self.enable_http2: - if not 0 < self.http2_max_concurrent_streams < 2**32: + if cfg.enable_http2: + if not 0 < cfg.http2_max_concurrent_streams < 2**32: raise ValueError( "--http2-max-concurrent-streams must be between 1 and " "4294967295." @@ -4352,7 +4358,7 @@ class ServerArgs: 'Install it with: pip install "sglang[http2]"' ) - if self.enable_ssl_refresh: + if cfg.enable_ssl_refresh: raise ValueError( "--enable-ssl-refresh is not supported with --enable-http2. " "Granian does not support SSL certificate hot-reloading. " @@ -4361,42 +4367,45 @@ class ServerArgs: def _handle_multimodal(self): """Validate mm_process_config structure before model loading.""" + cfg = resolving_view(self) if ( - self.mm_preprocess_cache_size_mb is not None - and self.mm_preprocess_cache_size_mb < 0 + cfg.mm_preprocess_cache_size_mb is not None + and cfg.mm_preprocess_cache_size_mb < 0 ): raise ValueError("mm_preprocess_cache_size_mb must be non-negative") - if self.mm_process_config is not None: - if not isinstance(self.mm_process_config, dict): + if cfg.mm_process_config is not None: + if not isinstance(cfg.mm_process_config, dict): raise TypeError( f"mm_process_config must be a dict, " - f"but got {type(self.mm_process_config)}" + f"but got {type(cfg.mm_process_config)}" ) for key in ("image", "video", "audio"): - if key in self.mm_process_config and not isinstance( - self.mm_process_config[key], dict + if key in cfg.mm_process_config and not isinstance( + cfg.mm_process_config[key], dict ): raise TypeError( f"mm_process_config['{key}'] must be a dict, " - f"but got {type(self.mm_process_config[key])}" + f"but got {type(cfg.mm_process_config[key])}" ) def _handle_media_url_security(self): """Normalize and publish the media URL policy before workers start.""" + cfg = resolving_view(self) self._declare( "_handle_media_url_security", allowed_media_domains=configure_media_url_security( - self.allowed_media_domains, - self.media_url_max_file_size_mb, + cfg.allowed_media_domains, + cfg.media_url_max_file_size_mb, ), ) def _handle_deprecated_args(self): - if self.disable_fast_image_processor: - if self.image_processor_backend not in {"auto", "pil"}: + cfg = resolving_view(self) + if cfg.disable_fast_image_processor: + if cfg.image_processor_backend not in {"auto", "pil"}: raise ValueError( "--disable-fast-image-processor conflicts with " - f"--image-processor-backend={self.image_processor_backend}." + f"--image-processor-backend={cfg.image_processor_backend}." ) logger.warning( "--disable-fast-image-processor is deprecated; use " @@ -4406,19 +4415,19 @@ class ServerArgs: # Handle deprecated tool call parsers deprecated_tool_call_parsers = {"qwen25": "qwen", "glm45": "glm"} - if self.tool_call_parser in deprecated_tool_call_parsers: + if cfg.tool_call_parser in deprecated_tool_call_parsers: logger.warning( - f"The tool_call_parser '{self.tool_call_parser}' is deprecated. Please use '{deprecated_tool_call_parsers[self.tool_call_parser]}' instead." + f"The tool_call_parser '{cfg.tool_call_parser}' is deprecated. Please use '{deprecated_tool_call_parsers[cfg.tool_call_parser]}' instead." ) self._declare( "_handle_deprecated_args", - tool_call_parser=deprecated_tool_call_parsers[self.tool_call_parser], + tool_call_parser=deprecated_tool_call_parsers[cfg.tool_call_parser], ) # When user passes --enable-flashinfer-allreduce-fusion, enable with auto backend if ( - self.enable_flashinfer_allreduce_fusion - and self.flashinfer_allreduce_fusion_backend is None + cfg.enable_flashinfer_allreduce_fusion + and cfg.flashinfer_allreduce_fusion_backend is None ): logger.warning( "--enable-flashinfer-allreduce-fusion is deprecated. " @@ -4450,7 +4459,7 @@ class ServerArgs: self._declare("_handle_deprecated_args", **renamed) # --grpc-mode is a deprecated alias for --smg-grpc-mode. - if self.grpc_mode and not self.smg_grpc_mode: + if cfg.grpc_mode and not cfg.smg_grpc_mode: logger.warning( "--grpc-mode is deprecated and will be removed in a future " "version. Use --smg-grpc-mode for the legacy SMG gRPC server, " @@ -4466,7 +4475,7 @@ class ServerArgs: self.grpc_worker_threads = envs.SGLANG_GRPC_WORKER_THREADS.get() grpc_port_env = envs.SGLANG_GRPC_PORT.get() - if self.grpc_port is None and grpc_port_env is not None: + if cfg.grpc_port is None and grpc_port_env is not None: self._declare( "_handle_deprecated_args", grpc_port=grpc_port_env, @@ -4474,18 +4483,18 @@ class ServerArgs: # Legacy SMG defaults its port to --port + 10000. Derive/validate only # when gRPC is in use, so HTTP-only high ports don't fail validation. - legacy_grpc = self.smg_grpc_mode or self.grpc_mode - if legacy_grpc and self.grpc_port is None: + legacy_grpc = cfg.smg_grpc_mode or cfg.grpc_mode + if legacy_grpc and cfg.grpc_port is None: self._declare( "_handle_deprecated_args", - grpc_port=self.port + 10000, + grpc_port=cfg.port + 10000, ) - if self.grpc_port is not None: - if not (1 <= self.grpc_port <= 65535): + if cfg.grpc_port is not None: + if not (1 <= cfg.grpc_port <= 65535): raise ValueError( "--grpc-port / SGLANG_GRPC_PORT " - f"({self.grpc_port}) must be between 1 and 65535" + f"({cfg.grpc_port}) must be between 1 and 65535" ) if self.grpc_worker_threads < 1: raise ValueError( @@ -4495,41 +4504,41 @@ class ServerArgs: # Native gRPC is incompatible with launch paths it doesn't wire into. # Legacy takes precedence over grpc_port, keeping re-runs idempotent. - native_grpc = self.grpc_port is not None and not legacy_grpc - if self.sidecar_args is not None: - if self.sidecar is None: + native_grpc = cfg.grpc_port is not None and not legacy_grpc + if cfg.sidecar_args is not None: + if cfg.sidecar is None: raise ValueError("--sidecar-args requires --sidecar.") - if not isinstance(self.sidecar_args, list) or not all( - isinstance(arg, str) for arg in self.sidecar_args + if not isinstance(cfg.sidecar_args, list) or not all( + isinstance(arg, str) for arg in cfg.sidecar_args ): raise ValueError("--sidecar-args must be a JSON array of strings.") - if self.sidecar is not None: - if not self.sidecar.strip(): + if cfg.sidecar is not None: + if not cfg.sidecar.strip(): raise ValueError("--sidecar must not be empty.") if legacy_grpc: raise ValueError( "--sidecar requires SGLang's native gRPC server; " "it cannot be combined with --smg-grpc-mode/--grpc-mode." ) - if self.grpc_port is None: + if cfg.grpc_port is None: raise ValueError("--sidecar requires --grpc-port or SGLANG_GRPC_PORT.") if native_grpc: - if self.use_ray: + if cfg.use_ray: raise ValueError( "--grpc-port is not supported with --use-ray: the Ray " "serve launch path does not start the native gRPC server." ) - if self.encoder_only: + if cfg.encoder_only: raise ValueError( "--grpc-port is not supported with --encoder-only: " "encoder disaggregation uses its own server." ) - if self.tokenizer_worker_num > 1: + if cfg.tokenizer_worker_num > 1: raise ValueError( "Native gRPC does not yet support --tokenizer-worker-num > 1. " "Unset --grpc-port or set --tokenizer-worker-num 1." ) - if self.api_key or self.admin_api_key: + if cfg.api_key or cfg.admin_api_key: raise ValueError( "--grpc-port is incompatible with --api-key/--admin-api-key: " "the native gRPC listener bypasses HTTP auth middleware." @@ -4553,17 +4562,18 @@ class ServerArgs: ) def _handle_missing_default_values(self): - if self.tokenizer_path is None: + cfg = resolving_view(self) + if cfg.tokenizer_path is None: self._declare( "_handle_missing_default_values", - tokenizer_path=self.model_path, + tokenizer_path=cfg.model_path, ) - if self.served_model_name is None: + if cfg.served_model_name is None: self._declare( "_handle_missing_default_values", - served_model_name=self.model_path, + served_model_name=cfg.model_path, ) - if self.device is None: + if cfg.device is None: self._declare( "_handle_missing_default_values", device=get_device(), @@ -4571,14 +4581,14 @@ class ServerArgs: # strip device index from user if any (e.g. "cuda:0" -> "cuda") self._declare( "_handle_missing_default_values", - device=self.device.split(":")[0], + device=cfg.device.split(":")[0], ) - if self.random_seed is None: + if cfg.random_seed is None: self._declare( "_handle_missing_default_values", random_seed=random.randint(0, 1 << 30), ) - if self.mm_process_config is None: + if cfg.mm_process_config is None: self._declare("_handle_missing_default_values", mm_process_config={}) # Handle ModelScope model downloads @@ -4588,22 +4598,22 @@ class ServerArgs: # In speculative scenario: # - If `speculative_draft_model_quantization` is specified, the draft model uses this quantization method. # - Otherwise, the draft model defaults to the same quantization as the target model. - if self._speculative_draft_quantization_explicitly_set is None: + if cfg._speculative_draft_quantization_explicitly_set is None: self._declare( "_handle_missing_default_values", - _speculative_draft_quantization_explicitly_set=self.speculative_draft_model_quantization + _speculative_draft_quantization_explicitly_set=cfg.speculative_draft_model_quantization is not None, ) - if self.speculative_draft_model_quantization is None: + if cfg.speculative_draft_model_quantization is None: self._declare( "_handle_missing_default_values", - speculative_draft_model_quantization=self.quantization, + speculative_draft_model_quantization=cfg.quantization, ) # Resolve --quantization unquant before model config validation. Record # the explicit opt-out so later auto-detection does not re-enable # quantization. - if self.quantization == "unquant": + if cfg.quantization == "unquant": self._declare( "_handle_missing_default_values", quantization=None, @@ -4611,7 +4621,7 @@ class ServerArgs: self._quantization_explicitly_unset = True else: self._quantization_explicitly_unset = False - if self.speculative_draft_model_quantization == "unquant": + if cfg.speculative_draft_model_quantization == "unquant": self._declare( "_handle_missing_default_values", speculative_draft_model_quantization=None, @@ -4627,6 +4637,7 @@ class ServerArgs: plain repo ID. That resolution lives in :func:`sglang.srt.speculative.spec_utils.load_token_map`. """ + cfg = resolving_view(self) ms_root = None ms_snapshot_download = None @@ -4656,42 +4667,43 @@ class ServerArgs: if os.path.exists(cached): return cached # Check user-specified download dir - if self.download_dir: - alt = os.path.join(self.download_dir, path) + if cfg.download_dir: + alt = os.path.join(cfg.download_dir, path) if os.path.exists(alt): return alt # Cache miss — download from ModelScope hub return ms_snapshot_download( path, - cache_dir=self.download_dir, + cache_dir=cfg.download_dir, revision=revision, **({"ignore_patterns": ignore_patterns} if ignore_patterns else {}), ) self._declare( "_handle_modelscope_paths", - model_path=_resolve_or_download(self.model_path, revision=self.revision), + model_path=_resolve_or_download(cfg.model_path, revision=cfg.revision), ) self._declare( "_handle_modelscope_paths", tokenizer_path=_resolve_or_download( - self.tokenizer_path, + cfg.tokenizer_path, ignore_patterns=["*.bin", "*.safetensors"], - revision=self.revision, + revision=cfg.revision, ), ) - if self.speculative_draft_model_path: + if cfg.speculative_draft_model_path: self._declare( "_handle_modelscope_paths", speculative_draft_model_path=_resolve_or_download( - self.speculative_draft_model_path, - revision=self.speculative_draft_model_revision or "main", + cfg.speculative_draft_model_path, + revision=cfg.speculative_draft_model_revision or "main", ), ) def _handle_hpu_backends(self): - if self.device == "hpu": + cfg = resolving_view(self) + if cfg.device == "hpu": self._declare( "_handle_hpu_backends", attention_backend="torch_native", @@ -4702,8 +4714,9 @@ class ServerArgs: ) def _handle_cpu_backends(self): - if self.device == "cpu": - if self.attention_backend is None: + cfg = resolving_view(self) + if cfg.device == "cpu": + if cfg.attention_backend is None: self._declare( "_handle_cpu_backends", attention_backend=( @@ -4723,21 +4736,23 @@ class ServerArgs: use_mlx() def _handle_npu_backends(self): - if self.device == "npu": + cfg = resolving_view(self) + if cfg.device == "npu": from sglang.srt.hardware_backend.npu.utils import set_default_server_args set_default_server_args(self) - current = self.cuda_graph_config.prefill.tc_compiler + current = cfg.cuda_graph_config.prefill.tc_compiler if current is not None and current != "eager": logger.warning( "At this moment Ascend platform only support prefill graph compilation with " "cuda_graph_config[prefill].tc_compiler='eager'." ) - self.cuda_graph_config.prefill.tc_compiler = "eager" + cfg.cuda_graph_config.prefill.tc_compiler = "eager" def _handle_mps_backends(self): - if self.device == "mps": + cfg = resolving_view(self) + if cfg.device == "mps": if not use_mlx(): self._declare( "_handle_mps_backends", @@ -4745,22 +4760,23 @@ class ServerArgs: ) def _handle_xpu_backends(self): - if self.device == "xpu": + cfg = resolving_view(self) + if cfg.device == "xpu": # Decode graph is opt-in on XPU: unless the user explicitly set # --cuda-graph-backend-decode (or --cuda-graph-config), keep it # disabled so the default startup doesn't require graph capture. if (Phase.DECODE, "backend") not in self._cuda_graph_config_locked: - self.cuda_graph_config.decode.backend = Backend.DISABLED - elif self.cuda_graph_config.decode.backend not in ( + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + elif cfg.cuda_graph_config.decode.backend not in ( Backend.DISABLED, Backend.FULL, ): logger.warning( "XPU platform only supports decode backend 'full'; " "disabling unsupported decode backend '%s'.", - self.cuda_graph_config.decode.backend, + cfg.cuda_graph_config.decode.backend, ) - self.cuda_graph_config.decode.backend = Backend.DISABLED + cfg.cuda_graph_config.decode.backend = Backend.DISABLED # ------------------------------------------------------------------ # CUDA graph configuration resolution @@ -4771,10 +4787,11 @@ class ServerArgs: auto-disabled for this multimodal arch, and declarative model overrides materialize too late to steer cuda-graph resolution. Honors an explicit --cuda-graph-backend-prefill / --disable-prefill-cuda-graph.""" + cfg = resolving_view(self) if ( - self.cuda_graph_backend_prefill is not None - or self.disable_prefill_cuda_graph - or parse_connector_type(self.model_path) == ConnectorType.INSTANCE + cfg.cuda_graph_backend_prefill is not None + or cfg.disable_prefill_cuda_graph + or parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE ): return arch = self.get_model_config().hf_config.architectures[0] @@ -4788,9 +4805,10 @@ class ServerArgs: ) def _apply_muse_glimmer_prefill_cuda_graph_max_bs_default(self): + cfg = resolving_view(self) if ( - self.cuda_graph_max_bs_prefill is not None - or parse_connector_type(self.model_path) == ConnectorType.INSTANCE + cfg.cuda_graph_max_bs_prefill is not None + or parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE ): return arch = self.get_model_config().hf_config.architectures[0] @@ -4801,6 +4819,7 @@ class ServerArgs: ) def _handle_cuda_graph_config(self): + cfg = resolving_view(self) from sglang.srt.arg_groups.kimi_k3_hook import disable_kimi_k3_symm_mem self._parse_cuda_graph_config() @@ -4814,7 +4833,7 @@ class ServerArgs: # Warn on the final resolved config (not inside the compat cascade — # that path is skipped when the user explicitly sets the backend, # which is the only way to get 'full' for prefill today). - if self.cuda_graph_config.prefill.backend == Backend.FULL: + if cfg.cuda_graph_config.prefill.backend == Backend.FULL: logger.warning( "cuda_graph_config[prefill].backend='full' is experimental. " "Use breakable or tc_piecewise for production workloads." @@ -4822,16 +4841,17 @@ class ServerArgs: def _apply_deepep_adjustments(self): """Config adjustments required by the DeepEP a2a backend.""" + cfg = resolving_view(self) if resolved_view(self).moe_a2a_backend != "deepep": return # Non-multiple-of-8 prefill buckets can hang DeepEP a2a capture under # breakable CUDA graph - if self.cuda_graph_config.prefill.backend == Backend.BREAKABLE: - bs = self.cuda_graph_config.prefill.bs + if cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE: + bs = cfg.cuda_graph_config.prefill.bs if bs is None: # 2048 = documented prefill default; max_bs unresolved here. - max_bs = self.cuda_graph_config.prefill.max_bs or 2048 + max_bs = cfg.cuda_graph_config.prefill.max_bs or 2048 bs = self._generate_prefill_cuda_graph_batch_sizes(max_bs) aligned = sorted({((b + 7) // 8) * 8 for b in bs}) if aligned != sorted(bs): @@ -4841,8 +4861,8 @@ class ServerArgs: sorted(bs), aligned, ) - self.cuda_graph_config.prefill.bs = aligned - self.cuda_graph_config.prefill.max_bs = aligned[-1] + cfg.cuda_graph_config.prefill.bs = aligned + cfg.cuda_graph_config.prefill.max_bs = aligned[-1] def _parse_cuda_graph_config(self): """Resolve cuda_graph_config from explicit JSON, per-phase @@ -4853,7 +4873,8 @@ class ServerArgs: auto-disable cascade respects this lock (the old --enforce-piecewise-cuda-graph semantics generalized). """ - raw_input = self.cuda_graph_config + cfg = resolving_view(self) + raw_input = cfg.cuda_graph_config if isinstance(raw_input, CudaGraphConfig): explicit_input = raw_input.to_dict() else: @@ -4866,36 +4887,36 @@ class ServerArgs: locked.add((phase, key)) # ---- Legacy global flags (lowest precedence above defaults) ---- - if self.disable_cuda_graph: + if cfg.disable_cuda_graph: _set(Phase.DECODE, "backend", Backend.DISABLED) _set(Phase.PREFILL, "backend", Backend.DISABLED) # ---- Boolean per-phase off-switches ---- # Below the explicit backend selectors so --cuda-graph-backend-* # wins if both are given. - if self.disable_prefill_cuda_graph: + if cfg.disable_prefill_cuda_graph: _set(Phase.PREFILL, "backend", Backend.DISABLED) - if self.disable_decode_cuda_graph: + if cfg.disable_decode_cuda_graph: _set(Phase.DECODE, "backend", Backend.DISABLED) # ---- Per-phase convenience flags ---- - if self.cuda_graph_backend_decode is not None: - _set(Phase.DECODE, "backend", self.cuda_graph_backend_decode) - if self.cuda_graph_backend_prefill is not None: - _set(Phase.PREFILL, "backend", self.cuda_graph_backend_prefill) - if self.cuda_graph_max_bs_decode is not None: - _set(Phase.DECODE, "max_bs", self.cuda_graph_max_bs_decode) - if self.cuda_graph_max_bs_prefill is not None: - _set(Phase.PREFILL, "max_bs", self.cuda_graph_max_bs_prefill) - if self.cuda_graph_bs_decode is not None: - _set(Phase.DECODE, "bs", self.cuda_graph_bs_decode) - if self.cuda_graph_bs_prefill is not None: - _set(Phase.PREFILL, "bs", self.cuda_graph_bs_prefill) - if self.cuda_graph_tc_compiler is not None: + if cfg.cuda_graph_backend_decode is not None: + _set(Phase.DECODE, "backend", cfg.cuda_graph_backend_decode) + if cfg.cuda_graph_backend_prefill is not None: + _set(Phase.PREFILL, "backend", cfg.cuda_graph_backend_prefill) + if cfg.cuda_graph_max_bs_decode is not None: + _set(Phase.DECODE, "max_bs", cfg.cuda_graph_max_bs_decode) + if cfg.cuda_graph_max_bs_prefill is not None: + _set(Phase.PREFILL, "max_bs", cfg.cuda_graph_max_bs_prefill) + if cfg.cuda_graph_bs_decode is not None: + _set(Phase.DECODE, "bs", cfg.cuda_graph_bs_decode) + if cfg.cuda_graph_bs_prefill is not None: + _set(Phase.PREFILL, "bs", cfg.cuda_graph_bs_prefill) + if cfg.cuda_graph_tc_compiler is not None: # Written to both phases so the value is in place when TC_PIECEWISE # decode is implemented; today decode ignores it. - _set(Phase.DECODE, "tc_compiler", self.cuda_graph_tc_compiler) - _set(Phase.PREFILL, "tc_compiler", self.cuda_graph_tc_compiler) + _set(Phase.DECODE, "tc_compiler", cfg.cuda_graph_tc_compiler) + _set(Phase.PREFILL, "tc_compiler", cfg.cuda_graph_tc_compiler) # ---- Explicit JSON config (highest precedence) ---- for phase, phase_config in explicit_input.items(): @@ -4917,6 +4938,7 @@ class ServerArgs: prefill backend (this folds in the old --enforce-piecewise-cuda-graph contract). """ + cfg = resolving_view(self) if (Phase.PREFILL, "backend") in self._cuda_graph_config_locked: return @@ -4925,7 +4947,7 @@ class ServerArgs: # there instead. Archs also on the breakable allowlist keep it -- # this runs first, so piecewise would otherwise silently win. if ( - self.cuda_graph_config.prefill.backend == Backend.BREAKABLE + cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE and self.get_model_config().is_multimodal_piecewise_cuda_graph_supported and not self.get_model_config().is_multimodal_breakable_cuda_graph_supported # Keep trtllm_mla on the preferred breakable path, which now serves @@ -4936,27 +4958,29 @@ class ServerArgs: "Using tc_piecewise CUDA graph for validated multimodal " "decoder prefill." ) - self.cuda_graph_config.prefill.backend = Backend.TC_PIECEWISE + cfg.cuda_graph_config.prefill.backend = Backend.TC_PIECEWISE - if self.cuda_graph_config.prefill.backend == Backend.TC_PIECEWISE: + if cfg.cuda_graph_config.prefill.backend == Backend.TC_PIECEWISE: self._disable_tc_piecewise_cudagraph_if_incompatible() - elif self.cuda_graph_config.prefill.backend == Backend.BREAKABLE: + elif cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE: self._disable_breakable_cudagraph_if_incompatible() - elif self.cuda_graph_config.prefill.backend == Backend.FULL: + elif cfg.cuda_graph_config.prefill.backend == Backend.FULL: self._disable_full_prefill_cudagraph_if_incompatible() def _apply_cuda_graph_disaggregation_roles(self): - if self.disaggregation_mode == "prefill": + cfg = resolving_view(self) + if cfg.disaggregation_mode == "prefill": if (Phase.DECODE, "backend") not in self._cuda_graph_config_locked: - self.cuda_graph_config.decode.backend = Backend.DISABLED - elif self.disaggregation_mode == "decode": + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + elif cfg.disaggregation_mode == "decode": if (Phase.PREFILL, "backend") not in self._cuda_graph_config_locked: - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED def _disable_tc_piecewise_cudagraph_if_incompatible(self): """TcPiecewise (torch.compile + piecewise) is incompatible with these configurations. Most are torch.compile / dynamo limitations. """ + cfg = resolving_view(self) rules = [ ( @@ -4964,8 +4988,8 @@ class ServerArgs: lambda: self.get_model_config().is_piecewise_cuda_graph_disabled_model, ), ("DP attention", lambda: self._resolved().enable_dp_attention), - ("full torch.compile mode", lambda: self.enable_torch_compile), - ("pipeline parallelism (pp_size > 1)", lambda: self.pp_size > 1), + ("full torch.compile mode", lambda: cfg.enable_torch_compile), + ("pipeline parallelism (pp_size > 1)", lambda: cfg.pp_size > 1), ( "non-CUDA hardware (HIP/NPU/CPU/MPS/XPU)", lambda: is_hip() or is_npu() or is_cpu() or is_mps() or is_xpu(), @@ -4981,7 +5005,7 @@ class ServerArgs: ), # Dynamo blocks LoRA under tc_piecewise (per-batch LoRABatchInfo # rebinds break guards); breakable/full support LoRA. - ("LoRA", lambda: bool(self.lora_paths) or self.enable_lora), + ("LoRA", lambda: bool(cfg.lora_paths) or cfg.enable_lora), ( "multimodal model", lambda: self.get_model_config().is_multimodal @@ -4989,50 +5013,51 @@ class ServerArgs: ), ( "GGUF quantization", - lambda: self.load_format == "gguf" + lambda: cfg.load_format == "gguf" or resolved_view(self).quantization == "gguf" - or check_gguf_file(self.model_path), + or check_gguf_file(cfg.model_path), ), - ("DLLM (diffusion LLM)", lambda: self.dllm_algorithm is not None), + ("DLLM (diffusion LLM)", lambda: cfg.dllm_algorithm is not None), ( "CPU offload / hierarchical cache", - lambda: self.cpu_offload_gb > 0 or self.enable_hierarchical_cache, + lambda: cfg.cpu_offload_gb > 0 or cfg.enable_hierarchical_cache, ), ( "deterministic inference", - lambda: self.enable_deterministic_inference, + lambda: cfg.enable_deterministic_inference, ), - ("PD disaggregation", lambda: self.disaggregation_mode != "null"), - ("symmetric memory", lambda: self.enable_symm_mem), + ("PD disaggregation", lambda: cfg.disaggregation_mode != "null"), + ("symmetric memory", lambda: cfg.enable_symm_mem), ( "expert distribution recorder", - lambda: self.enable_eplb - or self.expert_distribution_recorder_mode is not None, + lambda: cfg.enable_eplb + or cfg.expert_distribution_recorder_mode is not None, ), ( "context parallel (attn_cp_size > 1)", lambda: self._resolved().attn_cp_size > 1, ), - ("CUDA graph debug mode", lambda: self.debug_cuda_graph), + ("CUDA graph debug mode", lambda: cfg.debug_cuda_graph), ( "DSA prefill context parallelism", - lambda: self.enable_dsa_prefill_context_parallel, + lambda: cfg.enable_dsa_prefill_context_parallel, ), # Capture builds a dummy extend forward with attn_dcp_metadata=None. ( "decode context parallel (dcp_size > 1)", - lambda: self.dcp_size > 1, + lambda: cfg.dcp_size > 1, ), ] for _name, predicate in rules: if predicate(): - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED def _disable_breakable_cudagraph_if_incompatible(self): """Breakable (segmented capture, no torch.compile). Breakable enforces memory-saver rejection in its own __init__; config-time rules can be added here as they're discovered. """ + cfg = resolving_view(self) from sglang.srt.configs.model_config import is_deepseek_v4 from sglang.srt.layers.cp.bcg import supports_prefill_cp_bcg @@ -5052,12 +5077,12 @@ class ServerArgs: # Capture builds a dummy extend forward with attn_dcp_metadata=None. ( "decode context parallel (dcp_size > 1)", - lambda: self.dcp_size > 1, + lambda: cfg.dcp_size > 1, ), # TBO capture is unsupported. ( "two-batch overlap", - lambda: self.enable_two_batch_overlap, + lambda: cfg.enable_two_batch_overlap, ), ( "unvalidated a2a backend", @@ -5078,11 +5103,12 @@ class ServerArgs: "disabling prefill CUDA graph.", name, ) - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED return def _disable_full_prefill_cudagraph_if_incompatible(self): """Full prefill CG: empty rule list today; see the experimental warning.""" + cfg = resolving_view(self) rules = [] for name, predicate in rules: if predicate(): @@ -5091,7 +5117,7 @@ class ServerArgs: "disabling prefill CUDA graph.", name, ) - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED return def _disable_prefill_cuda_graph_for_deepseek_trtllm_mla(self): @@ -5100,10 +5126,11 @@ class ServerArgs: breakable) trtllm_mla falls back to FlashAttention for prefill and regresses performance, so disable whichever prefill graph backend is in effect. """ + cfg = resolving_view(self) if (Phase.PREFILL, "backend") in self._cuda_graph_config_locked: return - if self.cuda_graph_config.prefill.backend == Backend.DISABLED: + if cfg.cuda_graph_config.prefill.backend == Backend.DISABLED: return if ( "DeepseekV3ForCausalLM" @@ -5118,15 +5145,16 @@ class ServerArgs: "the trtllm_mla attention backend (a captured prefill graph forces a " "FlashAttention fallback that regresses prefill). Set the prefill cuda graph " "backend explicitly (e.g. --cuda-graph-backend-prefill tc_piecewise) to override.", - self.cuda_graph_config.prefill.backend, + cfg.cuda_graph_config.prefill.backend, ) - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED def _validate_cuda_graph_config(self): - if self.cuda_graph_config is None: + cfg = resolving_view(self) + if cfg.cuda_graph_config is None: return for phase in Phase.ALL: - backend = getattr(self.cuda_graph_config, phase).backend + backend = getattr(cfg.cuda_graph_config, phase).backend if backend not in ALLOWED_BACKENDS_PER_PHASE[phase]: raise ValueError( f"--cuda-graph-config[{phase}].backend={backend!r} not allowed; " @@ -5141,22 +5169,23 @@ class ServerArgs: changing it silently could surprise users who intentionally picked a non-flashinfer backend. """ - if not self.enable_mis: + cfg = resolving_view(self) + if not cfg.enable_mis: return - if self.cuda_graph_config.decode.backend != Backend.DISABLED: + if cfg.cuda_graph_config.decode.backend != Backend.DISABLED: logger.warning("CUDA graph is disabled because --enable-mis is set.") - self.cuda_graph_config.decode.backend = Backend.DISABLED - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED - if not self.disable_radix_cache: + if not cfg.disable_radix_cache: logger.warning("Radix cache is disabled because --enable-mis is set.") self._declare( "_handle_multi_item_scoring", disable_radix_cache=True, ) - if self.chunked_prefill_size != -1: + if cfg.chunked_prefill_size != -1: logger.warning("Chunked prefill is disabled because --enable-mis is set.") self._declare( "_handle_multi_item_scoring", @@ -5194,14 +5223,15 @@ class ServerArgs: The coefficient 1.5 is a heuristic value, in the future, we can do better estimation by looking at the model types, hidden sizes or even do a dummy run. """ - decode_cuda_graph_config = self.cuda_graph_config.decode - prefill_cuda_graph_config = self.cuda_graph_config.prefill + cfg = resolving_view(self) + decode_cuda_graph_config = cfg.cuda_graph_config.decode + prefill_cuda_graph_config = cfg.cuda_graph_config.prefill if gpu_mem is not None: if gpu_mem < 20 * 1024: # T4, 4080 # (chunked_prefill_size 2k, max_bs 8) - if self.chunked_prefill_size is None: + if cfg.chunked_prefill_size is None: self._declare( "_handle_gpu_memory_settings", chunked_prefill_size=2048, @@ -5211,59 +5241,59 @@ class ServerArgs: elif gpu_mem < 35 * 1024: # A10, 4090, 5090 # (chunked_prefill_size 2k, max_bs 24 if tp < 4 else 80) - if self.chunked_prefill_size is None: + if cfg.chunked_prefill_size is None: self._declare( "_handle_gpu_memory_settings", chunked_prefill_size=2048, ) if decode_cuda_graph_config.max_bs is None: - if self.tp_size < 4: + if cfg.tp_size < 4: decode_cuda_graph_config.max_bs = 24 else: decode_cuda_graph_config.max_bs = 80 elif gpu_mem < 60 * 1024: # A100 (40GB), L40, # (chunked_prefill_size 4k, max_bs 32 if tp < 4 else 160) - if self.chunked_prefill_size is None: + if cfg.chunked_prefill_size is None: self._declare( "_handle_gpu_memory_settings", chunked_prefill_size=4096, ) if decode_cuda_graph_config.max_bs is None: - if self.tp_size < 4: + if cfg.tp_size < 4: decode_cuda_graph_config.max_bs = 32 else: decode_cuda_graph_config.max_bs = 160 elif gpu_mem < 90 * 1024: # H100, A100 # (chunked_prefill_size 8k, max_bs 256 if tp < 4 else 512) - if self.chunked_prefill_size is None: + if cfg.chunked_prefill_size is None: self._declare( "_handle_gpu_memory_settings", chunked_prefill_size=8192, ) if decode_cuda_graph_config.max_bs is None: - if self.tp_size < 4: + if cfg.tp_size < 4: decode_cuda_graph_config.max_bs = 256 else: decode_cuda_graph_config.max_bs = 512 elif gpu_mem < 160 * 1024: # H20, H200 # (chunked_prefill_size 8k, max_bs 256 if tp < 4 else 512) - if self.chunked_prefill_size is None: + if cfg.chunked_prefill_size is None: self._declare( "_handle_gpu_memory_settings", chunked_prefill_size=8192, ) if decode_cuda_graph_config.max_bs is None: - if self.tp_size < 4: + if cfg.tp_size < 4: decode_cuda_graph_config.max_bs = 256 else: decode_cuda_graph_config.max_bs = 512 else: # B200, MI300 # (chunked_prefill_size 16k, max_bs 512) - if self.chunked_prefill_size is None: + if cfg.chunked_prefill_size is None: self._declare( "_handle_gpu_memory_settings", chunked_prefill_size=16384, @@ -5272,7 +5302,7 @@ class ServerArgs: decode_cuda_graph_config.max_bs = 512 else: # Fallback defaults when gpu_mem is None - if self.chunked_prefill_size is None: + if cfg.chunked_prefill_size is None: self._declare( "_handle_gpu_memory_settings", chunked_prefill_size=4096, @@ -5281,7 +5311,7 @@ class ServerArgs: decode_cuda_graph_config.max_bs = 160 # Set cuda graph batch sizes - if self.device != "cpu": + if cfg.device != "cpu": if decode_cuda_graph_config.bs is None: decode_cuda_graph_config.bs = ( self._generate_decode_cuda_graph_batch_sizes( @@ -5303,34 +5333,34 @@ class ServerArgs: # to generate decode_cuda_graph_config.bs self._declare( "_handle_gpu_memory_settings", - torch_compile_max_bs=self.torch_compile_max_bs + torch_compile_max_bs=cfg.torch_compile_max_bs or decode_cuda_graph_config.max_bs, ) decode_cuda_graph_config.bs = self._generate_cpu_graph_batch_sizes() assert ( - self.torch_compile_max_bs > 0 + cfg.torch_compile_max_bs > 0 ), "cuda_graph_config[decode].bs should contain positive batch sizes" - decode_cuda_graph_config.max_bs = self.torch_compile_max_bs + decode_cuda_graph_config.max_bs = cfg.torch_compile_max_bs if prefill_cuda_graph_config.max_bs is None: # Refer to pr #15927, by default we set the prefill max_bs to the chunked prefill size. # For MLA backend, the introduction of piecewise cuda graph will influence the kernel dispatch difference compared to the original mode. # To avoid the performance regression, we set max_bs to 2048 by default. if not self.use_mla_backend(): - prefill_cuda_graph_config.max_bs = self.chunked_prefill_size + prefill_cuda_graph_config.max_bs = cfg.chunked_prefill_size else: prefill_cuda_graph_config.max_bs = 2048 # If max_total_tokens is set, cap prefill max_bs to not exceed max_total_tokens. - if self.max_total_tokens is not None: + if cfg.max_total_tokens is not None: prefill_cuda_graph_config.max_bs = min( - prefill_cuda_graph_config.max_bs, self.max_total_tokens + prefill_cuda_graph_config.max_bs, cfg.max_total_tokens ) # For Llama2 series models, max_bs is limited to 4096. # TODO(yuwei): remove this after the issue is fixed - if "llama-2" in self.model_path.lower(): + if "llama-2" in cfg.model_path.lower(): prefill_cuda_graph_config.max_bs = min( prefill_cuda_graph_config.max_bs, 4096 ) @@ -5342,31 +5372,29 @@ class ServerArgs: ) ) - if self.mem_fraction_static is None: + if cfg.mem_fraction_static is None: if self.post_capture_kv_sizing_planned(): # Post-capture sizing measures free memory after graph capture, so # skip the graph/activation reserve; keep only the floor + parallel slack. reserved_mem = 1536 - reserved_mem += self.tp_size * self.pp_size / 8 * 1024 + reserved_mem += cfg.tp_size * cfg.pp_size / 8 * 1024 else: # Tokens the activation working set scales with (per serving mode). - if self.disaggregation_mode == "decode": + if cfg.disaggregation_mode == "decode": running_requests = ( - self.max_running_requests - or decode_cuda_graph_config.max_bs - or 1 + cfg.max_running_requests or decode_cuda_graph_config.max_bs or 1 ) - draft_tokens = self.speculative_num_draft_tokens or 1 + draft_tokens = cfg.speculative_num_draft_tokens or 1 activation_tokens = max(running_requests * draft_tokens, 2048) - elif self.chunked_prefill_size > 0: - activation_tokens = max(self.chunked_prefill_size, 2048) + elif cfg.chunked_prefill_size > 0: + activation_tokens = max(cfg.chunked_prefill_size, 2048) else: - activation_tokens = max(self.max_prefill_tokens, 2048) + activation_tokens = max(cfg.max_prefill_tokens, 2048) # Constant meta data (e.g., from attention backend) + activation slack. reserved_mem = 512 reserved_mem += activation_tokens * 1.5 # Some adjustments for large parallel size - reserved_mem += self.tp_size * self.pp_size / 8 * 1024 + reserved_mem += cfg.tp_size * cfg.pp_size / 8 * 1024 reserved_mem += self.reserve_for_graph_mb() if gpu_mem is not None and gpu_mem > 60 * 1024: reserved_mem = max(reserved_mem, 10 * 1024) @@ -5389,14 +5417,14 @@ class ServerArgs: model_config = self.get_model_config() if ( model_config.is_multimodal - and not self.language_only - and not self.language_model_only - and self.disaggregation_mode != "decode" + and not cfg.language_only + and not cfg.language_model_only + and cfg.disaggregation_mode != "decode" ): self.adjust_mem_fraction_for_vlm(model_config) # If symm mem is enabled and prealloc size is not set, set it to 4GB - if self.enable_symm_mem and not envs.SGLANG_SYMM_MEM_PREALLOC_GB_SIZE.is_set(): + if cfg.enable_symm_mem and not envs.SGLANG_SYMM_MEM_PREALLOC_GB_SIZE.is_set(): envs.SGLANG_SYMM_MEM_PREALLOC_GB_SIZE.set(4) logger.warning( "Symmetric memory is enabled, setting symmetric memory prealloc size to 4GB as default." @@ -5407,41 +5435,42 @@ class ServerArgs: """Whether the mem_fraction heuristic may skip the graph reserve; must be False for any config the runtime won't post-capture-size, else it gets an under-reserved fraction.""" + cfg = resolving_view(self) # use_mla_backend is a method at args time but ModelRunner overwrites it # with a bool on global_server_args (see the FIXME there) -- handle both. use_mla = self.use_mla_backend mla_enabled = use_mla() if callable(use_mla) else use_mla if not envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING.get(): return False - if self.device != "cuda": + if cfg.device != "cuda": return False - if self.dcp_size != 1: + if cfg.dcp_size != 1: return False if mla_enabled: return False - if self.kv_cache_dtype == "fp4_e2m1": + if cfg.kv_cache_dtype == "fp4_e2m1": return False - if self.prefill_only_disable_kv_cache: + if cfg.prefill_only_disable_kv_cache: return False - if self.enable_memory_saver: + if cfg.enable_memory_saver: return False if envs.SGLANG_MOONCAKE_CUSTOM_MEM_POOL.get() is not None: return False if ( - self.disaggregation_mode != "prefill" - and self.cuda_graph_config.decode.backend == Backend.DISABLED + cfg.disaggregation_mode != "prefill" + and cfg.cuda_graph_config.decode.backend == Backend.DISABLED ): return False - if self.disaggregation_mode != "decode": - prefill_cfg = self.cuda_graph_config.prefill + if cfg.disaggregation_mode != "decode": + prefill_cfg = cfg.cuda_graph_config.prefill # We can only skip eager activation headroom when the largest # prefill forward batch size is already graph-captured. Otherwise, # an eager forward will need more memory and lead to OOM. if ( prefill_cfg.backend == Backend.DISABLED - or self.chunked_prefill_size <= 0 + or cfg.chunked_prefill_size <= 0 or self.max_prefill_buffer_tokens() > max(prefill_cfg.bs or (0,)) ): return False @@ -5457,47 +5486,49 @@ class ServerArgs: def pre_capture_activation_reserve_mb(self, gpu_mem: Optional[float]) -> float: # Runtime activation working-set reserve for eager decode above the captured # max_bs and transient prefill/logits; also covers fixed state caches. - if self.disaggregation_mode == "decode": + cfg = resolving_view(self) + if cfg.disaggregation_mode == "decode": running_requests = ( - self.max_running_requests or self.cuda_graph_config.decode.max_bs or 1 + cfg.max_running_requests or cfg.cuda_graph_config.decode.max_bs or 1 ) activation_tokens = max( - running_requests * (self.speculative_num_draft_tokens or 1), 2048 + running_requests * (cfg.speculative_num_draft_tokens or 1), 2048 ) - elif self.chunked_prefill_size > 0: - activation_tokens = max(self.chunked_prefill_size, 2048) + elif cfg.chunked_prefill_size > 0: + activation_tokens = max(cfg.chunked_prefill_size, 2048) else: - activation_tokens = max(self.max_prefill_tokens, 2048) + activation_tokens = max(cfg.max_prefill_tokens, 2048) reserved_mem = ( - 512 + activation_tokens * 1.5 + self.tp_size * self.pp_size / 8 * 1024 + 512 + activation_tokens * 1.5 + cfg.tp_size * cfg.pp_size / 8 * 1024 ) if gpu_mem is not None and gpu_mem > 60 * 1024: reserved_mem = max(reserved_mem, 10 * 1024) return reserved_mem def reserve_for_graph_mb(self) -> float: - decode_cuda_graph_config = self.cuda_graph_config.decode - prefill_cuda_graph_config = self.cuda_graph_config.prefill + cfg = resolving_view(self) + decode_cuda_graph_config = cfg.cuda_graph_config.decode + prefill_cuda_graph_config = cfg.cuda_graph_config.prefill reserved_mem = 0.0 if ( - self.disaggregation_mode != "prefill" + cfg.disaggregation_mode != "prefill" and decode_cuda_graph_config.backend != Backend.DISABLED ): reserved_mem += decode_cuda_graph_config.max_bs * 2 if ( self._resolved().enable_dp_attention - and self.disaggregation_mode != "prefill" + and cfg.disaggregation_mode != "prefill" ): # DP attention needs more padding for some operations, and much more for large # cuda graph max bs (torch allocator / implementation inefficiencies). - reserved_mem += decode_cuda_graph_config.max_bs * self.dp_size * 3 + reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 3 if decode_cuda_graph_config.max_bs > 300: - reserved_mem += decode_cuda_graph_config.max_bs * self.dp_size * 1.5 + reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 1.5 if ( - self.disaggregation_mode != "decode" + cfg.disaggregation_mode != "decode" and prefill_cuda_graph_config.backend != Backend.DISABLED ): if not self.use_mla_backend(): @@ -5521,9 +5552,10 @@ class ServerArgs: # DeepEP all-to-all buffers captured in the decode graph are real extra # allocations, reserved on top of the floor. - decode_cuda_graph_config = self.cuda_graph_config.decode + cfg = resolving_view(self) + decode_cuda_graph_config = cfg.cuda_graph_config.decode if ( - self.disaggregation_mode != "prefill" + cfg.disaggregation_mode != "prefill" and decode_cuda_graph_config.backend != Backend.DISABLED and resolved_view(self).moe_a2a_backend == "deepep" ): @@ -5535,10 +5567,11 @@ class ServerArgs: Generate the list of batch sizes for CUDA graph capture based on max_bs. This integrates the logic from cuda_graph_runner.py. """ + cfg = resolving_view(self) # Handle disable_cuda_graph_padding as the first condition for both spec and non-spec - if self.disable_cuda_graph_padding: + if cfg.disable_cuda_graph_padding: capture_bs = list(range(1, max_bs + 1)) - elif self.speculative_algorithm is None: + elif cfg.speculative_algorithm is None: # Normal case: capture_bs = ( [1, 2, 4, 8, 12] @@ -5567,19 +5600,20 @@ class ServerArgs: """ Generate the list of batch sizes for CPU graph capture based on torch_compile_max_bs. """ - if self.disable_cuda_graph_padding: - capture_bs = list(range(1, self.torch_compile_max_bs + 1)) + cfg = resolving_view(self) + if cfg.disable_cuda_graph_padding: + capture_bs = list(range(1, cfg.torch_compile_max_bs + 1)) else: capture_bs = sorted( set().union( range(1, 17), range(18, 31, 2), range(32, 81, 4), - range(84, self.torch_compile_max_bs + 1, 8), - {self.torch_compile_max_bs}, + range(84, cfg.torch_compile_max_bs + 1, 8), + {cfg.torch_compile_max_bs}, ) ) - capture_bs = [bs for bs in capture_bs if bs <= self.torch_compile_max_bs] + capture_bs = [bs for bs in capture_bs if bs <= cfg.torch_compile_max_bs] return capture_bs @@ -5633,12 +5667,13 @@ class ServerArgs: validate_hisparse_kv_cache_dtype(self) def _handle_model_specific_adjustments(self): + cfg = resolving_view(self) from sglang.srt.configs.model_config import ( get_mimo_v2_fused_qkv_expected_tp_size, is_deepseek_dsa, ) - if self.enable_deterministic_inference: + if cfg.enable_deterministic_inference: self._declare( "_handle_model_specific_adjustments", enforce_disable_flashinfer_allreduce_fusion=True, @@ -5648,7 +5683,7 @@ class ServerArgs: "_handle_model_specific_adjustments", uses_mamba_radix_cache=False, ) - if parse_connector_type(self.model_path) == ConnectorType.INSTANCE: + if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE: # No model overrides for an instance connector: no hf_config to # key them on. return @@ -5659,22 +5694,22 @@ class ServerArgs: if model_arch == "InternS2MobiusForConditionalGeneration": unsupported = [] - if self.pp_size != 1: + if cfg.pp_size != 1: unsupported.append("pipeline parallelism (--pp-size must be 1)") - if self.ep_size != 1: + if cfg.ep_size != 1: unsupported.append("expert parallelism (--ep-size must be 1)") if unsupported: raise ValueError( "Intern-S2-Mobius does not support: " + "; ".join(unsupported) + "." ) - if self.enable_dsa_cache_layer_split and not is_deepseek_dsa(hf_config): + if cfg.enable_dsa_cache_layer_split and not is_deepseek_dsa(hf_config): raise ValueError( "--enable-dsa-cache-layer-split is only supported for DSA " "(DeepSeek Sparse Attention) models." ) - if self.enable_cp_decode_attn_tp: + if cfg.enable_cp_decode_attn_tp: from sglang.srt.layers.cp.cp_decode_attn_tp import ( CP_DECODE_ATTN_TP_SUPPORTED_ARCHS, ) @@ -5759,7 +5794,7 @@ class ServerArgs: index_topk_freq = getattr(hf_config, "index_topk_freq", 1) or 1 index_topk_pattern = getattr(hf_config, "index_topk_pattern", None) - if self.enable_two_batch_overlap and ( + if cfg.enable_two_batch_overlap and ( index_topk_freq > 1 or (index_topk_pattern is not None and "S" in index_topk_pattern) ): @@ -5772,17 +5807,17 @@ class ServerArgs: ) if not is_npu() and not is_xpu(): # CUDA or ROCm GPU - if self.enable_prefill_cp: + if cfg.enable_prefill_cp: # The DSA CP field declarations moved to the override # registry (arg_groups/overrides.py: # _deepseek_family_overrides). - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED else: # Pure TP and partial DP Attention mode is active for DSA, logging a warning - if self.dp_size < self.tp_size: + if cfg.dp_size < cfg.tp_size: logger.warning( - f"DSA with TP mode is active, dp_size={self.dp_size}, tp_size={self.tp_size}, " - f"attn_tp_size={self.tp_size}, attention weights will be sharded across {self.tp_size} ranks." + f"DSA with TP mode is active, dp_size={cfg.dp_size}, tp_size={cfg.tp_size}, " + f"attn_tp_size={cfg.tp_size}, attention weights will be sharded across {cfg.tp_size} ranks." ) # The DSA page-size selection moved to the override registry @@ -5796,15 +5831,15 @@ class ServerArgs: ) self._set_default_dsa_backends(major) - if self.enable_prefill_cp: + if cfg.enable_prefill_cp: assert ( - self.disaggregation_mode != "decode" + cfg.disaggregation_mode != "decode" ), "CP is only supported for prefill when PD disaggregation, please remove --enable-prefill-cp." if ( - self.enable_dsa_cache_layer_split - and self.disaggregation_mode != "prefill" + cfg.enable_dsa_cache_layer_split + and cfg.disaggregation_mode != "prefill" ): - if self.disaggregation_mode == "decode": + if cfg.disaggregation_mode == "decode": raise ValueError( "--enable-dsa-cache-layer-split is not supported on " "decode workers. This flag is a prefill-CP " @@ -5816,8 +5851,8 @@ class ServerArgs: "prefill workers. Non-PD workers also run decode and " "require ordinary local decode cache semantics." ) - if self.enable_dsa_cache_layer_split and ( - not self.enable_prefill_cp or self.cp_strategy != "interleave" + if cfg.enable_dsa_cache_layer_split and ( + not cfg.enable_prefill_cp or cfg.cp_strategy != "interleave" ): raise ValueError( "--enable-dsa-cache-layer-split requires " @@ -5829,17 +5864,17 @@ class ServerArgs: # transfer path. mori/nixl support is a temporary limitation # and will be added later by the community. if ( - self.enable_dsa_cache_layer_split - and self.disaggregation_transfer_backend != "mooncake" + cfg.enable_dsa_cache_layer_split + and cfg.disaggregation_transfer_backend != "mooncake" ): raise ValueError( "--enable-dsa-cache-layer-split currently only supports " "the mooncake transfer backend (mooncake / mooncake_tcp). " f"Got --disaggregation-transfer-backend " - f"{self.disaggregation_transfer_backend!r}. mori/nixl " + f"{cfg.disaggregation_transfer_backend!r}. mori/nixl " "support will be added later by the community." ) - if self.enable_dsa_cache_layer_split and self.pp_size > 1: + if cfg.enable_dsa_cache_layer_split and cfg.pp_size > 1: raise ValueError( "--enable-dsa-cache-layer-split is not supported with " "pipeline parallelism (pp_size > 1) yet. It requires " @@ -5849,7 +5884,7 @@ class ServerArgs: else: # DeepSeek V3/R1/V3.1 - if self.cuda_graph_config.prefill.backend != Backend.DISABLED: + if cfg.cuda_graph_config.prefill.backend != Backend.DISABLED: logger.info("Piecewise CUDA graph is enabled, use MLA for prefill.") # The sm100 trtllm_mla fill moved to the override registry @@ -5858,8 +5893,8 @@ class ServerArgs: # MLA prefill CP auto-config: the field declarations moved to # the override registry (arg_groups/overrides.py: # _deepseek_family_overrides). - if self.enable_prefill_cp and self.use_mla_backend(): - self.cuda_graph_config.prefill.backend = Backend.DISABLED + if cfg.enable_prefill_cp and self.use_mla_backend(): + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED # Set moe backend for DeepSeek: the sm100 quant/moe resolution # moved to the resolution pipeline (arg_groups/overrides.py: @@ -5884,7 +5919,7 @@ class ServerArgs: # here for the rest of the DSA family (DeepSeek-V3.2 / # GLM-5.x) that shares the same decode top-k path. envs.SGLANG_OPT_USE_TOPK_V2.set(False) - if not self._resolved().enable_dp_attention and self.nnodes == 1: + if not self._resolved().enable_dp_attention and cfg.nnodes == 1: # TODO (Hubert): Put this back later # self.enable_aiter_allreduce_fusion = True logger.info( @@ -5970,7 +6005,7 @@ class ServerArgs: is_mxfp4_quant_format = quant_method == "mxfp4" if ( not self._resolved().enable_dp_attention - and self.nnodes == 1 + and cfg.nnodes == 1 and is_hip() ): # TODO (Hubert): Put this back later @@ -5993,14 +6028,14 @@ class ServerArgs: ), "Triton kernel MoE is only supported when ep_size == 1" elif model_arch in ("MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM"): - if model_arch == "MiMoV2ForCausalLM" and not self.encoder_only: + if model_arch == "MiMoV2ForCausalLM" and not cfg.encoder_only: expected_attn_tp_size = get_mimo_v2_fused_qkv_expected_tp_size( hf_config ) view = self._resolved() - attn_dp_size = self.dp_size if view.enable_dp_attention else 1 + attn_dp_size = cfg.dp_size if view.enable_dp_attention else 1 effective_attn_tp_size = ( - self.tp_size // attn_dp_size // view.attn_cp_size + cfg.tp_size // attn_dp_size // view.attn_cp_size ) if ( expected_attn_tp_size is not None @@ -6012,7 +6047,7 @@ class ServerArgs: "qkv_proj weights are " f"TP={expected_attn_tp_size}-interleaved; got " f"{effective_attn_tp_size} " - f"(tp_size={self.tp_size}, dp_size={self.dp_size}, " + f"(tp_size={cfg.tp_size}, dp_size={cfg.dp_size}, " f"enable_dp_attention={view.enable_dp_attention}, " f"attn_cp_size={view.attn_cp_size}). " "Set --tp, --dp, --enable-dp-attention, and " @@ -6038,7 +6073,7 @@ class ServerArgs: pass elif ( model_arch in ("Llama4ForConditionalGeneration", "Llama4ForCausalLM") - and self.device != "cpu" + and cfg.device != "cpu" ): # Attention backend auto-select moved to the override registry # (arg_groups/overrides.py: _llama4_overrides). @@ -6267,6 +6302,7 @@ class ServerArgs: 2.2 We will use Flashinfer backend on blackwell. 2.3 Otherwise, we will use triton backend. """ + cfg = resolving_view(self) # OOT platforms provide their own default attention backend. if current_platform.is_out_of_tree(): return current_platform.get_default_attention_backend() @@ -6291,8 +6327,8 @@ class ServerArgs: is_sm100_supported() and is_no_spec_infer_or_topk_one(resolved_view(self)) and ( - self.speculative_algorithm is None - or self.speculative_eagle_topk is not None + cfg.speculative_algorithm is None + or cfg.speculative_eagle_topk is not None ) ): # trtllm_mha requires equal K/V row widths; fa4 carries @@ -6328,6 +6364,7 @@ class ServerArgs: return "triton" def _handle_attention_backend_compatibility(self): + cfg = resolving_view(self) model_config = self.get_model_config() # The attention_backend write clusters of this handler moved to the @@ -6353,24 +6390,24 @@ class ServerArgs: logger.warning( "Cuda graph is disabled because of using torch native attention backend" ) - self.cuda_graph_config.decode.backend = Backend.DISABLED - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED if attention_backend == "flex_attention": logger.warning( "Cuda graph is disabled because of using torch Flex Attention backend" ) - self.cuda_graph_config.decode.backend = Backend.DISABLED - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED assert ( - self.speculative_algorithm is None + cfg.speculative_algorithm is None ), "Speculative decoding is currently not supported with Flex Attention backend" # Whisper's encoder token padding conflicts with prefix caching. # Only disable for Whisper; other encoder-decoder models (e.g., mllama) use radix cache. if ( model_config.is_encoder_decoder - and not self.disable_radix_cache + and not cfg.disable_radix_cache and "WhisperForConditionalGeneration" in (model_config.hf_config.architectures or []) ): @@ -6413,7 +6450,7 @@ class ServerArgs: prefill_backend == "trtllm_mha" and is_sm120_supported() and ( - self.kv_cache_dtype == "fp8_e4m3" + cfg.kv_cache_dtype == "fp8_e4m3" or ( envs.SGLANG_SKIP_SOFTMAX_PREFILL_THRESHOLD_SCALE_FACTOR.get() or 0.0 @@ -6434,7 +6471,7 @@ class ServerArgs: if ( prefill_backend == "trtllm_mha" and not is_sm100_supported() - and (self.enable_prefill_context_parallel or self.attn_cp_size > 1) + and (cfg.enable_prefill_context_parallel or cfg.attn_cp_size > 1) ): raise ValueError( "Prefill context parallelism with the TRTLLM MHA prefill backend " @@ -6451,7 +6488,7 @@ class ServerArgs: if model_config.context_len > 8192: self._declare( "_handle_attention_backend_compatibility", - mem_fraction_static=self.mem_fraction_static * 0.85, + mem_fraction_static=cfg.mem_fraction_static * 0.85, ) # Other platforms backends @@ -6482,7 +6519,8 @@ class ServerArgs: def _handle_mxfp8_kv_cache_compatibility(self): """MXFP8 KV cache uses operands available only on SM100+ (Blackwell).""" - if self.kv_cache_dtype != "mxfp8": + cfg = resolving_view(self) + if cfg.kv_cache_dtype != "mxfp8": return if not is_blackwell_supported(): raise ValueError( @@ -6492,8 +6530,9 @@ class ServerArgs: def _handle_kv4_compatibility(self): """Check FP4 KV cache compatibility with the attention backend""" + cfg = resolving_view(self) - if self.kv_cache_dtype not in ("nvfp4", "fp4_mx_block16"): + if cfg.kv_cache_dtype not in ("nvfp4", "fp4_mx_block16"): return use_mla_backend = self.use_mla_backend() @@ -6501,7 +6540,7 @@ class ServerArgs: attention_backend = resolved_view(self).attention_backend if is_cuda(): - if self.kv_cache_dtype == "nvfp4" and not ( + if cfg.kv_cache_dtype == "nvfp4" and not ( is_sm100_supported() or is_sm120_supported() ): raise RuntimeError( @@ -6579,7 +6618,8 @@ class ServerArgs: def _handle_nccl_pre_warm(self): # pre_warm_nccl is only used with CUDA or HIP hardware or NPU hardware - if self.pre_warm_nccl and not (is_cuda() or is_hip() or is_npu()): + cfg = resolving_view(self) + if cfg.pre_warm_nccl and not (is_cuda() or is_hip() or is_npu()): logger.warning( "pre_warm_nccl is only applicable for CUDA or HIP hardware or NPU hardware. " "Ignoring pre_warm_nccl setting on current hardware." @@ -6587,24 +6627,26 @@ class ServerArgs: self._declare("_handle_nccl_pre_warm", pre_warm_nccl=False) def _handle_grammar_backend(self): - if self.grammar_backend is None: + cfg = resolving_view(self) + if cfg.grammar_backend is None: self._declare("_handle_grammar_backend", grammar_backend="xgrammar") def _handle_mamba_backend(self): - if self.mamba_cache_philox_rounds < 0: + cfg = resolving_view(self) + if cfg.mamba_cache_philox_rounds < 0: raise ValueError("--mamba-cache-philox-rounds must be non-negative.") - if self.mamba_max_states_per_path == 0 or self.mamba_max_states_per_path < -1: + if cfg.mamba_max_states_per_path == 0 or cfg.mamba_max_states_per_path < -1: raise ValueError( "--mamba-max-states-per-path must be -1 (unlimited) or a positive " - f"integer, got {self.mamba_max_states_per_path}." + f"integer, got {cfg.mamba_max_states_per_path}." ) - if self.enable_mamba_cache_stochastic_rounding: - if self.mamba_ssm_dtype != "float16": + if cfg.enable_mamba_cache_stochastic_rounding: + if cfg.mamba_ssm_dtype != "float16": raise ValueError( "Stochastic rounding for the Mamba SSM cache requires " - f"--mamba-ssm-dtype float16, got {self.mamba_ssm_dtype!r}. " + f"--mamba-ssm-dtype float16, got {cfg.mamba_ssm_dtype!r}. " "Run with --mamba-ssm-dtype float16 or disable " "--enable-mamba-cache-stochastic-rounding." ) @@ -6614,7 +6656,7 @@ class ServerArgs: "supported on NVIDIA CUDA platforms. Disable " "--enable-mamba-cache-stochastic-rounding on this platform." ) - if self.mamba_backend == "triton" and not is_sm100_supported(): + if cfg.mamba_backend == "triton" and not is_sm100_supported(): raise ValueError( "Stochastic rounding for the Mamba SSM cache with " "--mamba-backend triton requires SM100 with CUDA >= 12.8 " @@ -6624,12 +6666,12 @@ class ServerArgs: "--enable-mamba-cache-stochastic-rounding." ) - if self.mamba_backend == "flashinfer": + if cfg.mamba_backend == "flashinfer": flashinfer_error = ( "FlashInfer mamba module not available, please check the " "FlashInfer installation." ) - if self.enable_mamba_cache_stochastic_rounding: + if cfg.enable_mamba_cache_stochastic_rounding: flashinfer_error += ( " Stochastic rounding with --mamba-backend flashinfer " "requires FlashInfer Mamba and --mamba-ssm-dtype float16." @@ -6651,22 +6693,24 @@ class ServerArgs: # int8-aware: they would read int8 checkpoint slots as bf16 active slots # (wrong pool / out-of-range). Reject the combination up front rather than # silently corrupting state. - if not self.enable_int8_mamba_checkpoint: + cfg = resolving_view(self) + if not cfg.enable_int8_mamba_checkpoint: return - if self.enable_hierarchical_cache: + if cfg.enable_hierarchical_cache: raise ValueError( "--enable-int8-mamba-checkpoint is not supported together with " "--enable-hierarchical-cache: the host-offload path " "is not int8-aware. Disable one of them." ) - if self.radix_cache_backend is not None: + if cfg.radix_cache_backend is not None: raise ValueError( "--enable-int8-mamba-checkpoint only supports the built-in mamba " - f"radix cache; --radix-cache-backend={self.radix_cache_backend!r} " + f"radix cache; --radix-cache-backend={cfg.radix_cache_backend!r} " "is not int8-aware. Omit --radix-cache-backend." ) def _handle_linear_attn_backend(self): + cfg = resolving_view(self) import torch # SM100+: default to FlashInfer GDN decode (and MTP verify, via pool API) @@ -6674,10 +6718,10 @@ class ServerArgs: # mamba-ssm-dtype is bf16 (required by FlashInfer GDN on SM100+). # Fixed in FlashInfer v0.6.7: flashinfer-ai/flashinfer#2810 if ( - self.linear_attn_decode_backend is None - and self.linear_attn_backend != "helion" + cfg.linear_attn_decode_backend is None + and cfg.linear_attn_backend != "helion" and is_sm100_supported() - and self.mamba_ssm_dtype == "bfloat16" + and cfg.mamba_ssm_dtype == "bfloat16" # Stage 4: flashinfer's recurrent_kda compiles the state slot stride # as a free int64, so it reads the page-major/unified envelope-strided # state correctly — the unified-memory skip is no longer needed (the @@ -6693,7 +6737,7 @@ class ServerArgs: ) # SM100+ FlashInfer GDN decode requires bf16 state; SM90 uses float32. - decode = self.linear_attn_decode_backend or self.linear_attn_backend + decode = cfg.linear_attn_decode_backend or cfg.linear_attn_backend # FlashKDA is a prefill-only KDA kernel (no decode kernel) but shares the # backend choice list, so guard it from being selected for decode: error @@ -6701,7 +6745,7 @@ class ServerArgs: # triton decode when it was only inherited from base=flashkda (prefill # keeps FlashKDA). if decode == "flashkda": - if self.linear_attn_decode_backend == "flashkda": + if cfg.linear_attn_decode_backend == "flashkda": raise ValueError( "--linear-attn-decode-backend flashkda is not supported: " "FlashKDA is prefill-only. Use " @@ -6719,34 +6763,34 @@ class ServerArgs: if ( decode == "flashinfer" - and self.mamba_ssm_dtype != "bfloat16" + and cfg.mamba_ssm_dtype != "bfloat16" and is_cuda() and torch.cuda.get_device_capability()[0] >= 10 ): raise ValueError( "--linear-attn-decode-backend flashinfer on SM100+ requires " "--mamba-ssm-dtype bfloat16, " - f"got {self.mamba_ssm_dtype!r}" + f"got {cfg.mamba_ssm_dtype!r}" ) - verify = self.linear_attn_verify_backend + verify = cfg.linear_attn_verify_backend if verify is None and decode == "flashinfer": verify = "flashinfer" if ( verify == "flashinfer" - and self.mamba_ssm_dtype != "bfloat16" + and cfg.mamba_ssm_dtype != "bfloat16" and is_cuda() and torch.cuda.get_device_capability()[0] >= 10 ): raise ValueError( "--linear-attn-verify-backend flashinfer on SM100+ requires " "--mamba-ssm-dtype bfloat16, " - f"got {self.mamba_ssm_dtype!r}" + f"got {cfg.mamba_ssm_dtype!r}" ) # SM100+ FlashInfer GDN prefill requires CUDA 13+ (CuTe DSL kernel) # for correctness and best performance. - prefill = self.linear_attn_prefill_backend or self.linear_attn_backend + prefill = cfg.linear_attn_prefill_backend or cfg.linear_attn_backend cuda_version = torch.version.cuda cuda_major = int(cuda_version.split(".")[0]) if cuda_version is not None else 0 if ( @@ -6774,7 +6818,7 @@ class ServerArgs: # does NOT route through MambaPool.copy_from, so the ReplaySSM ring # cursor of the donated/kept slot would not be reset there. Handling # that donation path is a follow-up; for now require no_buffer. - if self.enable_linear_replayssm: + if cfg.enable_linear_replayssm: if decode not in {"triton", "helion"}: raise ValueError( "--enable-linear-replayssm requires Triton, or Helion for " @@ -6790,9 +6834,9 @@ class ServerArgs: "--enable-linear-replayssm requires --mamba-radix-cache-strategy " "no_buffer (the default); the extra_buffer ping-pong " "donation path is not yet supported (follow-up). Got " - f"--mamba-radix-cache-strategy={self.mamba_radix_cache_strategy!r}." + f"--mamba-radix-cache-strategy={cfg.mamba_radix_cache_strategy!r}." ) - if self.disaggregation_mode != "null": + if cfg.disaggregation_mode != "null": # The disaggregated decode pool (HybridMambaDecodeReqToTokenPool) # is not wired for the ReplaySSM ring, so the flag would silently # no-op there; disagg also runs a different cache/coordination @@ -6800,12 +6844,12 @@ class ServerArgs: raise ValueError( "--enable-linear-replayssm is not supported under PD " "disaggregation yet (follow-up). Got " - f"--disaggregation-mode={self.disaggregation_mode!r}." + f"--disaggregation-mode={cfg.disaggregation_mode!r}." ) - if self.linear_replayssm_cache_len < 1: + if cfg.linear_replayssm_cache_len < 1: raise ValueError( "--linear-replayssm-cache-len must be >= 1, got " - f"{self.linear_replayssm_cache_len}." + f"{cfg.linear_replayssm_cache_len}." ) # ReplaySSM spec-verify (Part B of #28511): linear-chain target verify via @@ -6818,14 +6862,14 @@ class ServerArgs: # GDN sizes the window to the draft maximum; KDA (kda_backend) keeps a # --linear-replayssm-cache-len window and folds via its own fused # verify ring-write + commit_kda_replayssm_after_verify. - if self.enable_linear_replayssm_spec: - if self.speculative_eagle_topk not in (None, 1): + if cfg.enable_linear_replayssm_spec: + if cfg.speculative_eagle_topk not in (None, 1): raise ValueError( "--enable-linear-replayssm-spec requires a linear draft chain " "(--speculative-eagle-topk in {None, 1}); the chunked verify " "kernel uses a strictly-lower causal mask and is invalid for " "EAGLE tree verify. Got " - f"--speculative-eagle-topk={self.speculative_eagle_topk!r}." + f"--speculative-eagle-topk={cfg.speculative_eagle_topk!r}." ) if decode not in ("triton", "flashinfer"): raise ValueError( @@ -6846,8 +6890,8 @@ class ServerArgs: # not take the ragged layout and the flashinfer verify kernel # never writes the ring -> a stale ring would be folded; keep # refusing those combinations. - _algo = (self.speculative_algorithm or "").upper() - verify = self.linear_attn_verify_backend + _algo = (cfg.speculative_algorithm or "").upper() + verify = cfg.linear_attn_verify_backend if _algo not in ("DSPARK", "DFLASH") or verify not in ( "triton", "nv_cutedsl", @@ -6858,23 +6902,23 @@ class ServerArgs: "KDA fold-every-commit family (DSPARK/DFLASH) and a " "ring-writing verify kernel (--linear-attn-verify-backend " "triton or nv_cutedsl); got " - f"algorithm={self.speculative_algorithm!r}, " + f"algorithm={cfg.speculative_algorithm!r}, " f"verify={verify!r}. Use SGLANG_RAGGED_VERIFY_MODE=static." ) - if self.disaggregation_mode == "prefill": + if cfg.disaggregation_mode == "prefill": raise ValueError( "--enable-linear-replayssm-spec is not supported on a PD " "prefill server: the ring is spec-verify-only scratch and " "the prefill server never runs spec verify." ) - if self.enable_linear_replayssm: + if cfg.enable_linear_replayssm: raise ValueError( "--enable-linear-replayssm-spec and --enable-linear-replayssm are " "mutually exclusive: they share the ring storage but drive it " "with incompatible cursor protocols (per-decode-forward vs " "per-verify-commit advance)." ) - if self.mamba_ssm_dtype is None: + if cfg.mamba_ssm_dtype is None: logger.info( "--enable-linear-replayssm-spec: setting --mamba-ssm-dtype " "float32 (the closed-loop exact fold keeps the SSM checkpoint " @@ -6884,17 +6928,18 @@ class ServerArgs: "_handle_linear_attn_backend", mamba_ssm_dtype="float32", ) - elif self.mamba_ssm_dtype != "float32": + elif cfg.mamba_ssm_dtype != "float32": logger.warning( "--enable-linear-replayssm-spec with --mamba-ssm-dtype=%s: the " "closed-loop fold re-quantizes the committed state each " "commit/flush (fp32 keeps it bit-exact to the fp32 recurrent " "baseline), so it may drift over long sequences. Validate " "accuracy for your model.", - self.mamba_ssm_dtype, + cfg.mamba_ssm_dtype, ) def _handle_legacy_cp_arguments(self): + cfg = resolving_view(self) legacy_mode_to_strategy = { "in-seq-split": "zigzag", "round-robin-split": "interleave", @@ -6905,36 +6950,36 @@ class ServerArgs: } if ( - self.enable_prefill_context_parallel - or self.enable_dsa_prefill_context_parallel + cfg.enable_prefill_context_parallel + or cfg.enable_dsa_prefill_context_parallel ): self._declare( "_handle_legacy_cp_arguments", enable_prefill_cp=True, ) - if self.enable_prefill_context_parallel and self.cp_strategy is None: + if cfg.enable_prefill_context_parallel and cfg.cp_strategy is None: self._declare( "_handle_legacy_cp_arguments", - cp_strategy=legacy_mode_to_strategy[self.prefill_cp_mode], + cp_strategy=legacy_mode_to_strategy[cfg.prefill_cp_mode], ) - if self.enable_dsa_prefill_context_parallel and self.cp_strategy is None: + if cfg.enable_dsa_prefill_context_parallel and cfg.cp_strategy is None: self._declare( "_handle_legacy_cp_arguments", - cp_strategy=legacy_mode_to_strategy[self.dsa_prefill_cp_mode], + cp_strategy=legacy_mode_to_strategy[cfg.dsa_prefill_cp_mode], ) if ( - self.enable_prefill_context_parallel - and self.enable_dsa_prefill_context_parallel + cfg.enable_prefill_context_parallel + and cfg.enable_dsa_prefill_context_parallel ): return - if not self.enable_prefill_cp or self.cp_strategy is None: + if not cfg.enable_prefill_cp or cfg.cp_strategy is None: return - mode = strategy_to_legacy_mode[self.cp_strategy] - use_dsa_legacy_aliases = self.enable_dsa_prefill_context_parallel or getattr( + mode = strategy_to_legacy_mode[cfg.cp_strategy] + use_dsa_legacy_aliases = cfg.enable_dsa_prefill_context_parallel or getattr( self._resolved(), "attention_backend", None ) in ("dsa", "dsv4") if use_dsa_legacy_aliases: @@ -6961,7 +7006,8 @@ class ServerArgs: ) def _handle_context_parallelism(self): - if parse_connector_type(self.model_path) != ConnectorType.INSTANCE: + cfg = resolving_view(self) + if parse_connector_type(cfg.model_path) != ConnectorType.INSTANCE: from sglang.srt.configs.model_config import is_deepseek_dsa from sglang.srt.layers.cp.utils import CP_V2_DEFAULT_MODEL_CLASSES @@ -6972,38 +7018,38 @@ class ServerArgs: is_dsa_default_model = is_deepseek_dsa(hf_config) # DSA CP-v2 currently supports only the interleave strategy. enable_default_cp_v2 = not is_dsa_default_model or ( - self.enable_prefill_cp and self.cp_strategy == "interleave" + cfg.enable_prefill_cp and cfg.cp_strategy == "interleave" ) if enable_default_cp_v2 and not envs.SGLANG_ENABLE_CP_V2.is_set(): envs.SGLANG_ENABLE_CP_V2.set(True) if ( - self.enable_prefill_cp + cfg.enable_prefill_cp and model_arch in ("MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM") and envs.SGLANG_ENABLE_CP_V2.get() ): - if self.cp_strategy != "zigzag": + if cfg.cp_strategy != "zigzag": raise ValueError( "MiMo V2 CP-v2 only supports --cp-strategy zigzag." ) if ( model_config.is_multimodal - and not self.language_only - and not self.language_model_only + and not cfg.language_only + and not cfg.language_model_only ): raise ValueError( "MiMo V2 CP-v2 only supports text inference; add " "--language-only." ) - if self.enable_prefill_cp and self.cp_strategy is None: + if cfg.enable_prefill_cp and cfg.cp_strategy is None: raise ValueError( "--cp-strategy must be set when --enable-prefill-cp is enabled." ) if ( - self.enable_prefill_context_parallel - and self.enable_dsa_prefill_context_parallel + cfg.enable_prefill_context_parallel + and cfg.enable_dsa_prefill_context_parallel ): raise ValueError( "--enable-prefill-context-parallel and " @@ -7018,38 +7064,38 @@ class ServerArgs: if view.attn_cp_size > 1: # The tp_size is the world size, not the real tensor parallel size assert ( - self.tp_size % view.attn_cp_size == 0 + cfg.tp_size % view.attn_cp_size == 0 ), "tp_size must be divisible by attn_cp_size" assert ( - self.tp_size % (self.dp_size * view.attn_cp_size) == 0 + cfg.tp_size % (cfg.dp_size * view.attn_cp_size) == 0 ), "tp_size must be divisible by dp_size * attn_cp_size" assert ( - not self.enable_aiter_allreduce_fusion + not cfg.enable_aiter_allreduce_fusion ), "Aiter allreduce fusion is not supported with context parallelism" - if self.moe_dp_size > 1: + if cfg.moe_dp_size > 1: # The tp_size is the world size, not the real tensor parallel size assert ( - self.tp_size % self.moe_dp_size == 0 + cfg.tp_size % cfg.moe_dp_size == 0 ), "tp_size must be divisible by moe_dp_size" assert ( - view.ep_size * self.moe_dp_size <= self.tp_size + view.ep_size * cfg.moe_dp_size <= cfg.tp_size ), "ep_size * moe_dp_size must be less than or equal to tp_size" - assert self.pp_size == 1, "PP is not supported with context parallelism" + assert cfg.pp_size == 1, "PP is not supported with context parallelism" if view.ep_size > 1: assert ( - view.ep_size * self.moe_dp_size == self.tp_size + view.ep_size * cfg.moe_dp_size == cfg.tp_size ), "ep_size * moe_dp_size must be equal to tp_size" assert ( - not self.enable_aiter_allreduce_fusion + not cfg.enable_aiter_allreduce_fusion ), "Aiter allreduce fusion is not supported with context parallelism" - if view.attn_cp_size != self.moe_dp_size: + if view.attn_cp_size != cfg.moe_dp_size: assert ( - self.moe_dp_size == 1 + cfg.moe_dp_size == 1 ), "attn_cp_size != moe_dp_size is only supported when moe_dp_size == 1" from sglang.srt.layers.cp.base import init_cp_strategy @@ -7057,31 +7103,32 @@ class ServerArgs: init_cp_strategy(self) def _handle_dwdp(self): - if self.dwdp_size <= 1: + cfg = resolving_view(self) + if cfg.dwdp_size <= 1: return assert ( - self.dwdp_size >= 2 - ), f"dwdp_size must be >= 2 when enabled, got {self.dwdp_size}" + cfg.dwdp_size >= 2 + ), f"dwdp_size must be >= 2 when enabled, got {cfg.dwdp_size}" assert ( - self.dwdp_size == self.tp_size - ), f"dwdp_size ({self.dwdp_size}) must equal tp_size ({self.tp_size})" - assert self.disaggregation_mode in ( + cfg.dwdp_size == cfg.tp_size + ), f"dwdp_size ({cfg.dwdp_size}) must equal tp_size ({cfg.tp_size})" + assert cfg.disaggregation_mode in ( "null", "prefill", ), "DWDP requires --disaggregation-mode null or prefill" assert ( - not self.enable_eplb + not cfg.enable_eplb ), "EPLB dynamic migration conflicts with static DWDP partitioning" assert ( - self.speculative_algorithm is None + cfg.speculative_algorithm is None ), "DWDP does not support speculative decoding (MTP/draft workers)" - assert self.pp_size == 1, "DWDP requires pp_size == 1" + assert cfg.pp_size == 1, "DWDP requires pp_size == 1" assert ( - not self.enable_two_batch_overlap + not cfg.enable_two_batch_overlap ), "DWDP's prefetch event protocol does not support two-batch overlap" - if self.disaggregation_mode == "null": + if cfg.disaggregation_mode == "null": logger.warning( "DWDP with --disaggregation-mode null: decode steps re-fetch all " "remote expert weights every step, which is slow. DWDP is " @@ -7090,7 +7137,7 @@ class ServerArgs: self._declare( "_handle_dwdp", - dp_size=self.dwdp_size, + dp_size=cfg.dwdp_size, ) self._declare( "_handle_dwdp", @@ -7107,9 +7154,9 @@ class ServerArgs: ) self._declare( "_handle_dwdp", - ep_size=self.dwdp_size, + ep_size=cfg.dwdp_size, ) - self.moe_ep_size = self.dwdp_size + self.moe_ep_size = cfg.dwdp_size self._declare( "_handle_dwdp", moe_dp_size=1, @@ -7127,8 +7174,8 @@ class ServerArgs: ) logger.info( - f"DWDP enabled: dwdp_size={self.dwdp_size}, " - f"auto-forced dp_size={self.dp_size}, moe_ep_size={self.moe_ep_size}, " + f"DWDP enabled: dwdp_size={cfg.dwdp_size}, " + f"auto-forced dp_size={cfg.dp_size}, moe_ep_size={self.moe_ep_size}, " f"moe_dense_tp_size=1, moe_a2a_backend=none, " f"dp_attention_local_control_broadcast=True, " f"enable_dp_lm_head=True, SCHEDULER_SKIP_ALL_GATHER=True, " @@ -7138,6 +7185,7 @@ class ServerArgs: def _handle_data_parallelism(self): # The dp_size==1 resets moved to the resolution pipeline # (arg_groups/overrides.py: _data_parallelism_defaults). + cfg = resolving_view(self) from sglang.srt.arg_groups.overrides import ( _data_parallelism_defaults, run_post_process_pass, @@ -7145,8 +7193,8 @@ class ServerArgs: run_post_process_pass(self, _data_parallelism_defaults) - if self.mm_enable_dp_encoder: - if self.tp_size == 1: + if cfg.mm_enable_dp_encoder: + if cfg.tp_size == 1: logger.warning( "--mm-enable-dp-encoder is enabled with TP=1, so the encoder " "has no data-parallel work to distribute. Disable it unless " @@ -7160,23 +7208,23 @@ class ServerArgs: "prefill is a material part of TTFT. Measure against the default " "for small-image workloads because replication and aggregation " "can increase memory use and overhead.", - self.tp_size, + cfg.tp_size, ) if self._resolved().enable_dp_attention: self._declare( "_handle_data_parallelism", - schedule_conservativeness=self.schedule_conservativeness * 0.3, + schedule_conservativeness=cfg.schedule_conservativeness * 0.3, ) - assert self.tp_size % self.dp_size == 0 - original_chunked_prefill_size = self.chunked_prefill_size + assert cfg.tp_size % cfg.dp_size == 0 + original_chunked_prefill_size = cfg.chunked_prefill_size self._declare( "_handle_data_parallelism", - chunked_prefill_size=self.chunked_prefill_size // self.dp_size, + chunked_prefill_size=cfg.chunked_prefill_size // cfg.dp_size, ) logger.warning( f"DP attention is enabled. chunked prefill size is adjusted " - f"from {original_chunked_prefill_size} to {self.chunked_prefill_size}." + f"from {original_chunked_prefill_size} to {cfg.chunked_prefill_size}." ) # The prefill CUDA graph max_bs was derived from the pre-DP-division @@ -7185,14 +7233,14 @@ class ServerArgs: # the per-DP-rank chunked_prefill_size so breakable CUDA graph # capture never exceeds the MoE all-to-all's max_num_tokens budget, # which is also sized from the DP-adjusted chunked_prefill_size. - prefill_cfg = self.cuda_graph_config.prefill + prefill_cfg = cfg.cuda_graph_config.prefill if ( prefill_cfg.backend != Backend.DISABLED and prefill_cfg.max_bs is not None - and prefill_cfg.max_bs > self.chunked_prefill_size + and prefill_cfg.max_bs > cfg.chunked_prefill_size and (Phase.PREFILL, "max_bs") not in self._cuda_graph_config_locked ): - prefill_cfg.max_bs = self.chunked_prefill_size + prefill_cfg.max_bs = cfg.chunked_prefill_size if (Phase.PREFILL, "bs") not in self._cuda_graph_config_locked: prefill_cfg.bs = self._generate_prefill_cuda_graph_batch_sizes( prefill_cfg.max_bs @@ -7212,6 +7260,7 @@ class ServerArgs: # The quantization-driven runner resolutions moved to the pipeline # (arg_groups/overrides.py: _moe_runner_backend_quant_constraints); # the compatibility asserts and fusion writes stay below. + cfg = resolving_view(self) from sglang.srt.arg_groups.overrides import ( _moe_runner_backend_quant_constraints, _moe_runner_fusion_disable, @@ -7230,7 +7279,7 @@ class ServerArgs: ], f"Invalid quantization '{view.quantization}'. \nFlashInfer Cutlass MOE supports only: 'modelopt_fp4', 'modelopt_fp8', 'modelopt_mixed', or bfloat16 (None)." assert view.ep_size in [ 1, - self.tp_size, + cfg.tp_size, ], "The expert parallel size must be 1 or the same as the tensor parallel size" if view.moe_runner_backend == "flashinfer_cutedsl": @@ -7241,7 +7290,7 @@ class ServerArgs: ), f"Invalid quantization '{view.quantization}'. \nFlashInfer CuteDSL MOE currently supports only: 'modelopt_fp4', 'modelopt_mixed' (with NVFP4 MoE layers), 'nvfp4_online', or hybrid NVFP4 models." assert view.ep_size in [ 1, - self.tp_size, + cfg.tp_size, ], "The expert parallel size must be 1 or the same as the tensor parallel size" assert view.moe_a2a_backend in [ "none", @@ -7305,12 +7354,13 @@ class ServerArgs: capture, and decode/verify bounds; num_tokens_per_req is speculative_num_draft_tokens under speculative decoding, else 1. """ - if self.speculative_algorithm: - num_tokens_per_req = self.speculative_num_draft_tokens or 1 + cfg = resolving_view(self) + if cfg.speculative_algorithm: + num_tokens_per_req = cfg.speculative_num_draft_tokens or 1 else: num_tokens_per_req = 1 - prefill_tokens = self.max_prefill_tokens - cg_config = self.cuda_graph_config + prefill_tokens = cfg.max_prefill_tokens + cg_config = cfg.cuda_graph_config if cg_config is not None and cg_config.prefill.backend == Backend.TC_PIECEWISE: prefill_tokens = max(prefill_tokens, cg_config.prefill.max_bs or 0) decode_max_bs = (cg_config.decode.max_bs if cg_config is not None else 0) or 0 @@ -7320,29 +7370,29 @@ class ServerArgs: def max_prefill_buffer_tokens(self) -> int: """Prefill-buffer ceiling: chunked_prefill_size, except PP dynamic chunking can grow chunks toward max_prefill_tokens and probe at 1.25x.""" + cfg = resolving_view(self) chunked = ( - self.chunked_prefill_size - if self.chunked_prefill_size and self.chunked_prefill_size > 0 + cfg.chunked_prefill_size + if cfg.chunked_prefill_size and cfg.chunked_prefill_size > 0 else 0 ) tokens = chunked - if self.enable_dynamic_chunking and self.pp_size > 1 and chunked: - tokens = max( - tokens, self.max_prefill_tokens or 0, math.ceil(chunked * 1.25) - ) + if cfg.enable_dynamic_chunking and cfg.pp_size > 1 and chunked: + tokens = max(tokens, cfg.max_prefill_tokens or 0, math.ceil(chunked * 1.25)) return tokens def _validate_cutedsl_a2a_token_budget(self): """Fail fast if the FlashInfer A2A dispatcher workspace cannot cover the largest CuteDSL MoE forward. Runs after speculative decoding is resolved so cutedsl_moe_max_num_tokens() sees the final num_tokens_per_req.""" + cfg = resolving_view(self) view = resolved_view(self) if not ( view.moe_a2a_backend == "flashinfer" and view.moe_runner_backend == "flashinfer_cutedsl" - and self.max_prefill_tokens > 0 - and self.disaggregation_mode != "decode" + and cfg.max_prefill_tokens > 0 + and cfg.disaggregation_mode != "decode" ): return required_tokens = self.cutedsl_moe_max_num_tokens() @@ -7374,6 +7424,7 @@ class ServerArgs: # the resolution pipeline (arg_groups/overrides.py: # _a2a_backend_overrides / _a2a_ep_size); the per-backend logs, # asserts, fusion/deepep_mode/env/cuda-graph writes stay below. + cfg = resolving_view(self) from sglang.srt.arg_groups.overrides import ( _a2a_backend_overrides, _a2a_ep_size, @@ -7390,13 +7441,13 @@ class ServerArgs: run_post_process_pass(self, _a2a_fusion_adjustments) a2a_backend = resolved_view(self).moe_a2a_backend - if self.enable_waterfill: + if cfg.enable_waterfill: self._declare("_handle_a2a_moe", enforce_shared_experts_fusion=True) logger.info(f"Waterfill is enabled with moe_a2a_backend='{a2a_backend}'.") if a2a_backend == "deepep": - if self.moe_runner_backend == "flashinfer_cutedsl": - if self.deepep_mode == "auto": + if cfg.moe_runner_backend == "flashinfer_cutedsl": + if cfg.deepep_mode == "auto": self._declare( "_handle_a2a_moe", deepep_mode="low_latency", @@ -7407,17 +7458,17 @@ class ServerArgs: "deepep auto mode would crash during prefill. " "low_latency covers both prefill and decode." ) - elif self.deepep_mode == "normal": + elif cfg.deepep_mode == "normal": raise ValueError( "flashinfer_cutedsl FP4 MoE only supports DeepEP " "low_latency dispatch (masked layout). DeepEP normal " "(prefill) dispatch has no CuteDSL FP4 handler. Pass " "--deepep-mode low_latency or auto." ) - if self.deepep_mode == "normal": + if cfg.deepep_mode == "normal": logger.warning("Cuda graph is disabled because deepep_mode=`normal`") - self.cuda_graph_config.decode.backend = Backend.DISABLED - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED # The resolving view, not the field: `_a2a_backend_overrides` may have # moved this already (waterfill forces `deepep`). @@ -7429,11 +7480,11 @@ class ServerArgs: moe_a2a_backend="none", ) - if self.moe_a2a_backend == "flashinfer": + if cfg.moe_a2a_backend == "flashinfer": assert ( - resolved_view(self).enable_dp_attention and self.dp_size == self.tp_size + resolved_view(self).enable_dp_attention and cfg.dp_size == cfg.tp_size ), "Flashinfer MoE A2A is only supported with dp_size == tp_size and --enable-dp-attention" - if self.deepep_mode != "auto": + if cfg.deepep_mode != "auto": logger.warning("--deepep-mode is ignored for Flashinfer MoE A2A") if not envs.SGLANG_MOE_NVFP4_DISPATCH.is_set() and ( resolved_view(self).quantization == "modelopt_fp4" @@ -7450,7 +7501,7 @@ class ServerArgs: ], "Flashinfer MoE A2A is only supported with flashinfer_cutlass, flashinfer_cutedsl or flashinfer_trtllm_routed moe runner backend" if a2a_backend == "mori": - if self.deepep_mode == "auto": + if cfg.deepep_mode == "auto": self._declare( "_handle_a2a_moe", deepep_mode="normal", @@ -7460,7 +7511,7 @@ class ServerArgs: # Check chunked prefill for mori # Skip validation if chunked prefill is disabled (i.e., size <= 0). # Skip validation if disaggregation mode is decode. - if self.chunked_prefill_size > 0 and self.disaggregation_mode != "decode": + if cfg.chunked_prefill_size > 0 and cfg.disaggregation_mode != "decode": assert ( self._required_mori_dispatch_tokens_per_rank() ) <= envs.SGLANG_MORI_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get(), ( @@ -7470,12 +7521,12 @@ class ServerArgs: ) if a2a_backend == "pplx": - if self.deepep_mode == "normal": + if cfg.deepep_mode == "normal": raise ValueError( "moe_a2a_backend='pplx' only supports low-latency mode; " "set --deepep-mode to 'low_latency' or 'auto'." ) - if self.deepep_mode == "auto": + if cfg.deepep_mode == "auto": self._declare( "_handle_a2a_moe", deepep_mode="low_latency", @@ -7484,7 +7535,7 @@ class ServerArgs: # pplx-kernels' AllToAll needs numDPGroups (== attention dp_size) > 1; # without DP attention numDPGroups == 1 and construction fails deep in # the kernel. This also implies ep_size >= 2. - assert resolved_view(self).enable_dp_attention and self.dp_size >= 2, ( + assert resolved_view(self).enable_dp_attention and cfg.dp_size >= 2, ( "moe_a2a_backend='pplx' requires --enable-dp-attention with at " "least 2 DP groups (--dp-size >= 2)." ) @@ -7496,7 +7547,7 @@ class ServerArgs: "moe_a2a_backend='pplx' is only supported with --moe-runner-backend " "deep_gemm (or auto)." ) - if self.moe_runner_backend == "auto": + if cfg.moe_runner_backend == "auto": self._declare( "_handle_a2a_moe", moe_runner_backend="deep_gemm", @@ -7506,7 +7557,7 @@ class ServerArgs: # Check per-rank dispatch tokens for pplx # Skip validation if chunked prefill is disabled (i.e., size <= 0) # Skip validation if disaggregation mode is decode - if self.chunked_prefill_size > 0 and self.disaggregation_mode != "decode": + if cfg.chunked_prefill_size > 0 and cfg.disaggregation_mode != "decode": assert ( self._required_pplx_dispatch_tokens_per_rank() ) <= envs.SGLANG_PPLX_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get(), ( @@ -7517,17 +7568,20 @@ class ServerArgs: def _required_mori_dispatch_tokens_per_rank(self) -> int: """Max tokens a single rank dispatches through MoRI in one forward.""" - return self.chunked_prefill_size + cfg = resolving_view(self) + return cfg.chunked_prefill_size def _required_pplx_dispatch_tokens_per_rank(self) -> int: """Max tokens a single rank dispatches through pplx in one forward.""" - required = self.chunked_prefill_size - if self.cuda_graph_max_bs_decode is not None: - required = max(required, self.cuda_graph_max_bs_decode) + cfg = resolving_view(self) + required = cfg.chunked_prefill_size + if cfg.cuda_graph_max_bs_decode is not None: + required = max(required, cfg.cuda_graph_max_bs_decode) return required def _handle_eplb_and_dispatch(self): - if self.enable_eplb and (self.expert_distribution_recorder_mode is None): + cfg = resolving_view(self) + if cfg.enable_eplb and (cfg.expert_distribution_recorder_mode is None): self._declare( "_handle_eplb_and_dispatch", expert_distribution_recorder_mode="stat", @@ -7540,8 +7594,8 @@ class ServerArgs: # sum their partial outputs, so the pick has to agree across ranks. needs_rank_invariant_dispatch = self._resolved().moe_a2a_backend == "none" - if (self.enable_eplb or (self.init_expert_location != "trivial")) and ( - self.ep_dispatch_algorithm is None + if (cfg.enable_eplb or (cfg.init_expert_location != "trivial")) and ( + cfg.ep_dispatch_algorithm is None ): self._declare( "_handle_eplb_and_dispatch", @@ -7552,23 +7606,24 @@ class ServerArgs: # `dynamic` / `fake` switch to the row-index pick; `static` reads a # per-rank table and `lp` samples inside its kernel. - if needs_rank_invariant_dispatch and self.ep_dispatch_algorithm in ( + if needs_rank_invariant_dispatch and cfg.ep_dispatch_algorithm in ( "static", "lp", ): raise ValueError( - f"--ep-dispatch-algorithm {self.ep_dispatch_algorithm} picks a " + f"--ep-dispatch-algorithm {cfg.ep_dispatch_algorithm} picks a " "different physical replica per rank, which only holds up when an " "a2a backend routes each token to a single rank. Use " "--ep-dispatch-algorithm dynamic with --moe-a2a-backend none." ) - if self.enable_eplb and self.ep_join_mode != "scale": + if cfg.enable_eplb and cfg.ep_join_mode != "scale": assert self._resolved().ep_size > 1 def _handle_elastic_ep(self): - if self.elastic_ep_rejoin: - if self.ep_join_mode is None: + cfg = resolving_view(self) + if cfg.elastic_ep_rejoin: + if cfg.ep_join_mode is None: logger.warning( "--elastic-ep-rejoin is deprecated, use --elastic-ep-join-mode recover instead." ) @@ -7577,65 +7632,65 @@ class ServerArgs: ep_join_mode="recover", ) else: - assert self.ep_join_mode == "recover", ( + assert cfg.ep_join_mode == "recover", ( "--elastic-ep-rejoin (deprecated) conflicts with " - f"--elastic-ep-join-mode {self.ep_join_mode}." + f"--elastic-ep-join-mode {cfg.ep_join_mode}." ) - if self.elastic_ep_backend is not None: - if self.enable_eplb: - if self.eplb_algorithm == "auto": + if cfg.elastic_ep_backend is not None: + if cfg.enable_eplb: + if cfg.eplb_algorithm == "auto": self._declare( "_handle_elastic_ep", eplb_algorithm="elasticity_aware", ) - assert self.eplb_algorithm in [ + assert cfg.eplb_algorithm in [ "elasticity_aware", "elasticity_aware_hierarchical", ], "Elastic EP requires eplb_algorithm to be set to 'auto' or 'elasticity_aware(_hierarchical)'." - assert self.pp_size == 1, "PP size should be set to 1 under elastic EP" + assert cfg.pp_size == 1, "PP size should be set to 1 under elastic EP" - if self.elastic_ep_backend == "mooncake": + if cfg.elastic_ep_backend == "mooncake": self._declare( "_handle_elastic_ep", mooncake_ib_device=self._validate_ib_devices( - self.mooncake_ib_device + cfg.mooncake_ib_device ), ) - if self.ep_join_mode is not None: + if cfg.ep_join_mode is not None: assert ( - self.elastic_ep_backend is not None + cfg.elastic_ep_backend is not None ), "--elastic-ep-join-mode requires --elastic-ep-backend to be set." - if self.ep_join_mode == "scale": - assert self.node_rank == 1, ( + if cfg.ep_join_mode == "scale": + assert cfg.node_rank == 1, ( "Elastic EP scale-up requires one joining TP group at " - f"--node-rank 1 (got {self.node_rank})." + f"--node-rank 1 (got {cfg.node_rank})." ) - assert self.ep_join_rank_offset > 0, ( + assert cfg.ep_join_rank_offset > 0, ( "Elastic EP scale joiners require " "--elastic-ep-join-rank-offset set to the current " "effective EP size." ) - if self.ep_join_rank_offset != 0: - assert self.ep_join_mode == "scale", ( + if cfg.ep_join_rank_offset != 0: + assert cfg.ep_join_mode == "scale", ( "--elastic-ep-join-rank-offset is only valid with " "--elastic-ep-join-mode scale." ) assert ( - self.ep_join_rank_offset >= 0 + cfg.ep_join_rank_offset >= 0 ), "elastic EP join rank offset must be >= 0." - if self.max_ep_size is not None: + if cfg.max_ep_size is not None: assert ( - self.elastic_ep_backend is not None + cfg.elastic_ep_backend is not None ), "--max-ep-size requires --elastic-ep-backend to be set." - assert self.max_ep_size > 0, "--max-ep-size must be a positive integer." + assert cfg.max_ep_size > 0, "--max-ep-size must be a positive integer." scaling_active = ( - self.elastic_ep_backend is not None - and self.max_ep_size is not None - and self.max_ep_size > self.tp_size + cfg.elastic_ep_backend is not None + and cfg.max_ep_size is not None + and cfg.max_ep_size > cfg.tp_size ) - if self.elastic_ep_initial_size is not None: + if cfg.elastic_ep_initial_size is not None: assert scaling_active, ( "--elastic-ep-initial-size is only valid for an Elastic EP " "deployment with --max-ep-size larger than its local TP size." @@ -7643,16 +7698,16 @@ class ServerArgs: if scaling_active: resolved = self._resolved() assert ( - self.elastic_ep_scale_timeout > 0 + cfg.elastic_ep_scale_timeout > 0 ), "--elastic-ep-scale-timeout must be greater than zero." - assert self.tokenizer_worker_num == 1, ( + assert cfg.tokenizer_worker_num == 1, ( "Elastic EP runtime scale-up currently requires " "--tokenizer-worker-num 1." ) assert ( - not self.use_ray + not cfg.use_ray ), "Elastic EP runtime scale-up does not support --use-ray." - assert not self.enable_elastic_expert_backup, ( + assert not cfg.enable_elastic_expert_backup, ( "Elastic EP runtime scale-up does not support " "--enable-elastic-expert-backup." ) @@ -7660,57 +7715,57 @@ class ServerArgs: "_handle_elastic_ep", enable_dp_attention_local_control_broadcast=True, ) - if self.ep_join_mode == "scale": - assert self.elastic_ep_initial_size is not None, ( + if cfg.ep_join_mode == "scale": + assert cfg.elastic_ep_initial_size is not None, ( "Elastic EP scale joiners require --elastic-ep-initial-size " "set to the primary deployment's launch-time EP size." ) - assert self.elastic_ep_initial_size <= self.ep_join_rank_offset, ( + assert cfg.elastic_ep_initial_size <= cfg.ep_join_rank_offset, ( "--elastic-ep-initial-size cannot exceed the current EP size " - f"(initial={self.elastic_ep_initial_size}, " - f"current={self.ep_join_rank_offset})." + f"(initial={cfg.elastic_ep_initial_size}, " + f"current={cfg.ep_join_rank_offset})." ) - join_target = self.ep_join_rank_offset + self.tp_size - assert join_target <= self.max_ep_size, ( + join_target = cfg.ep_join_rank_offset + cfg.tp_size + assert join_target <= cfg.max_ep_size, ( "Elastic EP joining group exceeds --max-ep-size " - f"(join_target={join_target}, max_ep_size={self.max_ep_size})." + f"(join_target={join_target}, max_ep_size={cfg.max_ep_size})." ) - if self.tp_size == 1: - assert self.moe_dense_tp_size == 1, ( + if cfg.tp_size == 1: + assert cfg.moe_dense_tp_size == 1, ( "A single-rank Elastic EP joining group requires " "--moe-dense-tp-size 1." ) else: - if self.elastic_ep_initial_size is None: + if cfg.elastic_ep_initial_size is None: self._declare( "_handle_elastic_ep", - elastic_ep_initial_size=self.tp_size, + elastic_ep_initial_size=cfg.tp_size, ) - assert self.elastic_ep_initial_size == self.tp_size, ( + assert cfg.elastic_ep_initial_size == cfg.tp_size, ( "The primary --elastic-ep-initial-size must equal its " - f"launch-time TP size ({self.tp_size})." + f"launch-time TP size ({cfg.tp_size})." ) - assert self.elastic_ep_initial_size > 0 - assert self.load_balance_method == "round_robin", ( + assert cfg.elastic_ep_initial_size > 0 + assert cfg.load_balance_method == "round_robin", ( "Elastic EP scale-up requires --load-balance-method round_robin; " "load-aware methods " "require global-rank load snapshots after scale " - f"(got {self.load_balance_method})." + f"(got {cfg.load_balance_method})." ) - assert self.elastic_ep_backend == "mooncake", ( + assert cfg.elastic_ep_backend == "mooncake", ( "Elastic EP runtime scale-up requires --elastic-ep-backend " - f"mooncake (got elastic_ep_backend={self.elastic_ep_backend})." + f"mooncake (got elastic_ep_backend={cfg.elastic_ep_backend})." ) - assert self.pp_size == 1, ( + assert cfg.pp_size == 1, ( "Elastic EP scale-up requires --pp-size 1 " - f"(got pp_size={self.pp_size}); WORLD must not span PP stages." + f"(got pp_size={cfg.pp_size}); WORLD must not span PP stages." ) decode_cuda_graph_disabled = ( - self.cuda_graph_config.decode.backend == Backend.DISABLED + cfg.cuda_graph_config.decode.backend == Backend.DISABLED ) prefill_cuda_graph_disabled = ( - self.cuda_graph_config.prefill.backend == Backend.DISABLED + cfg.cuda_graph_config.prefill.backend == Backend.DISABLED ) assert decode_cuda_graph_disabled and prefill_cuda_graph_disabled, ( "Elastic EP runtime scale-up requires decode and prefill CUDA " @@ -7729,18 +7784,18 @@ class ServerArgs: "Elastic EP scale-up requires --attn-cp-size 1 " f"(got attn_cp_size={resolved.attn_cp_size})." ) - assert self.moe_dp_size == 1, ( + assert cfg.moe_dp_size == 1, ( "Elastic EP scale-up requires --moe-dp-size 1 " - f"(got moe_dp_size={self.moe_dp_size})." + f"(got moe_dp_size={cfg.moe_dp_size})." ) - assert resolved.ep_size == self.tp_size, ( + assert resolved.ep_size == cfg.tp_size, ( "Elastic EP scale-up requires ep_size == tp_size " - f"(got ep_size={resolved.ep_size}, tp_size={self.tp_size}); EP, TP " + f"(got ep_size={resolved.ep_size}, tp_size={cfg.tp_size}); EP, TP " "and the attention DP group must all coincide with WORLD." ) - assert self.dp_size == self.tp_size, ( + assert cfg.dp_size == cfg.tp_size, ( "Elastic EP scale-up requires dp_size == tp_size " - f"(got dp_size={self.dp_size}, tp_size={self.tp_size})." + f"(got dp_size={cfg.dp_size}, tp_size={cfg.tp_size})." ) assert resolved.moe_a2a_backend == "nixl", ( "Elastic EP scale-up requires --moe-a2a-backend nixl " @@ -7761,6 +7816,7 @@ class ServerArgs: # ===== END TO BE REFACTORED ==== def _handle_expert_distribution_metrics(self): + cfg = resolving_view(self) if "SGLANG_ENABLE_EPLB_BALANCEDNESS_METRIC" in os.environ: raise ValueError( "SGLANG_ENABLE_EPLB_BALANCEDNESS_METRIC is no longer supported. Use " @@ -7769,20 +7825,20 @@ class ServerArgs: ) if self.should_report_expert_balancedness() and ( - self.expert_distribution_recorder_mode is None + cfg.expert_distribution_recorder_mode is None ): self._declare( "_handle_expert_distribution_metrics", expert_distribution_recorder_mode="stat", ) - if self.expert_distribution_recorder_buffer_size is None: - if (x := self.eplb_rebalance_num_iterations) is not None: + if cfg.expert_distribution_recorder_buffer_size is None: + if (x := cfg.eplb_rebalance_num_iterations) is not None: self._declare( "_handle_expert_distribution_metrics", expert_distribution_recorder_buffer_size=x, ) - elif self.expert_distribution_recorder_mode is not None: + elif cfg.expert_distribution_recorder_mode is not None: self._declare( "_handle_expert_distribution_metrics", expert_distribution_recorder_buffer_size=1000, @@ -7804,26 +7860,27 @@ class ServerArgs: Backend resolution is checked separately by _handle_prefill_only_disable_kv_cache after backends settle. """ - if not self.prefill_only_disable_kv_cache: + cfg = resolving_view(self) + if not cfg.prefill_only_disable_kv_cache: return # This flag is intentionally scoped to embedding mode for now. Other # prefill-only paths (for example scoring and MIS) can benefit from # the same idea later, but some of them still stage K/V through the # paged cache today. - if not self.is_embedding: + if not cfg.is_embedding: raise ValueError( "--prefill-only-disable-kv-cache currently requires --is-embedding. " "Other prefill-only workloads may be supported in a future change once " "their attention paths stop reading or writing the paged KV cache." ) - if self.kv_cache_dtype in ("nvfp4", "fp4_mx_block16"): + if cfg.kv_cache_dtype in ("nvfp4", "fp4_mx_block16"): raise ValueError( "--prefill-only-disable-kv-cache does not currently support " "--kv-cache-dtype=nvfp4 or --kv-cache-dtype=fp4_mx_block16 because " "the FP4 pool uses a separate allocation path." ) - if self.kv_cache_dtype == "mxfp8": + if cfg.kv_cache_dtype == "mxfp8": raise ValueError( "--prefill-only-disable-kv-cache does not currently support " "--kv-cache-dtype=mxfp8 because the MXFP8 pool stores separate " @@ -7836,13 +7893,13 @@ class ServerArgs: # so K/V never has to be reused across prefill chunks. # - disable_radix_cache stops the prefix cache from indexing pool # slots that no longer hold real data. - if self.chunked_prefill_size != -1: + if cfg.chunked_prefill_size != -1: raise ValueError( "--prefill-only-disable-kv-cache requires --chunked-prefill-size=-1 so the FA " "backend takes the fa_skip_kv_cache path; otherwise the pool would be touched " "between prefill chunks." ) - if not self.disable_radix_cache: + if not cfg.disable_radix_cache: raise ValueError( "--prefill-only-disable-kv-cache requires --disable-radix-cache because the " "radix cache indexes KV pool slots that no longer hold real data." @@ -7857,7 +7914,7 @@ class ServerArgs: "the context-parallel attention path writes K/V to the pool via set_kv_buffer, " "which the no-op pool intentionally rejects." ) - if self.enable_prefill_cp: + if cfg.enable_prefill_cp: raise ValueError( "--prefill-only-disable-kv-cache is incompatible with " "--enable-prefill-cp: the prefill-CP path stages K/V through " @@ -7866,7 +7923,7 @@ class ServerArgs: # HiSparse selects a different pool class (HiSparseDSATokenToKVPool / # HiSparseTokenToKVPoolAllocator) that is not the no-op pool. - if self.enable_hisparse: + if cfg.enable_hisparse: raise ValueError( "--prefill-only-disable-kv-cache is incompatible with --enable-hisparse: " "HiSparse uses a dedicated pool family that is not the no-op MHA pool." @@ -7882,8 +7939,9 @@ class ServerArgs: still None, backends haven't settled yet and the resolved (prefill, decode) pair would be a stale (None, None). """ + cfg = resolving_view(self) - if not self.prefill_only_disable_kv_cache: + if not cfg.prefill_only_disable_kv_cache: return assert resolved_view(self).attention_backend is not None, ( @@ -7910,11 +7968,12 @@ class ServerArgs: A decode server keeps the ratio unset here: kv_cache_builder resolves it against the retraction-backup backend (1.0 for host_pool, else 2.0). """ - if self.hicache_ratio is None and self.disaggregation_mode != "decode": + cfg = resolving_view(self) + if cfg.hicache_ratio is None and cfg.disaggregation_mode != "decode": self._declare( "_handle_hicache_ratio_default", hicache_ratio=( - 1.2 if self.hicache_host_memory_mode == "buffer_only" else 2.0 + 1.2 if cfg.hicache_host_memory_mode == "buffer_only" else 2.0 ), ) @@ -7925,13 +7984,14 @@ class ServerArgs: 1) Layout <-> I/O compatibility for direct conflicts. 2) Storage <-> layout compatibility (may rewrite layout). """ + cfg = resolving_view(self) # Skip all normalization when neither hicache nor decode-offload path is active. if not ( - self.enable_hierarchical_cache - or self.disaggregation_decode_enable_offload_kvcache + cfg.enable_hierarchical_cache + or cfg.disaggregation_decode_enable_offload_kvcache or ( - self.disaggregation_mode == "decode" - and self.disaggregation_decode_retraction_backup in (None, "host_pool") + cfg.disaggregation_mode == "decode" + and cfg.disaggregation_decode_retraction_backup in (None, "host_pool") ) ): return @@ -7948,42 +8008,43 @@ class ServerArgs: self._resolve_hicache_dcp_compatibility() def _validate_hicache_host_memory_mode(self): - if self.hicache_host_memory_mode not in ("cache", "buffer_only"): + cfg = resolving_view(self) + if cfg.hicache_host_memory_mode not in ("cache", "buffer_only"): raise ValueError( "hicache_host_memory_mode must be 'cache' or 'buffer_only', " - f"got {self.hicache_host_memory_mode!r}" + f"got {cfg.hicache_host_memory_mode!r}" ) # Both modes are defaulted upstream (a decode server resolves the # ratio later, in kv_cache_builder), so this fires only if that # defaulting regresses -- never build an unsized host pool. if ( - self.hicache_size <= 0 - and self.hicache_ratio is None - and self.disaggregation_mode != "decode" + cfg.hicache_size <= 0 + and cfg.hicache_ratio is None + and cfg.disaggregation_mode != "decode" ): raise ValueError( - f"--hicache-host-memory-mode {self.hicache_host_memory_mode} " + f"--hicache-host-memory-mode {cfg.hicache_host_memory_mode} " "requires a host pool size: pass --hicache-size or " "--hicache-ratio." ) - if self.hicache_host_memory_mode == "cache": + if cfg.hicache_host_memory_mode == "cache": return - if self.hicache_storage_backend is None: + if cfg.hicache_storage_backend is None: raise ValueError( "--hicache-host-memory-mode buffer_only requires a storage backend " "(--hicache-storage-backend): host memory is only a staging buffer " "and all cached data lives in storage." ) - if self.hicache_write_policy == "write_back": + if cfg.hicache_write_policy == "write_back": raise ValueError( "--hicache-host-memory-mode buffer_only does not support " "--hicache-write-policy write_back; use write_through or " "write_through_selective." ) - if self.disaggregation_mode == "decode": + if cfg.disaggregation_mode == "decode": raise ValueError( "--hicache-host-memory-mode buffer_only is not supported on " "decode instances: the decode-side prefetch and offload paths " @@ -7993,9 +8054,10 @@ class ServerArgs: ) def _resolve_hicache_dcp_compatibility(self): - if self.dcp_size <= 1 or not self.enable_hierarchical_cache: + cfg = resolving_view(self) + if cfg.dcp_size <= 1 or not cfg.enable_hierarchical_cache: return - if self.hicache_storage_backend is not None: + if cfg.hicache_storage_backend is not None: raise NotImplementedError( "--hicache-storage-backend (L3) with --dcp-size > 1 is not " "supported yet: under DCP each rank holds a distinct " @@ -8003,18 +8065,18 @@ class ServerArgs: "backup and the storage keys must become dcp_rank-aware " "first. Run HiCache+DCP with L1/L2 only." ) - if self.speculative_algorithm not in (None, "DSPARK"): + if cfg.speculative_algorithm not in (None, "DSPARK"): raise NotImplementedError( "HiCache with --dcp-size > 1 only supports DSPARK speculative " "decoding; other draft-model host pools have no DCP index " "translation." ) - if self.enable_lmcache: + if cfg.enable_lmcache: raise NotImplementedError( "--enable-lmcache with --dcp-size > 1 is not supported: " "LMCache has no DCP-aware index translation." ) - if self.enable_hisparse: + if cfg.enable_hisparse: raise NotImplementedError( "--enable-hisparse with --dcp-size > 1 is not supported: the " "HiSparse host pool is constructed without DCP translation." @@ -8029,13 +8091,14 @@ class ServerArgs: "HiCache + DCP enabled (L1/L2 only): host pool uses widened " "logical slot accounting with per-rank physical translation at " "the transfer boundary (dcp_size=%d).", - self.dcp_size, + cfg.dcp_size, ) def _resolve_layout_io_compatibility(self): + cfg = resolving_view(self) if ( - self.hicache_mem_layout == "page_first_direct" - and self.hicache_io_backend == "kernel" + cfg.hicache_mem_layout == "page_first_direct" + and cfg.hicache_io_backend == "kernel" ): self._declare( "_resolve_layout_io_compatibility", @@ -8046,8 +8109,8 @@ class ServerArgs: ) if ( - self.hicache_mem_layout == "page_first" - and self.hicache_io_backend == "direct" + cfg.hicache_mem_layout == "page_first" + and cfg.hicache_io_backend == "direct" ): self._declare( "_resolve_layout_io_compatibility", @@ -8058,19 +8121,20 @@ class ServerArgs: ) def _resolve_storage_layout_compatibility(self): + cfg = resolving_view(self) if ( - self.hicache_storage_backend != "mooncake" - or self.hicache_mem_layout != "layer_first" + cfg.hicache_storage_backend != "mooncake" + or cfg.hicache_mem_layout != "layer_first" ): return - if self.hicache_io_backend == "direct": + if cfg.hicache_io_backend == "direct": new_layout = "page_first_direct" - elif self.hicache_io_backend == "kernel": + elif cfg.hicache_io_backend == "kernel": new_layout = "page_first" else: # Keep current behavior for unknown backends (e.g., kernel_ascend). - new_layout = self.hicache_mem_layout + new_layout = cfg.hicache_mem_layout self._declare( "_resolve_storage_layout_compatibility", @@ -8078,17 +8142,18 @@ class ServerArgs: ) logger.warning( f"Mooncake storage backend does not support layer_first layout, " - f"switching to {new_layout} layout for {self.hicache_io_backend} io backend" + f"switching to {new_layout} layout for {cfg.hicache_io_backend} io backend" ) def _resolve_hf_gguf_model_path(self): """Turn a Hub reference to a .gguf into a local file path.""" + cfg = resolving_view(self) from sglang.srt.utils.hf_transformers_utils import resolve_hf_gguf_reference - resolved = resolve_hf_gguf_reference(self.model_path, revision=self.revision) + resolved = resolve_hf_gguf_reference(cfg.model_path, revision=cfg.revision) if resolved is not None: - logger.info("Resolved GGUF %s -> %s", self.model_path, resolved) - if self.tokenizer_path == self.model_path: + logger.info("Resolved GGUF %s -> %s", cfg.model_path, resolved) + if cfg.tokenizer_path == cfg.model_path: self._declare( "_resolve_hf_gguf_model_path", tokenizer_path=resolved, @@ -8100,15 +8165,15 @@ class ServerArgs: # A speculative draft can be a .gguf too, and it is loaded by path, so it # needs the same Hub-reference resolution as the target. - if self.speculative_draft_model_path: + if cfg.speculative_draft_model_path: resolved_draft = resolve_hf_gguf_reference( - self.speculative_draft_model_path, - revision=self.speculative_draft_model_revision, + cfg.speculative_draft_model_path, + revision=cfg.speculative_draft_model_revision, ) if resolved_draft is not None: logger.info( "Resolved draft GGUF %s -> %s", - self.speculative_draft_model_path, + cfg.speculative_draft_model_path, resolved_draft, ) self._declare( @@ -8125,21 +8190,22 @@ class ServerArgs: # The quantization side of the gguf coupling moved to the pipeline # (arg_groups/overrides.py: _gguf_quantization); load_format itself is # genuine config (runtime user updates write it) and stays imperative. + cfg = resolving_view(self) from sglang.srt.arg_groups.overrides import ( _gguf_quantization, run_post_process_pass, ) run_post_process_pass(self, _gguf_quantization) - if ( - self.load_format == "auto" or self.load_format == "gguf" - ) and check_gguf_file(self.model_path): + if (cfg.load_format == "auto" or cfg.load_format == "gguf") and check_gguf_file( + cfg.model_path + ): self._declare( "_handle_load_format", load_format="gguf", ) - if self.load_format == "auto" and self._is_mistral_native_format(): + if cfg.load_format == "auto" and self._is_mistral_native_format(): self._declare( "_handle_load_format", load_format="mistral", @@ -8148,34 +8214,34 @@ class ServerArgs: "Detected Mistral native format checkpoint, setting load_format='mistral'" ) - if is_runai_obj_uri(self.model_path): + if is_runai_obj_uri(cfg.model_path): self._declare( "_handle_load_format", load_format="runai_streamer", ) - elif is_remote_url(self.model_path): + elif is_remote_url(cfg.model_path): self._declare( "_handle_load_format", load_format="remote", ) if ( - self.speculative_draft_model_path is not None - and is_runai_obj_uri(self.speculative_draft_model_path) - and self.speculative_draft_load_format is None + cfg.speculative_draft_model_path is not None + and is_runai_obj_uri(cfg.speculative_draft_model_path) + and cfg.speculative_draft_load_format is None ): self._declare( "_handle_load_format", speculative_draft_load_format="runai_streamer", ) - if self.custom_weight_loader is None: + if cfg.custom_weight_loader is None: self._declare("_handle_load_format", custom_weight_loader=[]) - if self.load_format == "remote_instance": - if self.remote_instance_weight_loader_backend != "modelexpress" and ( - self.remote_instance_weight_loader_seed_instance_ip is None - or self.remote_instance_weight_loader_seed_instance_service_port is None + if cfg.load_format == "remote_instance": + if cfg.remote_instance_weight_loader_backend != "modelexpress" and ( + cfg.remote_instance_weight_loader_seed_instance_ip is None + or cfg.remote_instance_weight_loader_seed_instance_service_port is None ): logger.warning( "Fallback load_format to 'auto' due to incomplete remote instance weight loader settings." @@ -8185,8 +8251,8 @@ class ServerArgs: load_format="auto", ) elif ( - self.remote_instance_weight_loader_send_weights_group_ports is None - and self.remote_instance_weight_loader_backend == "nccl" + cfg.remote_instance_weight_loader_send_weights_group_ports is None + and cfg.remote_instance_weight_loader_backend == "nccl" ): logger.warning( "Fallback load_format to 'auto' due to incomplete remote instance weight loader NCCL group ports settings." @@ -8196,7 +8262,7 @@ class ServerArgs: load_format="auto", ) elif ( - self.remote_instance_weight_loader_backend == "transfer_engine" + cfg.remote_instance_weight_loader_backend == "transfer_engine" and not self.validate_transfer_engine() ): logger.warning( @@ -8208,7 +8274,7 @@ class ServerArgs: ) # Check whether TransferEngine can be used when users want to start seed service that supports TransferEngine backend. - if self.remote_instance_weight_loader_start_seed_via_transfer_engine: + if cfg.remote_instance_weight_loader_start_seed_via_transfer_engine: self._declare( "_handle_load_format", remote_instance_weight_loader_start_seed_via_transfer_engine=self.validate_transfer_engine(), @@ -8220,7 +8286,7 @@ class ServerArgs: # launched, and fallback_load_format inherits a nonsensical format), so # reject it and point at the knob (defense-in-depth; the CLI already # rejects it via LOAD_FORMAT_CHOICES). - if self.load_format == "ipc_cache": + if cfg.load_format == "ipc_cache": raise ValueError( "load_format='ipc_cache' is an internal-only format and must not " "be set directly. Enable the weight cache via --weight-cache-mode " @@ -8231,7 +8297,7 @@ class ServerArgs: # Speculative decoding loads an extra draft model whose weights the # daemon does not export, so refuse the combination up front instead of # failing deep inside draft-worker load (draft-model daemon TBD). - if self.weight_cache_mode != "off" and self.speculative_algorithm is not None: + if cfg.weight_cache_mode != "off" and cfg.speculative_algorithm is not None: raise ValueError( "--weight-cache-mode is not supported together with speculative " "decoding (--speculative-algorithm): the weight cache daemon does " @@ -8239,7 +8305,7 @@ class ServerArgs: "(--weight-cache-mode off) for this configuration." ) - if self.weight_cache_mode != "off" and self.enable_eplb: + if cfg.weight_cache_mode != "off" and cfg.enable_eplb: raise ValueError( "--weight-cache-mode is not supported together with --enable-eplb." ) @@ -8258,13 +8324,14 @@ class ServerArgs: is present -- those families need Mistral weight loading regardless of which weight files happen to be present. """ + cfg = resolving_view(self) _MISTRAL_NATIVE_PATTERNS = ( "mistral-large-3", "mistral-small-4", "leanstral", ) name_matches = any( - p in str(self.model_path).lower() for p in _MISTRAL_NATIVE_PATTERNS + p in str(cfg.model_path).lower() for p in _MISTRAL_NATIVE_PATTERNS ) def _check_format(has_params, has_consolidated, has_hf_weights) -> bool: @@ -8272,23 +8339,21 @@ class ServerArgs: return True return has_consolidated and not has_hf_weights - if os.path.isdir(self.model_path): + if os.path.isdir(cfg.model_path): return _check_format( - has_params=os.path.exists(os.path.join(self.model_path, "params.json")), + has_params=os.path.exists(os.path.join(cfg.model_path, "params.json")), has_consolidated=bool( - glob.glob( - os.path.join(self.model_path, "consolidated*.safetensors") - ) + glob.glob(os.path.join(cfg.model_path, "consolidated*.safetensors")) ), has_hf_weights=bool( - glob.glob(os.path.join(self.model_path, "model*.safetensors")) + glob.glob(os.path.join(cfg.model_path, "model*.safetensors")) ), ) try: from huggingface_hub import HfApi - files = {s.rfilename for s in HfApi().model_info(self.model_path).siblings} + files = {s.rfilename for s in HfApi().model_info(cfg.model_path).siblings} return _check_format( has_params="params.json" in files, has_consolidated=any( @@ -8308,23 +8373,24 @@ class ServerArgs: LANGUAGE_MODEL_ONLY_ARCHITECTURES = ("MuseGlimmerForConditionalGeneration",) def _handle_language_model_only(self): - if not self.language_model_only: + cfg = resolving_view(self) + if not cfg.language_model_only: return for flag, name in ( - (self.encoder_only, "--encoder-only"), - (self.language_only, "--language-only"), - (self.enable_prefix_mm_cache, "--enable-prefix-mm-cache"), + (cfg.encoder_only, "--encoder-only"), + (cfg.language_only, "--language-only"), + (cfg.enable_prefix_mm_cache, "--enable-prefix-mm-cache"), ( - self.enable_broadcast_mm_inputs_process, + cfg.enable_broadcast_mm_inputs_process, "--enable-broadcast-mm-inputs-process", ), - (self.mm_enable_dp_encoder, "--mm-enable-dp-encoder"), + (cfg.mm_enable_dp_encoder, "--mm-enable-dp-encoder"), ): if flag: raise ValueError( f"--language-model-only cannot be combined with {name}" ) - if self.disaggregation_mode != "null": + if cfg.disaggregation_mode != "null": raise ValueError( "--language-model-only is incompatible with --disaggregation-mode " "prefill/decode" @@ -8337,19 +8403,20 @@ class ServerArgs: ) def _handle_encoder_disaggregation(self): + cfg = resolving_view(self) self._handle_language_model_only() - if self.enable_prefix_mm_cache and not self.encoder_only: + if cfg.enable_prefix_mm_cache and not cfg.encoder_only: raise ValueError( "--enable-prefix-mm-cache requires --encoder-only to be enabled" ) - if self.encoder_only and self.language_only: + if cfg.encoder_only and cfg.language_only: raise ValueError("Cannot set --encoder-only and --language-only together") - if self.encoder_only and not self.disaggregation_mode == "null": + if cfg.encoder_only and not cfg.disaggregation_mode == "null": raise ValueError( "Cannot set --encoder-only and --disaggregation-mode prefill/decode together" ) - if self.language_only and len(self.encoder_urls) == 0: + if cfg.language_only and len(cfg.encoder_urls) == 0: logger.info( "--language-only is set without --encoder-urls. Encoders are " "expected to register dynamically via the " @@ -8358,34 +8425,34 @@ class ServerArgs: # Validate IB devices when mooncake backend is used if ( - self.disaggregation_transfer_backend == "mooncake" - and self.disaggregation_mode in ("prefill", "decode") - ) or self.encoder_transfer_backend == "mooncake": + cfg.disaggregation_transfer_backend == "mooncake" + and cfg.disaggregation_mode in ("prefill", "decode") + ) or cfg.encoder_transfer_backend == "mooncake": self._declare( "_handle_encoder_disaggregation", disaggregation_ib_device=self._validate_ib_devices( - self.disaggregation_ib_device + cfg.disaggregation_ib_device ), ) # Validate model type for encoder disaggregation hf_config = self.get_model_config().hf_config model_arch = hf_config.architectures[0] - if self.encoder_transfer_backend == "auto": + if cfg.encoder_transfer_backend == "auto": self._declare( "_handle_encoder_disaggregation", encoder_transfer_backend=resolve_encoder_transfer_backend( - self.encoder_transfer_backend, model_arch, self.tp_size + cfg.encoder_transfer_backend, model_arch, cfg.tp_size ), ) - if self.encoder_only or self.language_only: + if cfg.encoder_only or cfg.language_only: logger.info( "Encoder transfer backend auto-resolved to %s for %s at TP%d.", - self.encoder_transfer_backend, + cfg.encoder_transfer_backend, model_arch, - self.tp_size, + cfg.tp_size, ) - if (self.encoder_only or self.language_only) and model_arch not in [ + if (cfg.encoder_only or cfg.language_only) and model_arch not in [ "Qwen2VLForConditionalGeneration", "Qwen3VLForConditionalGeneration", "Qwen2_5_VLForConditionalGeneration", @@ -8484,23 +8551,24 @@ class ServerArgs: return json.dumps(normalized_mapping, separators=(",", ":")) def _handle_tokenizer_batching(self): - if self.enable_tokenizer_batch_encode and self.enable_dynamic_batch_tokenizer: + cfg = resolving_view(self) + if cfg.enable_tokenizer_batch_encode and cfg.enable_dynamic_batch_tokenizer: raise ValueError( "Cannot enable both --enable-tokenizer-batch-encode and --enable-dynamic-batch-tokenizer. " "Please choose one tokenizer batching approach." ) - if self.skip_tokenizer_init and not envs.SGLANG_RUST_SERVER.get(): + if cfg.skip_tokenizer_init and not envs.SGLANG_RUST_SERVER.get(): # Tokenizer workers still serve HTTP / state / output work, so # their fanout is preserved; detokenizer workers only decode. - if self.detokenizer_worker_num != 1: + if cfg.detokenizer_worker_num != 1: logger.warning( "skip_tokenizer_init=True leaves no decode work for detokenizer workers; " - f"forcing detokenizer_worker_num=1 (requested {self.detokenizer_worker_num})." + f"forcing detokenizer_worker_num=1 (requested {cfg.detokenizer_worker_num})." ) self._declare("_handle_tokenizer_batching", detokenizer_worker_num=1) - if self.enable_tokenizer_batch_encode: + if cfg.enable_tokenizer_batch_encode: logger.warning( "skip_tokenizer_init=True ignores --enable-tokenizer-batch-encode; disabling it." ) @@ -8509,7 +8577,7 @@ class ServerArgs: enable_tokenizer_batch_encode=False, ) - if self.enable_dynamic_batch_tokenizer: + if cfg.enable_dynamic_batch_tokenizer: logger.warning( "skip_tokenizer_init=True ignores --enable-dynamic-batch-tokenizer; disabling it." ) @@ -8531,11 +8599,12 @@ class ServerArgs: may still auto-select CUDA VMM. The legacy CUDA IPC flag and environment variable remain supported so existing deployments map to this policy. """ - requested_transport = self.mm_feature_transport + cfg = resolving_view(self) + requested_transport = cfg.mm_feature_transport legacy_ipc_is_set = envs.SGLANG_USE_CUDA_IPC_TRANSPORT.is_set() legacy_ipc_enabled = envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get() - if self.keep_mm_feature_on_device: + if cfg.keep_mm_feature_on_device: if requested_transport not in (None, "cuda_ipc"): raise ValueError( "--keep-mm-feature-on-device conflicts with " @@ -8556,7 +8625,7 @@ class ServerArgs: "--mm-feature-transport=%s instead.", requested_transport, ) - elif self.encoder_only: + elif cfg.encoder_only: requested_transport = "cpu" logger.info( "Multimodal feature transport auto-resolved to cpu for " @@ -8566,14 +8635,14 @@ class ServerArgs: elif ( self.get_model_config().is_multimodal and is_cuda() - and self.disaggregation_mode == "null" + and cfg.disaggregation_mode == "null" ): # A full GPU pool always degrades to CPU transport per tensor. # Keep CUDA IPC opt-in because even an idle pool consumes HBM # that would otherwise back the KV cache. Multi-node # auto-selection is limited to GB200/GB300 systems where the # runtime already enables the MNNVL/IMEX communication stack. - if self.nnodes == 1: + if cfg.nnodes == 1: requested_transport = "cpu" elif is_mnnvl_fabric_device() and os.path.exists( "/dev/nvidia-caps-imex-channels/channel0" @@ -8616,7 +8685,7 @@ class ServerArgs: int(legacy_ipc_enabled), ) - if self.encoder_only and requested_transport in ("cuda_ipc", "cuda_vmm"): + if cfg.encoder_only and requested_transport in ("cuda_ipc", "cuda_vmm"): logger.warning( "--mm-feature-transport=%s does not control encoder-only " "output transfer; using cpu for this inactive transport. Select " @@ -8630,7 +8699,7 @@ class ServerArgs: raise ValueError( "--mm-feature-transport=cuda_vmm requires NVIDIA CUDA." ) - if self.pp_size != 1: + if cfg.pp_size != 1: raise ValueError( "--mm-feature-transport=cuda_vmm does not support pipeline " "parallelism." @@ -8641,7 +8710,7 @@ class ServerArgs: "SGLANG_RUST_SERVER." ) pool_budget_mb = envs.SGLANG_MM_FEATURE_CACHE_MB.get() - handle_kind = "CUDA FABRIC" if self.nnodes > 1 else "POSIX FD" + handle_kind = "CUDA FABRIC" if cfg.nnodes > 1 else "POSIX FD" logger.info( "Using CUDA VMM for multimodal features with %s sharing: " "reserving up to %d MiB on base GPU %d across %d tokenizer " @@ -8649,8 +8718,8 @@ class ServerArgs: "back to inline CPU transport.", handle_kind, pool_budget_mb, - self.base_gpu_id, - self.tokenizer_worker_num, + cfg.base_gpu_id, + cfg.tokenizer_worker_num, ) if requested_transport == "cuda_ipc": @@ -8658,7 +8727,7 @@ class ServerArgs: raise ValueError( "--mm-feature-transport=cuda_ipc requires NVIDIA CUDA." ) - if self.nnodes != 1: + if cfg.nnodes != 1: raise ValueError( "--mm-feature-transport=cuda_ipc only supports a single node." ) @@ -8669,8 +8738,8 @@ class ServerArgs: "on base GPU %d across %d tokenizer worker(s). This reduces KV " "cache headroom; a full pool falls back to CPU transport.", pool_budget_mb, - self.base_gpu_id, - self.tokenizer_worker_num, + cfg.base_gpu_id, + cfg.tokenizer_worker_num, ) logger.info( "CUDA IPC pool-handle caching is %s. It reuses mappings to the " @@ -8698,19 +8767,20 @@ class ServerArgs: ) def _handle_environment_variables(self): + cfg = resolving_view(self) self._handle_multimodal_feature_transport() - envs.SGLANG_ENABLE_TORCH_COMPILE.set("1" if self.enable_torch_compile else "0") - if self.mamba_ssm_dtype is not None: - envs.SGLANG_MAMBA_SSM_DTYPE.set(self.mamba_ssm_dtype) + envs.SGLANG_ENABLE_TORCH_COMPILE.set("1" if cfg.enable_torch_compile else "0") + if cfg.mamba_ssm_dtype is not None: + envs.SGLANG_MAMBA_SSM_DTYPE.set(cfg.mamba_ssm_dtype) envs.SGLANG_DISABLE_OUTLINES_DISK_CACHE.set( - "1" if self.disable_outlines_disk_cache else "0" + "1" if cfg.disable_outlines_disk_cache else "0" ) envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.set( - "1" if self.enable_deterministic_inference else "0" + "1" if cfg.enable_deterministic_inference else "0" ) - if self.enable_deterministic_inference: + if cfg.enable_deterministic_inference: envs.SGLANG_FLASHINFER_MOE_FUSED_FINALIZE.set("0") - if self.debug_cuda_graph: + if cfg.debug_cuda_graph: if not (is_cuda() or is_hip()): logger.warning( "--debug-cuda-graph is not supported on non CUDA/HIP devices. " @@ -8723,7 +8793,7 @@ class ServerArgs: "Debug mode for CUDA graph is enabled via breakable CUDA graph. " "All operations will run eagerly through the graph capture/replay path." ) - if self.enable_deepseek_v4_fp4_indexer and not ( + if cfg.enable_deepseek_v4_fp4_indexer and not ( is_sm100_supported() or is_sm120_supported() ): raise ValueError( @@ -8755,48 +8825,49 @@ class ServerArgs: envs.SGLANG_OPT_FP8_WO_A_GEMM.set(False) def _handle_cache_compatibility(self): + cfg = resolving_view(self) if ( - self.disaggregation_decode_retraction_backup == "host_pool" - and self.disaggregation_mode != "decode" + cfg.disaggregation_decode_retraction_backup == "host_pool" + and cfg.disaggregation_mode != "decode" ): raise ValueError( "--disaggregation-decode-retraction-backup=host_pool is only " "supported on a PD decode server." ) if ( - self.disaggregation_decode_retraction_backup == "host_pool" - and self.dcp_size > 1 + cfg.disaggregation_decode_retraction_backup == "host_pool" + and cfg.dcp_size > 1 ): raise ValueError( "--disaggregation-decode-retraction-backup=host_pool does not " "support --dcp-size > 1." ) if ( - self.disaggregation_decode_retraction_backup == "host_pool" - and self.enable_priority_scheduling - and not self.disable_priority_preemption + cfg.disaggregation_decode_retraction_backup == "host_pool" + and cfg.enable_priority_scheduling + and not cfg.disable_priority_preemption ): raise ValueError( "--disaggregation-decode-retraction-backup=host_pool requires " "--disable-priority-preemption when priority scheduling is enabled." ) - if self.enable_hierarchical_cache and self.disable_radix_cache: + if cfg.enable_hierarchical_cache and cfg.disable_radix_cache: raise ValueError( "The arguments enable-hierarchical-cache and disable-radix-cache are mutually exclusive " "and cannot be used at the same time. Please use only one of them." ) - if self.disaggregation_decode_enable_offload_kvcache: - if self.disaggregation_mode != "decode": + if cfg.disaggregation_decode_enable_offload_kvcache: + if cfg.disaggregation_mode != "decode": raise ValueError( "The argument disaggregation-decode-enable-offload-kvcache is only supported for decode side." ) - if self.hicache_storage_backend is None: + if cfg.hicache_storage_backend is None: raise ValueError( "The argument disaggregation-decode-enable-offload-kvcache is only supported when hicache-storage-backend is provided." ) - if self.disaggregation_decode_retraction_backup == "host_pool": + if cfg.disaggregation_decode_retraction_backup == "host_pool": raise ValueError( "The arguments disaggregation-decode-enable-offload-kvcache and " "disaggregation-decode-retraction-backup=host_pool are mutually exclusive: " @@ -8810,7 +8881,8 @@ class ServerArgs: raise ValueError("--swa-full-tokens-ratio should be in range (0, 1.0].") def _handle_deterministic_inference(self): - if self.rl_on_policy_target is not None: + cfg = resolving_view(self) + if cfg.rl_on_policy_target is not None: logger.warning( "Enable deterministic inference because of rl_on_policy_target." ) @@ -8824,8 +8896,8 @@ class ServerArgs: # TODO remove this environment variable as a whole envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.set(True) - if self.enable_deterministic_inference: - if self.enable_aiter_allreduce_fusion: + if cfg.enable_deterministic_inference: + if cfg.enable_aiter_allreduce_fusion: logger.warning( "Disable --enable-aiter-allreduce-fusion because deterministic inference is enabled." ) @@ -8855,7 +8927,7 @@ class ServerArgs: run_post_process_pass(self, _deterministic_sampling_backend) is_deepseek_model = False - if parse_connector_type(self.model_path) != ConnectorType.INSTANCE: + if parse_connector_type(cfg.model_path) != ConnectorType.INSTANCE: try: hf_config = self.get_model_config().hf_config model_arch = hf_config.architectures[0] @@ -8902,7 +8974,7 @@ class ServerArgs: ) # Check TP size - if self.tp_size > 1: + if cfg.tp_size > 1: if is_hip(): # AMD: use 1-stage all-reduce kernel which is inherently deterministic # (each GPU reads all data from all GPUs, reduces locally in fixed order) @@ -8937,17 +9009,18 @@ class ServerArgs: ) def _handle_unified_memory_pool(self): - if not self.enable_unified_memory: + cfg = resolving_view(self) + if not cfg.enable_unified_memory: return - if self.disaggregation_mode != "null": + if cfg.disaggregation_mode != "null": # Constraints of the whole-envelope transfer; see # UnifiedMLATokenToKVPool.get_contiguous_buf_infos. - assert self.disaggregation_transfer_backend == "mooncake", ( + assert cfg.disaggregation_transfer_backend == "mooncake", ( "--enable-unified-memory with PD disaggregation supports only " "the mooncake transfer backend; got " - f"{self.disaggregation_transfer_backend!r}." + f"{cfg.disaggregation_transfer_backend!r}." ) - assert self.pp_size == 1, ( + assert cfg.pp_size == 1, ( "--enable-unified-memory with PD disaggregation does not support " "pipeline parallelism (whole-envelope transfer has no per-layer " "entries to subset)." @@ -8956,24 +9029,24 @@ class ServerArgs: "--enable-unified-memory with PD disaggregation requires lazy " "compaction; unset SGLANG_DISABLE_LAZY_COMPACTION." ) - assert not self.enable_hisparse, ( + assert not cfg.enable_hisparse, ( "--enable-unified-memory with PD disaggregation is not compatible " "with --enable-hisparse: the decode-side HiSparse prealloc path " "ships host/C4 rows straight from the allocator, bypassing the " "virtual->physical translation the unified pool needs." ) - assert self.speculative_algorithm in (None, "DSPARK"), ( + assert cfg.speculative_algorithm in (None, "DSPARK"), ( "--enable-unified-memory only supports --speculative-algorithm " "DSPARK (chain draft); other speculative algorithms are not yet " "audited for the unified pool's virtual/dense loc translation. Got " - f"--speculative-algorithm={self.speculative_algorithm!r}." + f"--speculative-algorithm={cfg.speculative_algorithm!r}." ) - if self.speculative_algorithm == "DSPARK": - assert self.speculative_eagle_topk in (None, 1), ( + if cfg.speculative_algorithm == "DSPARK": + assert cfg.speculative_eagle_topk in (None, 1), ( "--enable-unified-memory + DSPARK supports a linear draft " "chain only (--speculative-eagle-topk in {None, 1}); tree " "verify is not audited for the unified pool. Got " - f"--speculative-eagle-topk={self.speculative_eagle_topk!r}." + f"--speculative-eagle-topk={cfg.speculative_eagle_topk!r}." ) # Both roles: verify routes to either backend depending on # --speculative-attention-mode. @@ -8987,14 +9060,14 @@ class ServerArgs: "not translate speculative verify indices to the unified " "pool's dense space yet." ) - assert not (self.enable_hierarchical_cache or self.enable_lmcache), ( + assert not (cfg.enable_hierarchical_cache or cfg.enable_lmcache), ( "--enable-unified-memory is not yet compatible with hierarchical / " "host-tiered KV cache (--enable-hierarchical-cache / --enable-lmcache): " "the unified-memory-pool init wires up no host pools, and its device mamba / " "full-attention slots are VIRTUAL — the host-offload path does not " "translate them to physical." ) - assert self.dcp_size == 1, ( + assert cfg.dcp_size == 1, ( "--enable-unified-memory is not yet compatible with decode context " "parallelism (--dcp-size > 1): the pool has no DCP-aware masked write " "path (UnifiedMHATokenToKVPool.set_kv_buffer asserts dcp_kv_mask is None), " @@ -9002,7 +9075,7 @@ class ServerArgs: ) # Only monolithic decode cuda-graph capture is wired; piecewise prefill # capture is not. Guard when the user opts into it. - _cg_cfg = self.cuda_graph_config + _cg_cfg = cfg.cuda_graph_config if _cg_cfg is not None and _cg_cfg.prefill.backend == Backend.TC_PIECEWISE: raise ValueError( "--enable-unified-memory supports monolithic (decode) " @@ -9018,12 +9091,13 @@ class ServerArgs: # The unified pool stores state in the page-major envelope-strided layout, so # enabling it implies --enable-page-major-kv-layout — routing it through the # single page-major path + stride-aware Triton asserts (set before the guard). - if self.enable_unified_memory: + cfg = resolving_view(self) + if cfg.enable_unified_memory: self._declare( "_handle_page_major_kv_layout", enable_page_major_kv_layout=True, ) - if not self.enable_page_major_kv_layout: + if not cfg.enable_page_major_kv_layout: return # Only the Triton attention kernels read the strided 4-D envelope K/V # views; FA3 / FlashInfer do not. EXCEPTION: the unified-memory MLA pool @@ -9037,7 +9111,7 @@ class ServerArgs: # page_table (in-kernel for captured decode, one funnel for eager). # flashmla / cutlass_mla share the create_flashmla block-table path and # can be added the same way once exercised. - if self.enable_unified_memory and self.use_mla_backend(): + if cfg.enable_unified_memory and self.use_mla_backend(): allowed_full = { "triton", "fa3", @@ -9074,10 +9148,10 @@ class ServerArgs: decode_allowed.update({"cutedsl", "helion"}) prefill_allowed.update({"cutedsl", "helion"}) resolved_linear_decode = ( - self.linear_attn_decode_backend or self.linear_attn_backend + cfg.linear_attn_decode_backend or cfg.linear_attn_backend ) resolved_linear_prefill = ( - self.linear_attn_prefill_backend or self.linear_attn_backend + cfg.linear_attn_prefill_backend or cfg.linear_attn_backend ) assert resolved_linear_decode in decode_allowed | {None}, ( "--enable-page-major-kv-layout: linear-attention DECODE backend must " @@ -9089,28 +9163,29 @@ class ServerArgs: f"be one of {sorted(prefill_allowed)} for the strided conv/SSM state; " f"got {resolved_linear_prefill!r}." ) - assert self.mamba_backend in (None, "triton"), ( + assert cfg.mamba_backend in (None, "triton"), ( "--enable-page-major-kv-layout requires the Triton Mamba kernels for " - f"the strided conv/SSM state; got {self.mamba_backend!r}. Pass " + f"the strided conv/SSM state; got {cfg.mamba_backend!r}. Pass " "--mamba-backend triton." ) def _handle_dllm_inference(self): - if self.dllm_algorithm is None: + cfg = resolving_view(self) + if cfg.dllm_algorithm is None: return # On AMD/HIP, disable cuda graph for DLLM (the attention_backend # resolution moved to the pipeline: arg_groups/overrides.py # _dllm_attention_backend, invoked below at its legacy slot). if is_hip(): if ( - self.cuda_graph_config.decode.backend != Backend.DISABLED - or self.cuda_graph_config.prefill.backend != Backend.DISABLED + cfg.cuda_graph_config.decode.backend != Backend.DISABLED + or cfg.cuda_graph_config.prefill.backend != Backend.DISABLED ): logger.warning( "Cuda graph is disabled for diffusion LLM inference on AMD GPUs" ) - self.cuda_graph_config.decode.backend = Backend.DISABLED - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED from sglang.srt.arg_groups.overrides import ( _dllm_attention_backend, @@ -9130,8 +9205,8 @@ class ServerArgs: run_post_process_pass(self, _dllm_page_size) - if not self.disable_radix_cache: - if self.enable_hierarchical_cache: + if not cfg.disable_radix_cache: + if cfg.enable_hierarchical_cache: logger.warning( "Hierarchical cache is disabled because of using diffusion LLM inference" ) @@ -9139,18 +9214,18 @@ class ServerArgs: "_handle_dllm_inference", enable_hierarchical_cache=False, ) - if self.enable_lmcache: + if cfg.enable_lmcache: logger.warning( "LMCache is disabled because of using diffusion LLM inference" ) self._declare("_handle_dllm_inference", enable_lmcache=False) - if self.enable_flexkv: + if cfg.enable_flexkv: logger.warning( "FlexKV is disabled because of using diffusion LLM inference" ) self._declare("_handle_dllm_inference", enable_flexkv=False) - if self.pp_size > 1: + if cfg.pp_size > 1: logger.warning( "Pipeline parallelism is disabled because of using diffusion LLM inference" ) @@ -9159,13 +9234,13 @@ class ServerArgs: pp_size=1, ) - if self.enable_lora: + if cfg.enable_lora: logger.warning( "Currently LoRA is not supported by diffusion LLM inference." ) self._declare("_handle_dllm_inference", enable_lora=False) - if self.disaggregation_mode != "null": + if cfg.disaggregation_mode != "null": logger.warning( "Currently disaggregation is not supported by diffusion LLM inference." ) @@ -9174,7 +9249,7 @@ class ServerArgs: disaggregation_mode="null", ) - if self.enable_mixed_chunk: + if cfg.enable_mixed_chunk: logger.warning( "Mixed chunked prefill is disabled because of using diffusion LLM inference." ) @@ -9185,43 +9260,43 @@ class ServerArgs: def _handle_asr_validation(self): """Validate transcription/ASR-specific server args.""" - if self.asr_max_buffer_seconds <= 0: + cfg = resolving_view(self) + if cfg.asr_max_buffer_seconds <= 0: raise ValueError( f"--asr-max-buffer-seconds must be positive " - f"(got {self.asr_max_buffer_seconds})." + f"(got {cfg.asr_max_buffer_seconds})." ) - if self.asr_max_concurrent_sessions <= 0: + if cfg.asr_max_concurrent_sessions <= 0: raise ValueError( f"--asr-max-concurrent-sessions must be positive " - f"(got {self.asr_max_concurrent_sessions})." + f"(got {cfg.asr_max_concurrent_sessions})." ) def _validate_prefill_decode_interval(self): - if self.prefill_decode_interval < 0: + cfg = resolving_view(self) + if cfg.prefill_decode_interval < 0: raise ValueError("--prefill-decode-interval must be non-negative.") def _handle_other_validations(self): - if self.default_chat_template_kwargs is not None and not isinstance( - self.default_chat_template_kwargs, dict + cfg = resolving_view(self) + if cfg.default_chat_template_kwargs is not None and not isinstance( + cfg.default_chat_template_kwargs, dict ): raise ValueError( "--default-chat-template-kwargs must decode to a JSON object" ) # Handle optimistic prefill validation - if ( - self.optimistic_prefill_attempts > 0 - and self.disaggregation_mode == "prefill" - ): - if self.pp_size > 1: + if cfg.optimistic_prefill_attempts > 0 and cfg.disaggregation_mode == "prefill": + if cfg.pp_size > 1: logger.warning("Optimistic prefill does not support pp_size > 1") self._declare( "_handle_other_validations", optimistic_prefill_attempts=0, ) - elif self.enable_hierarchical_cache and ( - self.hicache_storage_backend is not None - or self.hicache_write_policy != "write_back" + elif cfg.enable_hierarchical_cache and ( + cfg.hicache_storage_backend is not None + or cfg.hicache_write_policy != "write_back" ): logger.warning( "Optimistic prefill only supports L2 hierarchical cache " @@ -9242,37 +9317,35 @@ class ServerArgs: ) # Handle model inference tensor dump. - if self.debug_tensor_dump_output_folder is not None: + if cfg.debug_tensor_dump_output_folder is not None: logger.warning( "Cuda graph and server warmup are disabled because of using tensor dump mode" ) - self.cuda_graph_config.decode.backend = Backend.DISABLED - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED self._declare("_handle_other_validations", skip_server_warmup=True) - if self.msprobe_dump_config is not None: + if cfg.msprobe_dump_config is not None: logger.warning( "When msProbe is enabled, " "cuda graph is disabled because msProbe only supports dump in eager mode, " "warmup is disabled(skip_server_warmup=True) because there is no need to dump data for this stage." ) - self.cuda_graph_config.decode.backend = Backend.DISABLED - self.cuda_graph_config.prefill.backend = Backend.DISABLED + cfg.cuda_graph_config.decode.backend = Backend.DISABLED + cfg.cuda_graph_config.prefill.backend = Backend.DISABLED self._declare("_handle_other_validations", skip_server_warmup=True) # Validate limit_mm_per_prompt modalities - if self.limit_mm_data_per_request: - if isinstance(self.limit_mm_data_per_request, str): + if cfg.limit_mm_data_per_request: + if isinstance(cfg.limit_mm_data_per_request, str): self._declare( "_handle_other_validations", - limit_mm_data_per_request=json.loads( - self.limit_mm_data_per_request - ), + limit_mm_data_per_request=json.loads(cfg.limit_mm_data_per_request), ) - if isinstance(self.limit_mm_data_per_request, dict): + if isinstance(cfg.limit_mm_data_per_request, dict): allowed_modalities = {"image", "video", "audio"} - for modality in self.limit_mm_data_per_request.keys(): + for modality in cfg.limit_mm_data_per_request.keys(): if modality not in allowed_modalities: raise ValueError( f"Invalid modality '{modality}' in --limit-mm-data-per-request." @@ -9280,25 +9353,24 @@ class ServerArgs: ) # Validate preferred_sampling_params - if self.preferred_sampling_params: - if isinstance(self.preferred_sampling_params, str): + if cfg.preferred_sampling_params: + if isinstance(cfg.preferred_sampling_params, str): self._declare( "_handle_other_validations", - preferred_sampling_params=json.loads( - self.preferred_sampling_params - ), + preferred_sampling_params=json.loads(cfg.preferred_sampling_params), ) # Validate preferred_sampling_params doesn't use tokenizer-dependent features - if self.skip_tokenizer_init: + if cfg.skip_tokenizer_init: from sglang.srt.sampling.sampling_params import SamplingParams - test_params = SamplingParams(**self.preferred_sampling_params) + test_params = SamplingParams(**cfg.preferred_sampling_params) # raises if tokenizer-dependent features used test_params.normalize(None) def _handle_crash_dump_env(self): - if not self.crash_dump_folder: + cfg = resolving_view(self) + if not cfg.crash_dump_folder: return _CUDA_COREDUMP_DEFAULTS = { "CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "1", @@ -9308,7 +9380,7 @@ class ServerArgs: "skip_nonrelocated_elf_images,skip_global_memory," "skip_shared_memory,skip_local_memory,skip_constbank_memory" ), - "CUDA_COREDUMP_FILE": f"{self.crash_dump_folder}/%h/core.cuda.%t.%p", + "CUDA_COREDUMP_FILE": f"{cfg.crash_dump_folder}/%h/core.cuda.%t.%p", "CUDA_COREDUMP_PIPE": "/tmp/corepipe.cuda.%h.%p", } for key, value in _CUDA_COREDUMP_DEFAULTS.items(): @@ -9338,7 +9410,8 @@ class ServerArgs: ) def _handle_debug_utils(self): - if is_in_ci() and self.soft_watchdog_timeout is None: + cfg = resolving_view(self) + if is_in_ci() and cfg.soft_watchdog_timeout is None: logger.info("Set soft_watchdog_timeout since in CI") self._declare("_handle_debug_utils", soft_watchdog_timeout=300) @@ -9638,15 +9711,21 @@ class ServerArgs: @property def is_ep_joiner(self) -> bool: """True for processes launched as elastic-EP joiners.""" - return self.ep_join_mode in ("scale", "recover") + cfg = resolving_view(self) + + return cfg.ep_join_mode in ("scale", "recover") @property def is_ep_scale_joiner(self) -> bool: - return self.ep_join_mode == "scale" + cfg = resolving_view(self) + + return cfg.ep_join_mode == "scale" @property def is_startup_weight_load_overlap(self) -> bool: - return self.startup_weight_load_mode == "overlap" + cfg = resolving_view(self) + + return cfg.startup_weight_load_mode == "overlap" def ssl_verify(self): """Return the value for the requests library's verify= parameter. @@ -9676,6 +9755,7 @@ class ServerArgs: def get_model_config(self): # Lazy init to avoid circular import + cfg = resolving_view(self) from sglang.srt.configs.model_config import ModelConfig memo = getattr(self, "model_config", None) @@ -9688,12 +9768,12 @@ class ServerArgs: # object-store URI, so its field is not the key. A configuration a # fixture supplied carries no key and is handed back as it is. built_from = getattr(self, "_model_config_built_from", None) - if built_from is None or built_from == self.model_path: + if built_from is None or built_from == cfg.model_path: return memo model_config = ModelConfig.from_server_args(self) self.model_config = model_config - self._model_config_built_from = self.model_path + self._model_config_built_from = cfg.model_path if model_config.is_hybrid_swa: logger.info( "Hybrid SWA model detected. architectures=%s", @@ -9757,17 +9837,18 @@ class ServerArgs: return model_config.attention_arch == AttentionArch.MLA def is_attention_backend_not_set(self): + cfg = resolving_view(self) return ( - self.attention_backend is None - and self.prefill_attention_backend is None - and self.decode_attention_backend is None + cfg.attention_backend is None + and cfg.prefill_attention_backend is None + and cfg.decode_attention_backend is None ) def enable_mamba_extra_buffer(self) -> bool: - return mamba_extra_buffer_of(self) + return mamba_extra_buffer_of(resolving_view(self)) def enable_mamba_extra_buffer_lazy(self) -> bool: - return mamba_extra_buffer_lazy_of(self) + return mamba_extra_buffer_lazy_of(resolving_view(self)) @property def max_speculative_num_draft_tokens(self) -> Optional[int]: @@ -9778,20 +9859,22 @@ class ServerArgs: sizing fills `speculative_num_draft_tokens` in), and a cache filled that early would keep answering with it. """ + cfg = resolving_view(self) + memo = self.__dict__.get("_max_speculative_num_draft_tokens") if memo is not None: return memo - if self.speculative_num_draft_tokens is None: + if cfg.speculative_num_draft_tokens is None: result = None - elif not self.speculative_adaptive: - result = self.speculative_num_draft_tokens + elif not cfg.speculative_adaptive: + result = cfg.speculative_num_draft_tokens else: from sglang.srt.speculative.adaptive_spec_params import ( resolve_candidate_steps_from_config, ) candidate_steps = resolve_candidate_steps_from_config( - cfg_path=self.speculative_adaptive_config, + cfg_path=cfg.speculative_adaptive_config, ) # TODO: adaptive spec currently requires topk=1, so each runtime # state needs steps + 1 draft-token slots. Revisit this if topk>1 @@ -9836,15 +9919,17 @@ class ServerArgs: # DP TP-MoE path (overlapping the DP all_gatherv / reduce_scatterv with # the other ubatch's compute), which requires DP attention. Enabling it # there needs no extra opt-in env flag. + cfg = resolving_view(self) + cp_tbo = ( is_hip() - and self.enable_dsa_prefill_context_parallel - and self.dsa_prefill_cp_mode == "round-robin-split" + and cfg.enable_dsa_prefill_context_parallel + and cfg.dsa_prefill_cp_mode == "round-robin-split" ) if ( - self.enable_two_batch_overlap - and self.moe_a2a_backend == "none" - and not self.enable_dp_attention + cfg.enable_two_batch_overlap + and cfg.moe_a2a_backend == "none" + and not cfg.enable_dp_attention and not cp_tbo ): raise ValueError( @@ -9854,20 +9939,22 @@ class ServerArgs: ) def check_server_args(self): + cfg = resolving_view(self) + # Check parallel size constraints - if self.ep_join_mode != "scale": + if cfg.ep_join_mode != "scale": assert ( - self.tp_size * self.pp_size - ) % self.nnodes == 0, "tp_size must be divisible by number of nodes" + cfg.tp_size * cfg.pp_size + ) % cfg.nnodes == 0, "tp_size must be divisible by number of nodes" assert ( - self.pp_max_micro_batch_size is None or self.pp_max_micro_batch_size >= 1 + cfg.pp_max_micro_batch_size is None or cfg.pp_max_micro_batch_size >= 1 ), ( "pp_max_micro_batch_size must be a positive integer or None (for auto-compute). " - f"Got: {self.pp_max_micro_batch_size}" + f"Got: {cfg.pp_max_micro_batch_size}" ) - assert not (self.disable_cuda_graph_padding and self.enable_torch_compile), ( + assert not (cfg.disable_cuda_graph_padding and cfg.enable_torch_compile), ( "--disable-cuda-graph-padding is incompatible with --enable-torch-compile. " "With padding disabled, every distinct batch size gets its own torch.compile + " "Triton autotune cycle (O(max_batch_size) compilations) instead of the small fixed " @@ -9875,67 +9962,67 @@ class ServerArgs: "Remove --disable-cuda-graph-padding or --enable-torch-compile." ) - if self.pp_size > 1: + if cfg.pp_size > 1: assert ( - self.disable_overlap_schedule and self.speculative_algorithm is None + cfg.disable_overlap_schedule and cfg.speculative_algorithm is None ), "Pipeline parallelism is not compatible with overlap schedule, speculative decoding" - assert self.min_free_slots_delay is None, ( + assert cfg.min_free_slots_delay is None, ( "--min-free-slots-delay is not supported with pipeline " "parallelism: allocatable slots per microbatch are bounded by " "pp-max-micro-batch-size, so the threshold may never be reached" ) assert not ( - self.dp_size > 1 and self.nnodes != 1 and not self.enable_dp_attention + cfg.dp_size > 1 and cfg.nnodes != 1 and not cfg.enable_dp_attention ), "multi-node data parallel is not supported unless dp attention!" - assert self.base_gpu_id >= 0, "base_gpu_id must be non-negative" - assert self.gpu_id_step >= 1, "gpu_id_step must be positive" + assert cfg.base_gpu_id >= 0, "base_gpu_id must be non-negative" + assert cfg.gpu_id_step >= 1, "gpu_id_step must be positive" - assert self.moe_dense_tp_size in ( + assert cfg.moe_dense_tp_size in ( None, 1, - self.tp_size, + cfg.tp_size, ), "moe_dense_tp_size only supports None, 1, or tp_size currently" # Check served model name to not have colon as it is reserved for LoRA adapter syntax - if not is_runai_obj_uri(self.served_model_name): - assert ":" not in self.served_model_name, ( + if not is_runai_obj_uri(cfg.served_model_name): + assert ":" not in cfg.served_model_name, ( "served_model_name cannot contain a colon (':') character. " "The colon is reserved for the 'model:adapter' syntax used in LoRA adapter specification. " - f"Invalid value: '{self.served_model_name}'" + f"Invalid value: '{cfg.served_model_name}'" ) # Check LoRA self.check_lora_server_args() # Check speculative decoding - if self.speculative_algorithm is not None: + if cfg.speculative_algorithm is not None: assert ( - not self.enable_mixed_chunk + not cfg.enable_mixed_chunk ), "enable_mixed_chunk is required for speculative decoding" # Check chunked prefill # Skip validation if chunked prefill is disabled (i.e., size <= 0). # Skip validation if disaggregation mode is decode. - if self.chunked_prefill_size > 0 and self.disaggregation_mode != "decode": + if cfg.chunked_prefill_size > 0 and cfg.disaggregation_mode != "decode": assert ( - self.chunked_prefill_size % self.page_size == 0 + cfg.chunked_prefill_size % cfg.page_size == 0 ), "chunked_prefill_size must be divisible by page_size" # Check pdmux - if self.enable_pdmux: + if cfg.enable_pdmux: assert ( - self.pp_size == 1 + cfg.pp_size == 1 ), "PD-Multiplexing is only supported with pipeline parallelism disabled (pp_size=1)." assert ( - self.chunked_prefill_size == -1 + cfg.chunked_prefill_size == -1 ), "PD-Multiplexing is not compatible with chunked prefill." assert ( - self.disaggregation_mode == "null" + cfg.disaggregation_mode == "null" ), "PD-Multiplexing is not compatible with disaggregation mode." assert ( - self.disable_overlap_schedule + cfg.disable_overlap_schedule ), "PD-Multiplexing is not compatible with overlap schedule." # NOTE: CUDA Green Context may encounter potential issues with CudaGraph on torch 2.7.x – 2.8.x, leading to performance degradation. @@ -9948,41 +10035,39 @@ class ServerArgs: " Please manually install torch 2.6.x." ) - assert self.tokenizer_worker_num > 0, "Tokenizer worker num must >= 1" - assert self.detokenizer_worker_num > 0, "Detokenizer worker num must >= 1" + assert cfg.tokenizer_worker_num > 0, "Tokenizer worker num must >= 1" + assert cfg.detokenizer_worker_num > 0, "Detokenizer worker num must >= 1" assert ( - self.mm_processor_worker_num >= 0 + cfg.mm_processor_worker_num >= 0 ), "Multimodal processor worker num must >= 0" - assert self.mm_io_worker_num >= 0, "Multimodal I/O worker num must >= 0" + assert cfg.mm_io_worker_num >= 0, "Multimodal I/O worker num must >= 0" + self.validate_buckets_rule("--prompt-tokens-buckets", cfg.prompt_tokens_buckets) self.validate_buckets_rule( - "--prompt-tokens-buckets", self.prompt_tokens_buckets - ) - self.validate_buckets_rule( - "--generation-tokens-buckets", self.generation_tokens_buckets + "--generation-tokens-buckets", cfg.generation_tokens_buckets ) # Check scheduling policy - if self.enable_priority_scheduling: - assert self.schedule_policy in [ + if cfg.enable_priority_scheduling: + assert cfg.schedule_policy in [ "fcfs", "lof", - ], f"To use priority scheduling, schedule_policy must be 'fcfs' or 'lof'. '{self.schedule_policy}' is not supported." - if self.default_priority_value is None: + ], f"To use priority scheduling, schedule_policy must be 'fcfs' or 'lof'. '{cfg.schedule_policy}' is not supported." + if cfg.default_priority_value is None: logger.warning( "--default-priority-value is not set while --enable-priority-scheduling is enabled. " "Requests without explicit priority will have priority=None, " "resulting in priority='None' string labels in Prometheus metrics." ) else: - if self.disable_priority_preemption: + if cfg.disable_priority_preemption: logger.warning( "--disable-priority-preemption has no effect without --enable-priority-scheduling" ) - if self.default_priority_value is not None: + if cfg.default_priority_value is not None: logger.warning( "--default-priority-value has no effect without --enable-priority-scheduling" ) - if self.retraction_policy == "priority" and not self.enable_priority_scheduling: + if cfg.retraction_policy == "priority" and not cfg.enable_priority_scheduling: raise ValueError( "--retraction-policy priority requires --enable-priority-scheduling" ) @@ -9998,23 +10083,23 @@ class ServerArgs: run_post_process_pass(self, _hisparse_validation) assert ( - self.schedule_conservativeness >= 0 + cfg.schedule_conservativeness >= 0 ), "schedule_conservativeness must be non-negative" - if self.model_impl == "mindspore": + if cfg.model_impl == "mindspore": assert is_npu(), "MindSpore model impl is only supported on Ascend npu." # Check metrics labels if ( - not self.tokenizer_metrics_custom_labels_header - and self.tokenizer_metrics_allowed_custom_labels + not cfg.tokenizer_metrics_custom_labels_header + and cfg.tokenizer_metrics_allowed_custom_labels ): raise ValueError( "Please set --tokenizer-metrics-custom-labels-header when setting --tokenizer-metrics-allowed-custom-labels." ) # Check metrics exporters - if self.export_metrics_to_file and self.export_metrics_to_file_dir is None: + if cfg.export_metrics_to_file and cfg.export_metrics_to_file_dir is None: raise ValueError( "--export-metrics-to-file-dir is required when --export-metrics-to-file is enabled" ) @@ -10023,65 +10108,67 @@ class ServerArgs: self._check_two_batch_overlap() # Check communications compression - if self.enable_quant_communications and self.tp_size == 1: + if cfg.enable_quant_communications and cfg.tp_size == 1: raise ValueError( "Communications quantization is only used with tp_size != 1" ) - if self.enable_quant_communications and self.device != "npu": + if cfg.enable_quant_communications and cfg.device != "npu": raise ValueError( "Communications quantization is only supported for NPU device" ) # grpc_port is None for HTTP-only launches, so the == comparison is # already False there; no explicit None check needed. - if not (self.smg_grpc_mode or self.grpc_mode) and self.grpc_port == self.port: + if not (cfg.smg_grpc_mode or cfg.grpc_mode) and cfg.grpc_port == cfg.port: raise ValueError( - f"--grpc-port ({self.grpc_port}) must differ from --port ({self.port})" + f"--grpc-port ({cfg.grpc_port}) must differ from --port ({cfg.port})" ) # TODO: Also validate grpc_port != metrics_http_port and grpc_port != nccl_port # to avoid opaque bind errors at runtime. Deferred because metrics_http_port # and nccl_port have dynamic defaults that may not be resolved yet here. - if self.gc_threshold: - if not (1 <= len(self.gc_threshold) <= 3): + if cfg.gc_threshold: + if not (1 <= len(cfg.gc_threshold) <= 3): raise ValueError( "When setting gc_threshold, it must contain 1 to 3 integers." ) - if self.kv_canary_sweep_interval > 0 and self.kv_canary == "none": + if cfg.kv_canary_sweep_interval > 0 and cfg.kv_canary == "none": raise ValueError( "--kv-canary-sweep-interval requires --kv-canary in {log, raise}" ) def check_lora_server_args(self): - assert self.max_loras_per_batch > 0, "max_loras_per_batch must be positive" + cfg = resolving_view(self) + + assert cfg.max_loras_per_batch > 0, "max_loras_per_batch must be positive" # Enable LoRA if any LoRA paths are provided for backward compatibility. - if self.lora_paths: - if self.enable_lora is None: + if cfg.lora_paths: + if cfg.enable_lora is None: self._late_resolution("check_lora_server_args", enable_lora=True) logger.warning( "--enable-lora is set to True because --lora-paths is provided." ) - elif self.enable_lora is False: + elif cfg.enable_lora is False: logger.warning( "--enable-lora is set to False, any provided lora_paths will be ignored." ) - if self.enable_lora: - if self.enable_lora_overlap_loading is None: + if cfg.enable_lora: + if cfg.enable_lora_overlap_loading is None: self._late_resolution( "check_lora_server_args", enable_lora_overlap_loading=False ) - if self.enable_lora_overlap_loading: + if cfg.enable_lora_overlap_loading: # TODO (glenliu21): use some sort of buffer with eviction instead of enforcing a limit - max_loaded_loras_limit = self.max_loras_per_batch * 2 + max_loaded_loras_limit = cfg.max_loras_per_batch * 2 assert ( - self.max_loaded_loras is not None - and self.max_loaded_loras <= max_loaded_loras_limit + cfg.max_loaded_loras is not None + and cfg.max_loaded_loras <= max_loaded_loras_limit ), ( "Enabling LoRA overlap loading requires pinning LoRA adapter weights in CPU memory, " f"so --max-loaded-loras must be less than or equal to double --max-loras-per-batch: {max_loaded_loras_limit}" @@ -10091,9 +10178,9 @@ class ServerArgs: self._check_lora_speculative_compatibility() # Parse lora_paths - if isinstance(self.lora_paths, list): + if isinstance(cfg.lora_paths, list): parsed_lora_paths = [] - for lora_path in self.lora_paths: + for lora_path in cfg.lora_paths: if isinstance(lora_path, str): if "=" in lora_path: name, path = lora_path.split("=", 1) @@ -10131,7 +10218,7 @@ class ServerArgs: self._late_resolution( "check_lora_server_args", lora_paths=parsed_lora_paths ) - elif isinstance(self.lora_paths, dict): + elif isinstance(cfg.lora_paths, dict): self._late_resolution( "check_lora_server_args", lora_paths=[ @@ -10141,56 +10228,56 @@ class ServerArgs: lora_path=v, pinned=False, ) - for k, v in self.lora_paths.items() + for k, v in cfg.lora_paths.items() ], ) - elif self.lora_paths is None: + elif cfg.lora_paths is None: self._late_resolution("check_lora_server_args", lora_paths=[]) else: raise ValueError( - f"Invalid type for --lora-paths: {type(self.lora_paths)}. " + f"Invalid type for --lora-paths: {type(cfg.lora_paths)}. " "Expected a list or a dictionary." ) # Normalize target modules to a set; keep {"all"} as a sentinel # that gets resolved model-awarely in lora_manager.init_lora_shapes(). - if self.lora_target_modules: + if cfg.lora_target_modules: self._late_resolution( "check_lora_server_args", - lora_target_modules=set(self.lora_target_modules), + lora_target_modules=set(cfg.lora_target_modules), ) - if "all" in self.lora_target_modules: + if "all" in cfg.lora_target_modules: assert ( - len(self.lora_target_modules) == 1 + len(cfg.lora_target_modules) == 1 ), "If 'all' is specified in --lora-target-modules, it should be the only module specified." # Ensure sufficient information is provided for LoRA initialization. - assert self.lora_paths or ( - self.max_lora_rank and self.lora_target_modules + assert cfg.lora_paths or ( + cfg.max_lora_rank and cfg.lora_target_modules ), "When no initial --lora-paths is provided, you need to specify both --max-lora-rank and --lora-target-modules for LoRA initialization." # Validate max_loaded_loras - if self.max_loaded_loras is not None: - assert self.max_loaded_loras >= self.max_loras_per_batch, ( + if cfg.max_loaded_loras is not None: + assert cfg.max_loaded_loras >= cfg.max_loras_per_batch, ( "max_loaded_loras should be greater than or equal to max_loras_per_batch. " - f"max_loaded_loras={self.max_loaded_loras}, max_loras_per_batch={self.max_loras_per_batch}" + f"max_loaded_loras={cfg.max_loaded_loras}, max_loras_per_batch={cfg.max_loras_per_batch}" ) - assert len(self.lora_paths) <= self.max_loaded_loras, ( + assert len(cfg.lora_paths) <= cfg.max_loaded_loras, ( "The number of LoRA paths should not exceed max_loaded_loras. " - f"max_loaded_loras={self.max_loaded_loras}, lora_paths={len(self.lora_paths)}" + f"max_loaded_loras={cfg.max_loaded_loras}, lora_paths={len(cfg.lora_paths)}" ) - if self.max_lora_chunk_size is not None: + if cfg.max_lora_chunk_size is not None: assert ( - 16 <= self.max_lora_chunk_size <= 128 - and (self.max_lora_chunk_size & (self.max_lora_chunk_size - 1)) == 0 + 16 <= cfg.max_lora_chunk_size <= 128 + and (cfg.max_lora_chunk_size & (cfg.max_lora_chunk_size - 1)) == 0 ), "--max-lora-chunk-size must be a power of 2 between 16 and 128." - if self.lora_use_virtual_experts: + if cfg.lora_use_virtual_experts: logger.info("Virtual expert computation enabled.") assert ( - self.lora_drain_wait_threshold >= 0.0 + cfg.lora_drain_wait_threshold >= 0.0 ), "--lora-drain-wait-threshold must be non-negative." def _check_lora_speculative_compatibility(self): @@ -10199,20 +10286,21 @@ class ServerArgs: Adapters apply to the target only; a shared draft runs unadapted. Matches resolved algorithm names (NEXTN has collapsed to EAGLE). """ - if self.speculative_algorithm in ["NGRAM", None]: + cfg = resolving_view(self) + if cfg.speculative_algorithm in ["NGRAM", None]: return - if self.speculative_algorithm not in _LORA_SPEC_ALGORITHMS: + if cfg.speculative_algorithm not in _LORA_SPEC_ALGORITHMS: promoted = ( " (NEXTN/EAGLE with a Gemma4 assistant draft is automatically " "promoted to FROZEN_KV_MTP, which does not support LoRA)" - if self.speculative_algorithm == "FROZEN_KV_MTP" + if cfg.speculative_algorithm == "FROZEN_KV_MTP" else "" ) raise ValueError( "LoRA is only compatible with NGRAM, EAGLE, NEXTN, EAGLE3, " "DFLASH, or DSPARK speculative decoding, not " - f"{self.speculative_algorithm}{promoted}." + f"{cfg.speculative_algorithm}{promoted}." ) ragged_mode = envs.SGLANG_RAGGED_VERIFY_MODE.get() @@ -10221,20 +10309,20 @@ class ServerArgs: # prefix so the message names the combination, not just the flag. unsupported = [ ( - self.speculative_algorithm == "DSPARK" and ragged_mode != "static", + cfg.speculative_algorithm == "DSPARK" and ragged_mode != "static", f"does not support SGLANG_RAGGED_VERIFY_MODE={ragged_mode!r}: " "the per-request verify lengths it schedules break the " "uniform-width LoRA segment layout", ), ( - self.speculative_adaptive, + cfg.speculative_adaptive, "does not support --speculative-adaptive: the draft is built " "from a static ServerArgs snapshot, and the runtime-state " "swap does not rebuild LoRA cuda-graph metadata", ), ( "experimental_sgl_trtllm" - in (self.moe_runner_backend, self.speculative_moe_runner_backend), + in (cfg.moe_runner_backend, cfg.speculative_moe_runner_backend), "does not support the experimental_sgl_trtllm MoE runner: its " "TopK reads the LoRA config per forward, which the draft " "resolves against the target's after its own publish ended", @@ -10301,12 +10389,13 @@ class ServerArgs: ), f"{arg_name} custom rule bucket values should be non-negative" def adjust_mem_fraction_for_vlm(self, model_config): + cfg = resolving_view(self) vision_config = getattr(model_config.hf_config, "vision_config", None) if vision_config is None: return # roughly reduce the mem_fraction_static base on params of Vit - original_server_arg_mem_fraction = self.mem_fraction_static + original_server_arg_mem_fraction = cfg.mem_fraction_static # a base mem_fraction_static factor for regular Vit base_mem_fraction_reduction_ratio = 0.95 @@ -10340,6 +10429,7 @@ class ServerArgs: ) def validate_transfer_engine(self): + cfg = resolving_view(self) try: mooncake_available = importlib.util.find_spec("mooncake.engine") is not None except (ModuleNotFoundError, ValueError): @@ -10349,7 +10439,7 @@ class ServerArgs: "Failed to import mooncake.engine. Does not support using TransferEngine as remote instance weight loader backend." ) return False - elif self.enable_memory_saver: + elif cfg.enable_memory_saver: logger.warning( "Memory saver is enabled, which is not compatible with TransferEngine. Does not support using TransferEngine as remote instance weight loader backend." ) @@ -10383,14 +10473,15 @@ class ServerArgs: def remote_instance_weight_loader_use_transfer_engine(self, load_format=None): """``load_format`` overrides the seed's: a draft runner loading under ``--speculative-draft-load-format`` needs its own transfer engine.""" - return remote_instance_transfer_engine_of(self, load_format) + return remote_instance_transfer_engine_of(resolving_view(self), load_format) @property def kv_event_block_size(self) -> int: """Width KV events are emitted at: under DCP the radix tree pages at ``page_size * dcp_size`` (``mem_cache/kv_cache_builder.py``). """ - return self.page_size * self.dcp_size + cfg = resolving_view(self) + return cfg.page_size * self.dcp_size def describe_kv_events_publisher(self) -> Optional[dict]: """Return a structured description of this server's KV-event @@ -10446,8 +10537,9 @@ class ServerArgs: # disaggregation / msgspec / zmq at module top level. from sglang.srt.disaggregation.kv_events import KVEventsConfig - raw = self.kv_events_config - page_size = self.page_size + resolved = resolving_view(self) + raw = resolved.kv_events_config + page_size = resolved.page_size if not raw or page_size is None or page_size <= 0: return None try: @@ -10483,13 +10575,18 @@ class ServerArgs: } def should_report_expert_balancedness(self) -> bool: - return self.expert_balancedness_report_mode != "off" + cfg = resolving_view(self) + return cfg.expert_balancedness_report_mode != "off" def should_log_expert_balancedness_to_server_log(self) -> bool: - return self.expert_balancedness_report_mode in ("server_log", "both") + cfg = resolving_view(self) + + return cfg.expert_balancedness_report_mode in ("server_log", "both") def should_export_expert_balancedness_to_prometheus(self) -> bool: - return self.expert_balancedness_report_mode in ("prometheus", "both") + cfg = resolving_view(self) + + return cfg.expert_balancedness_report_mode in ("prometheus", "both") def compute_world_size(server_args: ServerArgs) -> int: @@ -10657,6 +10754,7 @@ class PortArgs: dp_rank: Optional[int] = None, worker_ports: Optional[List[int]] = None, ) -> PortArgs: + cfg = resolving_view(server_args) if server_args.nccl_port is None: nccl_port = get_free_port() else: @@ -10689,7 +10787,7 @@ class PortArgs: rank=int(server_args.decoupled_spec_rank), ) - if not server_args.enable_dp_attention: + if not cfg.enable_dp_attention: # Normal case, use IPC within a single node return PortArgs( tokenizer_ipc_name=f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}", @@ -10720,7 +10818,7 @@ class PortArgs: # every init_new call agrees, decrementing below dist_init_port on # overflow. is_rust_server = envs.SGLANG_RUST_SERVER.get() - NUM_DERIVED_PORTS = 6 if not is_rust_server else 6 + server_args.dp_size + NUM_DERIVED_PORTS = 6 if not is_rust_server else 6 + cfg.dp_size if server_args.is_ep_scale_joiner: port_base = server_args.port + ZMQ_TCP_PORT_DELTA if port_base + NUM_DERIVED_PORTS > 65535: diff --git a/python/sglang/srt/speculative/adaptive_spec_params.py b/python/sglang/srt/speculative/adaptive_spec_params.py index d8b24dce0..c3c252f82 100644 --- a/python/sglang/srt/speculative/adaptive_spec_params.py +++ b/python/sglang/srt/speculative/adaptive_spec_params.py @@ -49,19 +49,19 @@ DEFAULT_ADAPTIVE_CONFIG: dict[str, dict] = { def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None: """Return why adaptive spec cannot run under the given server args, or None if supported.""" + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) from sglang.srt.arg_groups.overrides import resolved_view - if server_args.speculative_algorithm not in ("EAGLE", "EAGLE3"): + if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3"): return ( - f"speculative_algorithm={server_args.speculative_algorithm} " + f"speculative_algorithm={cfg.speculative_algorithm} " "(only EAGLE/EAGLE3 are supported)" ) - if ( - server_args.speculative_eagle_topk is not None - and server_args.speculative_eagle_topk != 1 - ): + if cfg.speculative_eagle_topk is not None and cfg.speculative_eagle_topk != 1: return ( - f"speculative_eagle_topk={server_args.speculative_eagle_topk} " + f"speculative_eagle_topk={cfg.speculative_eagle_topk} " "(only topk=1 is supported)" ) if resolved_view(server_args).enable_dp_attention: @@ -74,12 +74,12 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None: "enable_multi_layer_eagle=True is not supported " "(MultiLayerEagleWorkerV2 does not implement adaptive)" ) - if server_args.enable_two_batch_overlap: + if cfg.enable_two_batch_overlap: return ( "enable_two_batch_overlap=True is not supported " "(adaptive state swap would discard the TboAttnBackend wrapper)" ) - if server_args.enable_pdmux: + if cfg.enable_pdmux: return ( "enable_pdmux=True is not supported " "(adaptive state swap does not update decode_attn_backend_group)" diff --git a/python/sglang/srt/speculative/spec_info.py b/python/sglang/srt/speculative/spec_info.py index 03967668e..97ca92d31 100644 --- a/python/sglang/srt/speculative/spec_info.py +++ b/python/sglang/srt/speculative/spec_info.py @@ -254,6 +254,9 @@ class SpeculativeAlgorithm(Enum): def create_worker( self, server_args: ServerArgs ) -> Optional[Union[Type[BaseSpecWorker], Type[TpModelWorker], Type[NGRAMWorker]]]: + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) assert ( not self.is_none() ), "Cannot create worker for NONE speculative algorithm." @@ -283,7 +286,7 @@ class SpeculativeAlgorithm(Enum): # EAGLE / EAGLE3 / STANDALONE / MULTI_LAYER always use the V2 worker, # even with overlap disabled (scheduler drives it synchronously). - if self.is_eagle() and server_args.enable_multi_layer_eagle: + if self.is_eagle() and cfg.enable_multi_layer_eagle: from sglang.srt.speculative.multi_layer_eagle_worker_v2 import ( MultiLayerEagleWorkerV2, ) diff --git a/python/sglang/srt/speculative/spec_registry.py b/python/sglang/srt/speculative/spec_registry.py index 1e3b84853..933d4e05d 100644 --- a/python/sglang/srt/speculative/spec_registry.py +++ b/python/sglang/srt/speculative/spec_registry.py @@ -108,7 +108,10 @@ class CustomSpecAlgo: pass def create_worker(self, server_args: ServerArgs) -> Type: - if not server_args.disable_overlap_schedule and not self.supports_overlap: + from sglang.srt.arg_groups.overrides import resolving_view + + cfg = resolving_view(server_args) + if not cfg.disable_overlap_schedule and not self.supports_overlap: raise ValueError( f"Speculative algorithm {self.name} does not support overlap scheduling." ) diff --git a/test/registered/spec/dspark/test_dspark_draft_path_default.py b/test/registered/spec/dspark/test_dspark_draft_path_default.py index 53fbc1051..12b4c3621 100644 --- a/test/registered/spec/dspark/test_dspark_draft_path_default.py +++ b/test/registered/spec/dspark/test_dspark_draft_path_default.py @@ -1,6 +1,7 @@ import unittest from types import SimpleNamespace +from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.arg_groups.speculative_hook import ( _handle_dspark, _target_checkpoint_bundles_dspark_draft, @@ -63,8 +64,13 @@ class TestDsparkDraftPathDefaulting(CustomTestCase): model_path=_BUNDLED_MODEL_PATH, hf_config=_bundled_hf_config() ) _handle_dspark(server_args) - self.assertEqual(server_args.speculative_draft_model_path, _BUNDLED_MODEL_PATH) - self.assertEqual(server_args.speculative_num_draft_tokens, 6) + self.assertEqual( + resolution_result(server_args, "speculative_draft_model_path"), + _BUNDLED_MODEL_PATH, + ) + self.assertEqual( + resolution_result(server_args, "speculative_num_draft_tokens"), 6 + ) def test_plain_target_without_draft_path_raises(self): server_args = _make_dspark_server_args( @@ -80,7 +86,7 @@ class TestDsparkDraftPathDefaulting(CustomTestCase): server_args.speculative_draft_model_path = "deepseek-ai/some-other-dspark-draft" _handle_dspark(server_args) self.assertEqual( - server_args.speculative_draft_model_path, + resolution_result(server_args, "speculative_draft_model_path"), "deepseek-ai/some-other-dspark-draft", ) diff --git a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py index b81df8b7b..b16d22a0b 100644 --- a/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py +++ b/test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py @@ -4,6 +4,7 @@ import unittest from types import SimpleNamespace from unittest.mock import patch +from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.configs.embedding_model_spec import resolve_embedding_model_spec from sglang.srt.configs.model_config import ( is_multimodal_piecewise_cuda_graph_supported, @@ -90,7 +91,10 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): ): args._apply_cuda_graph_compatibility() - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.TC_PIECEWISE) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.TC_PIECEWISE, + ) disable_if_incompatible.assert_called_once() def test_trtllm_mla_stays_on_breakable(self): @@ -118,7 +122,10 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): ): args._apply_cuda_graph_compatibility() - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.BREAKABLE, + ) def test_explicit_tc_piecewise_overrides_trtllm_mla_default(self): args = ServerArgs(model_path="dummy") @@ -134,7 +141,10 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): ): args._apply_cuda_graph_compatibility() - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.TC_PIECEWISE) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.TC_PIECEWISE, + ) def test_multimodal_inputs_keep_tc_piecewise_prefill_enabled(self): runner = self._make_prefill_runner(Backend.TC_PIECEWISE) @@ -178,10 +188,16 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): ): args._handle_model_capability_adjustments() - self.assertTrue(args.disable_radix_cache) - self.assertEqual(args.chunked_prefill_size, -1) - self.assertEqual(args.cuda_graph_config.decode.backend, Backend.DISABLED) - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) + self.assertTrue(resolution_result(args, "disable_radix_cache")) + self.assertEqual(resolution_result(args, "chunked_prefill_size"), -1) + self.assertEqual( + resolution_result(args, "cuda_graph_config").decode.backend, + Backend.DISABLED, + ) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.BREAKABLE, + ) def test_encoder_embedding_model_enables_embedding_mode_without_flag(self): args = ServerArgs(model_path="dummy") @@ -199,7 +215,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase): with patch.object(args, "get_model_config", return_value=args.model_config): args._handle_model_capability_adjustments() - self.assertTrue(args.is_embedding) + self.assertTrue(resolution_result(args, "is_embedding")) if __name__ == "__main__": diff --git a/test/registered/unit/disaggregation/test_kimi_k3_encoder_mode.py b/test/registered/unit/disaggregation/test_kimi_k3_encoder_mode.py index 3e9c3dc79..dd1a19ebb 100644 --- a/test/registered/unit/disaggregation/test_kimi_k3_encoder_mode.py +++ b/test/registered/unit/disaggregation/test_kimi_k3_encoder_mode.py @@ -15,6 +15,7 @@ import zmq.asyncio from fastapi import HTTPException from PIL import Image +from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.disaggregation.encoder.preprocessor import ( EncoderPreprocessor, EncoderPreprocessResult, @@ -195,7 +196,7 @@ def test_epd_rejection_reads_the_resolved_transfer_backend(): finally: shutil.rmtree(config_dir, ignore_errors=True) - assert resolved.encoder_transfer_backend == "zmq_to_tokenizer" + assert resolution_result(resolved, "encoder_transfer_backend") == "zmq_to_tokenizer" # Publish that record: the guard reads the resolved value out of the bags, # so a raw record does not silently disable the rejection. publish(resolved, role="tokenizer") diff --git a/test/registered/unit/managers/test_mm_process_config.py b/test/registered/unit/managers/test_mm_process_config.py index 167837e5e..e22ac834c 100644 --- a/test/registered/unit/managers/test_mm_process_config.py +++ b/test/registered/unit/managers/test_mm_process_config.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patch import torch +from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.environ import envs from sglang.srt.runtime_context import get_context from sglang.srt.server_args import ServerArgs @@ -25,17 +26,20 @@ class TestMmProcessConfigValidation(CustomTestCase): def test_valid_config_accepted(self): args = self._validate_config({"image": {"max_pixels": 5000000}}) - self.assertEqual(args.mm_process_config, {"image": {"max_pixels": 5000000}}) + self.assertEqual( + resolution_result(args, "mm_process_config"), + {"image": {"max_pixels": 5000000}}, + ) def test_empty_config_accepted(self): args = self._validate_config({}) - self.assertEqual(args.mm_process_config, {}) + self.assertEqual(resolution_result(args, "mm_process_config"), {}) def test_none_config_defaults_to_empty_dict(self): args = self._validate_config(None) # None is kept as-is for dummy models (default happens after early return) # but for real models it would be set to {} - self.assertIsNone(args.mm_process_config) + self.assertIsNone(resolution_result(args, "mm_process_config")) def test_top_level_non_dict_rejected(self): with self.assertRaises(TypeError) as ctx: @@ -64,7 +68,7 @@ class TestMmProcessConfigValidation(CustomTestCase): "audio": {"sample_rate": 16000}, } args = self._validate_config(config) - self.assertEqual(args.mm_process_config, config) + self.assertEqual(resolution_result(args, "mm_process_config"), config) class TestBaseProcessorConfigExtraction(CustomTestCase): diff --git a/test/registered/unit/server_args/test_model_config_reads_resolved_input.py b/test/registered/unit/server_args/test_model_config_reads_resolved_input.py index e21d6a462..8dabf06aa 100644 --- a/test/registered/unit/server_args/test_model_config_reads_resolved_input.py +++ b/test/registered/unit/server_args/test_model_config_reads_resolved_input.py @@ -125,6 +125,13 @@ def _registry_collection_is_after_the_build(): def _server_args_names(tree, path): + """Every local that names the record, including the read views over it. + + A resolution-time reader reads through `resolving_view(server_args)` (the + declaration stash over the fields): declaration-only resolvers write no + field, so a field read there answers with the raw input. `cfg.dtype` after `cfg = resolving_view(sa)` is + the same read this scan is looking for, so the local it binds counts. + """ names = {"self"} if path.name == "server_args.py" else {"server_args"} for node in ast.walk(tree): if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): @@ -142,6 +149,32 @@ def _server_args_names(tree, path): continue if text == "ServerArgs": names.add(arg.arg) + # `cfg = resolving_view(server_args)` / `resolved_view(server_args)` + for _ in range(2): # a view over a view-holding local is still one + for node in ast.walk(tree): + if not isinstance(node, ast.Assign): + continue + value = node.value + bare = ( + isinstance(value, ast.Call) + and isinstance(value.func, ast.Name) + and value.func.id in ("resolving_view", "resolved_view") + and value.args + and isinstance(value.args[0], ast.Name) + and value.args[0].id in names + ) + # `resolved = self._resolved()` is the same view, spelled as the + # record's own member. + member = ( + isinstance(value, ast.Call) + and isinstance(value.func, ast.Attribute) + and value.func.attr == "_resolved" + and isinstance(value.func.value, ast.Name) + and value.func.value.id in names + ) + if not (bare or member): + continue + names |= {t.id for t in node.targets if isinstance(t, ast.Name)} return names @@ -191,11 +224,12 @@ def _late_resolution_fields(): for name in ( "server_args.py", "arg_groups/overrides.py", - "utils/template_detection.py", + "parser/template_detection.py", ): path = _SRT / name - if not path.exists(): - continue + # A named file that moved away has to be loud; skipping it silently + # leaves the scan believing it read a module it never opened. + assert path.exists(), f"{name} is not where this scan looks for it" tree = _parsed(path) for node in ast.walk(tree): if not isinstance(node, ast.Call): diff --git a/test/registered/unit/server_args/test_resolution_declarations.py b/test/registered/unit/server_args/test_resolution_declarations.py index 331e17779..ee37b06cd 100644 --- a/test/registered/unit/server_args/test_resolution_declarations.py +++ b/test/registered/unit/server_args/test_resolution_declarations.py @@ -764,7 +764,7 @@ class TestResolutionDeclarations(CustomTestCase): # Snapshot before publishing: the bag serves the very object the record # holds, so comparing them after the fact compares an object with # itself and passes however the projection behaves. - expected = copy.deepcopy(server_args.cuda_graph_config) + expected = copy.deepcopy(resolution_result(server_args, "cuda_graph_config")) publish(server_args, role="scheduler") published = get_exec().graph.cuda_graph_config resolved = expected diff --git a/test/registered/unit/server_args/test_resolution_is_reproducible.py b/test/registered/unit/server_args/test_resolution_is_reproducible.py index 0badd493b..dfeccd4b2 100644 --- a/test/registered/unit/server_args/test_resolution_is_reproducible.py +++ b/test/registered/unit/server_args/test_resolution_is_reproducible.py @@ -36,6 +36,7 @@ import unittest.mock import torch import sglang +from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.environ import EnvField, envs from sglang.srt.server_args import ServerArgs from sglang.srt.utils import is_cuda @@ -270,7 +271,10 @@ class TestResolutionIsReproducible(_RestoresProcessState, CustomTestCase): for field in dataclasses.fields(server_args): if field.name in _NOT_COMPARABLE: continue - value = getattr(server_args, field.name) + # The resolution result, not the field: a declaration-only resolver + # never writes the field, so comparing fields would miss exactly + # the decisions a leak would shift. + value = resolution_result(server_args, field.name) # Nested dataclasses (cuda_graph_config) compare structurally, and # everything else is deep-copied: a snapshot that stored the live # list/dict would follow an in-place mutation, which is exactly the @@ -382,10 +386,13 @@ class TestResolutionIsReproducible(_RestoresProcessState, CustomTestCase): # differs from the cpu that `default_before` resolved to. expected = ( "cuda_ipc" - if intermediate.mm_feature_transport == "cuda_ipc" + if resolution_result(intermediate, "mm_feature_transport") + == "cuda_ipc" else "cpu" ) - self.assertEqual(after.mm_feature_transport, expected) + self.assertEqual( + resolution_result(after, "mm_feature_transport"), expected + ) def test_resolving_a_sibling_leaves_the_first_alone(self): for label, config, kwargs in _SHAPES: @@ -847,9 +854,9 @@ class TestACopyStaysResolved(_RestoresProcessState, CustomTestCase): self.assertEqual(get_parallel().config.dist_init_addr, "1.2.3.4:5000") self.assertEqual( get_schedule().chunked_prefill_size, - parent.chunked_prefill_size, - "publishing the copy re-ran resolution; the bag disagrees with the " - "record the parent resolved", + resolution_result(parent, "chunked_prefill_size"), + "publishing the copy re-ran resolution; the bag disagrees with what " + "the parent's resolution decided", ) def test_no_bare_replace_of_a_record_outside_the_helper(self): diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index 5c3e085b5..c94a30d2a 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -10,6 +10,7 @@ from unittest.mock import MagicMock, patch import sglang.srt.server_args as server_args_module from sglang.srt.arg_groups import pd_disaggregation_hook +from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding from sglang.srt.entrypoints.sidecar import ( SGLANG_GRPC_ENDPOINT_ENV, @@ -61,7 +62,7 @@ class TestPrepareServerArgs(CustomTestCase): args.resolve_once() - self.assertTrue(args.enable_w4a4_mxfp4_megamoe) + self.assertTrue(resolution_result(args, "enable_w4a4_mxfp4_megamoe")) self.assertEqual(os.environ["DG_USE_FP4_ACTS"], "1") self.assertEqual(os.environ["DG_USE_MXF4_KIND"], "1") @@ -76,14 +77,14 @@ class TestPrepareServerArgs(CustomTestCase): # nothing to be untouched by. args.resolve_once() - self.assertFalse(args.enable_w4a4_mxfp4_megamoe) + self.assertFalse(resolution_result(args, "enable_w4a4_mxfp4_megamoe")) self.assertEqual(os.environ["DG_USE_FP4_ACTS"], "0") self.assertEqual(os.environ["DG_USE_MXF4_KIND"], "0") def test_prefill_decode_interval(self): args = ServerArgs(model_path="dummy", prefill_decode_interval=16) args.resolve_once() - self.assertEqual(args.prefill_decode_interval, 16) + self.assertEqual(resolution_result(args, "prefill_decode_interval"), 16) with self.assertRaisesRegex( ValueError, "--prefill-decode-interval must be non-negative" @@ -114,22 +115,24 @@ class TestPrepareServerArgs(CustomTestCase): return server_args disabled = _resolved(model_path="dummy") - self.assertFalse(disabled.enable_return_hidden_states) - self.assertIsNone(disabled.return_hidden_states_mode) + self.assertFalse(resolution_result(disabled, "enable_return_hidden_states")) + self.assertIsNone(resolution_result(disabled, "return_hidden_states_mode")) last = _resolved( model_path="dummy", return_hidden_states_mode="last", ) - self.assertTrue(last.enable_return_hidden_states) - self.assertEqual(last.return_hidden_states_mode, "last") + self.assertTrue(resolution_result(last, "enable_return_hidden_states")) + self.assertEqual(resolution_result(last, "return_hidden_states_mode"), "last") legacy_full = _resolved( model_path="dummy", enable_return_hidden_states=True, ) - self.assertTrue(legacy_full.enable_return_hidden_states) - self.assertEqual(legacy_full.return_hidden_states_mode, "full") + self.assertTrue(resolution_result(legacy_full, "enable_return_hidden_states")) + self.assertEqual( + resolution_result(legacy_full, "return_hidden_states_mode"), "full" + ) parsed_last = prepare_server_args( [ @@ -140,8 +143,10 @@ class TestPrepareServerArgs(CustomTestCase): ] ) parsed_last.resolve_once() - self.assertTrue(parsed_last.enable_return_hidden_states) - self.assertEqual(parsed_last.return_hidden_states_mode, "last") + self.assertTrue(resolution_result(parsed_last, "enable_return_hidden_states")) + self.assertEqual( + resolution_result(parsed_last, "return_hidden_states_mode"), "last" + ) # The rejection is resolution's, not the constructor's. with self.assertRaisesRegex( @@ -156,13 +161,24 @@ class TestPrepareServerArgs(CustomTestCase): def test_draft_quantization_explicitness_survives_asdict_round_trip(self): inherited = ServerArgs(model_path="dummy", quantization="modelopt_fp4") inherited._handle_missing_default_values() - self.assertEqual(inherited.speculative_draft_model_quantization, "modelopt_fp4") - self.assertFalse(inherited._speculative_draft_quantization_explicitly_set) + self.assertEqual( + resolution_result(inherited, "speculative_draft_model_quantization"), + "modelopt_fp4", + ) + self.assertFalse( + resolution_result( + inherited, "_speculative_draft_quantization_explicitly_set" + ) + ) reconstructed = ServerArgs(**dataclasses.asdict(inherited)) reconstructed._handle_missing_default_values() - self.assertFalse(reconstructed._speculative_draft_quantization_explicitly_set) + self.assertFalse( + resolution_result( + reconstructed, "_speculative_draft_quantization_explicitly_set" + ) + ) def test_config_nested_dict_args_are_json(self): with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: @@ -219,8 +235,10 @@ class TestImageProcessorBackend(CustomTestCase): server_args._handle_deprecated_args() - self.assertEqual(server_args.image_processor_backend, "pil") - self.assertFalse(server_args.disable_fast_image_processor) + self.assertEqual( + resolution_result(server_args, "image_processor_backend"), "pil" + ) + self.assertFalse(resolution_result(server_args, "disable_fast_image_processor")) def test_legacy_flag_maps_to_pil_with_one_warning(self): server_args = ServerArgs(model_path="dummy", disable_fast_image_processor=True) @@ -228,8 +246,10 @@ class TestImageProcessorBackend(CustomTestCase): with self.assertLogs(server_args_module.logger, level="WARNING") as logs: server_args._handle_deprecated_args() - self.assertEqual(server_args.image_processor_backend, "pil") - self.assertTrue(server_args.disable_fast_image_processor) + self.assertEqual( + resolution_result(server_args, "image_processor_backend"), "pil" + ) + self.assertTrue(resolution_result(server_args, "disable_fast_image_processor")) self.assertEqual( sum( "--disable-fast-image-processor is deprecated" in x for x in logs.output @@ -266,7 +286,9 @@ class TestMultimodalFeatureTransport(CustomTestCase): with self.assertLogs(server_args_module.logger, level="INFO") as logs: server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cuda_ipc") + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cuda_ipc" + ) self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) output = "\n".join(logs.output) @@ -281,8 +303,12 @@ class TestMultimodalFeatureTransport(CustomTestCase): with self.assertLogs(server_args_module.logger, level="WARNING") as logs: server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cuda_ipc") - self.assertFalse(server_args.keep_mm_feature_on_device) + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cuda_ipc" + ) + self.assertFalse( + resolution_result(server_args, "keep_mm_feature_on_device") + ) self.assertTrue(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertIn("deprecated", logs.output[0]) @@ -305,7 +331,9 @@ class TestMultimodalFeatureTransport(CustomTestCase): with self.assertLogs(server_args_module.logger, level="WARNING") as logs: server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cpu") + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cpu" + ) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) self.assertIn("overrides", logs.output[0]) @@ -316,7 +344,9 @@ class TestMultimodalFeatureTransport(CustomTestCase): with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "0"}): server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cpu") + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cpu" + ) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) @patch("sglang.srt.server_args.is_cuda", return_value=True) @@ -329,7 +359,9 @@ class TestMultimodalFeatureTransport(CustomTestCase): with self.assertNoLogs(server_args_module.logger, level="INFO"): server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cpu") + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cpu" + ) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) @patch("sglang.srt.server_args.is_cuda", return_value=True) @@ -342,7 +374,9 @@ class TestMultimodalFeatureTransport(CustomTestCase): with self.assertNoLogs(server_args_module.logger, level="INFO"): server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cpu") + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cpu" + ) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) @patch("sglang.srt.server_args.os.path.exists", return_value=True) @@ -367,7 +401,9 @@ class TestMultimodalFeatureTransport(CustomTestCase): with self.assertLogs(server_args_module.logger, level="INFO") as logs: server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cuda_vmm") + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cuda_vmm" + ) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) output = "\n".join(logs.output) @@ -394,7 +430,7 @@ class TestMultimodalFeatureTransport(CustomTestCase): with self.assertLogs(server_args_module.logger, level="INFO") as logs: server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cpu") + self.assertEqual(resolution_result(server_args, "mm_feature_transport"), "cpu") self.assertIn("has not opted into CUDA VMM", "\n".join(logs.output)) @patch("sglang.srt.server_args.os.path.exists", return_value=False) @@ -411,7 +447,9 @@ class TestMultimodalFeatureTransport(CustomTestCase): with self.assertLogs(server_args_module.logger, level="INFO") as logs: server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cpu") + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cpu" + ) self.assertIn("no IMEX channel", "\n".join(logs.output)) @@ -427,7 +465,9 @@ class TestMultimodalFeatureTransport(CustomTestCase): envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear() server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cpu") + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cpu" + ) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) @patch("sglang.srt.server_args.is_cuda", return_value=True) @@ -439,7 +479,9 @@ class TestMultimodalFeatureTransport(CustomTestCase): envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear() server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cpu") + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cpu" + ) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) @patch("sglang.srt.server_args.is_cuda", return_value=False) @@ -474,7 +516,9 @@ class TestMultimodalFeatureTransport(CustomTestCase): with self.assertLogs(server_args_module.logger, level="INFO") as logs: server_args._handle_multimodal_feature_transport() - self.assertEqual(server_args.mm_feature_transport, "cuda_vmm") + self.assertEqual( + resolution_result(server_args, "mm_feature_transport"), "cuda_vmm" + ) self.assertFalse(envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()) output = "\n".join(logs.output) @@ -555,15 +599,22 @@ class TestLoadBalanceMethod(unittest.TestCase): def test_non_pd_defaults_to_round_robin(self): server_args = self._load_balance_args(disaggregation_mode="null") - self.assertEqual(server_args.load_balance_method, "round_robin") + self.assertEqual( + resolution_result(server_args, "load_balance_method"), "round_robin" + ) def test_pd_prefill_defaults_to_follow_bootstrap_room(self): server_args = self._load_balance_args(disaggregation_mode="prefill") - self.assertEqual(server_args.load_balance_method, "follow_bootstrap_room") + self.assertEqual( + resolution_result(server_args, "load_balance_method"), + "follow_bootstrap_room", + ) def test_pd_decode_defaults_to_round_robin(self): server_args = self._load_balance_args(disaggregation_mode="decode") - self.assertEqual(server_args.load_balance_method, "round_robin") + self.assertEqual( + resolution_result(server_args, "load_balance_method"), "round_robin" + ) def test_pd_prefill_dcp_warns_about_performance(self): server_args = ServerArgs( @@ -581,7 +632,7 @@ class TestLoadBalanceMethod(unittest.TestCase): disaggregation_transfer_backend="mooncake", dcp_size=4, ) - self.assertTrue(server_args.disable_radix_cache) + self.assertTrue(resolution_result(server_args, "disable_radix_cache")) def test_pd_decode_dcp_rejects_unsupported_transfer_backend(self): server_args = ServerArgs( @@ -601,7 +652,7 @@ class TestLoadBalanceMethod(unittest.TestCase): disaggregation_transfer_backend="fake", dcp_size=4, ) - self.assertTrue(server_args.disable_radix_cache) + self.assertTrue(resolution_result(server_args, "disable_radix_cache")) def test_pd_decode_dcp_rejects_radix_cache(self): server_args = ServerArgs( @@ -665,8 +716,11 @@ class TestLoadBalanceMethod(unittest.TestCase): disaggregation_transfer_backend="mooncake_tcp", ) - self.assertFalse(server_args.disable_radix_cache) - self.assertEqual(server_args.disaggregation_transfer_backend, "mooncake") + self.assertFalse(resolution_result(server_args, "disable_radix_cache")) + self.assertEqual( + resolution_result(server_args, "disaggregation_transfer_backend"), + "mooncake", + ) class TestSkipTokenizerInit(unittest.TestCase): @@ -681,8 +735,8 @@ class TestSkipTokenizerInit(unittest.TestCase): server_args._handle_tokenizer_batching() # Tokenizer fanout preserved; detokenizer coerced to 1 (no decode work). - self.assertEqual(server_args.tokenizer_worker_num, 4) - self.assertEqual(server_args.detokenizer_worker_num, 1) + self.assertEqual(resolution_result(server_args, "tokenizer_worker_num"), 4) + self.assertEqual(resolution_result(server_args, "detokenizer_worker_num"), 1) class TestHiSparseDsaBackendPolicy(unittest.TestCase): @@ -870,7 +924,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase): from sglang.srt.arg_groups.overrides import resolved_view - self.assertEqual(args.page_size, 1) # dual-apply retired: pristine + self.assertEqual(args.page_size, 1) # the field stays pristine self.assertEqual(resolved_view(args).page_size, 128) @patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True) @@ -883,7 +937,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase): from sglang.srt.arg_groups.overrides import resolved_view - self.assertEqual(args.page_size, 1) # dual-apply retired: pristine + self.assertEqual(args.page_size, 1) # the field stays pristine self.assertEqual(resolved_view(args).page_size, 128) @@ -918,12 +972,12 @@ class TestContextParallelServerArgs(CustomTestCase): def test_canonical_prefill_cp_requires_strategy(self): args = self.parser.parse_args(["--model", "dummy", "--enable-prefill-cp"]) - self.assertTrue(args.enable_prefill_cp) - self.assertIsNone(args.cp_strategy) + self.assertTrue(resolution_result(args, "enable_prefill_cp")) + self.assertIsNone(resolution_result(args, "cp_strategy")) server_args = self._new_cp_args( - enable_prefill_cp=args.enable_prefill_cp, - cp_strategy=args.cp_strategy, + enable_prefill_cp=resolution_result(args, "enable_prefill_cp"), + cp_strategy=resolution_result(args, "cp_strategy"), ) with self.assertRaisesRegex(ValueError, "--cp-strategy"): server_args._handle_context_parallelism() @@ -940,16 +994,18 @@ class TestContextParallelServerArgs(CustomTestCase): ) server_args = self._new_cp_args( enable_dsa_prefill_context_parallel=( - args.enable_dsa_prefill_context_parallel + resolution_result(args, "enable_dsa_prefill_context_parallel") ), - dsa_prefill_cp_mode=args.dsa_prefill_cp_mode, + dsa_prefill_cp_mode=resolution_result(args, "dsa_prefill_cp_mode"), ) server_args._handle_legacy_cp_arguments() - self.assertTrue(server_args.enable_prefill_cp) - self.assertEqual(server_args.cp_strategy, "interleave") - self.assertEqual(server_args.dsa_prefill_cp_mode, "round-robin-split") + self.assertTrue(resolution_result(server_args, "enable_prefill_cp")) + self.assertEqual(resolution_result(server_args, "cp_strategy"), "interleave") + self.assertEqual( + resolution_result(server_args, "dsa_prefill_cp_mode"), "round-robin-split" + ) def test_canonical_interleave_cp_mirrors_to_dsa_runtime_aliases(self): server_args = self._new_cp_args( @@ -961,10 +1017,18 @@ class TestContextParallelServerArgs(CustomTestCase): server_args._handle_legacy_cp_arguments() server_args._handle_context_parallelism() - self.assertTrue(server_args.enable_dsa_prefill_context_parallel) - self.assertFalse(server_args.enable_prefill_context_parallel) - self.assertEqual(server_args.dsa_prefill_cp_mode, "round-robin-split") - self.assertEqual(server_args.prefill_cp_mode, "round-robin-split") + self.assertTrue( + resolution_result(server_args, "enable_dsa_prefill_context_parallel") + ) + self.assertFalse( + resolution_result(server_args, "enable_prefill_context_parallel") + ) + self.assertEqual( + resolution_result(server_args, "dsa_prefill_cp_mode"), "round-robin-split" + ) + self.assertEqual( + resolution_result(server_args, "prefill_cp_mode"), "round-robin-split" + ) def test_context_parallel_handler_initializes_cp_strategy(self): server_args = self._new_cp_args( @@ -1049,15 +1113,25 @@ class TestContextParallelServerArgs(CustomTestCase): server_args._handle_legacy_cp_arguments() server_args._handle_context_parallelism() - self.assertTrue(server_args.enable_prefill_cp) - self.assertEqual(server_args.cp_strategy, strategy) - self.assertEqual(server_args.dsa_prefill_cp_mode, mode) - self.assertEqual(server_args.prefill_cp_mode, mode) + self.assertTrue(resolution_result(server_args, "enable_prefill_cp")) self.assertEqual( - server_args.enable_dsa_prefill_context_parallel, expect_dsa + resolution_result(server_args, "cp_strategy"), strategy ) self.assertEqual( - server_args.enable_prefill_context_parallel, expect_generic + resolution_result(server_args, "dsa_prefill_cp_mode"), mode + ) + self.assertEqual( + resolution_result(server_args, "prefill_cp_mode"), mode + ) + self.assertEqual( + resolution_result( + server_args, "enable_dsa_prefill_context_parallel" + ), + expect_dsa, + ) + self.assertEqual( + resolution_result(server_args, "enable_prefill_context_parallel"), + expect_generic, ) @@ -1308,7 +1382,7 @@ class TestSSLArgs(unittest.TestCase): ssl_certfile="cert.pem", enable_ssl_refresh=True, ) - self.assertTrue(server_args.enable_ssl_refresh) + self.assertTrue(resolution_result(server_args, "enable_ssl_refresh")) class TestHiCacheArgs(unittest.TestCase): @@ -1328,10 +1402,17 @@ class TestHiCacheArgs(unittest.TestCase): expected_mem_layout: str, expected_decode_backend: str | None = None, ): - self.assertEqual(args.hicache_io_backend, expected_io_backend) - self.assertEqual(args.hicache_mem_layout, expected_mem_layout) + self.assertEqual( + resolution_result(args, "hicache_io_backend"), expected_io_backend + ) + self.assertEqual( + resolution_result(args, "hicache_mem_layout"), expected_mem_layout + ) if expected_decode_backend is not None: - self.assertEqual(args.decode_attention_backend, expected_decode_backend) + self.assertEqual( + resolution_result(args, "decode_attention_backend"), + expected_decode_backend, + ) def test_hicache_io_backend_and_mem_layout_compatibility(self): cases = [ @@ -1409,9 +1490,9 @@ class TestHiCacheArgs(unittest.TestCase): ) args._handle_hicache() - self.assertEqual(args.hicache_io_backend, "kernel") - self.assertEqual(args.hicache_mem_layout, "page_first") - self.assertIsNone(args.decode_attention_backend) + self.assertEqual(resolution_result(args, "hicache_io_backend"), "kernel") + self.assertEqual(resolution_result(args, "hicache_mem_layout"), "page_first") + self.assertIsNone(resolution_result(args, "decode_attention_backend")) def test_decode_offload_rejects_host_pool_retraction(self): args = self._make_args( @@ -1494,11 +1575,19 @@ class TestDecoupledSpecArgs(CustomTestCase): "/tmp/tr", ] ) - self.assertEqual(server_args.decoupled_spec_role, "verifier") - self.assertEqual(server_args.decoupled_spec_bind_endpoint, "ipc:///tmp/v") - self.assertEqual(server_args.decoupled_spec_connect_endpoints, ["ipc:///tmp/d"]) - self.assertEqual(server_args.decoupled_spec_rank, 0) - self.assertEqual(server_args.spec_trace_dir, "/tmp/tr") + self.assertEqual( + resolution_result(server_args, "decoupled_spec_role"), "verifier" + ) + self.assertEqual( + resolution_result(server_args, "decoupled_spec_bind_endpoint"), + "ipc:///tmp/v", + ) + self.assertEqual( + resolution_result(server_args, "decoupled_spec_connect_endpoints"), + ["ipc:///tmp/d"], + ) + self.assertEqual(resolution_result(server_args, "decoupled_spec_rank"), 0) + self.assertEqual(resolution_result(server_args, "spec_trace_dir"), "/tmp/tr") def test_decoupled_spec_role_rejects_invalid_choice(self): with self.assertRaises(SystemExit): @@ -1533,10 +1622,10 @@ class TestAdaptiveSpecArgs(CustomTestCase): handle_speculative_decoding(args) - self.assertTrue(args.speculative_adaptive) - self.assertEqual(args.speculative_eagle_topk, 1) - self.assertEqual(args.speculative_num_steps, 3) - self.assertEqual(args.speculative_num_draft_tokens, 4) + self.assertTrue(resolution_result(args, "speculative_adaptive")) + self.assertEqual(resolution_result(args, "speculative_eagle_topk"), 1) + self.assertEqual(resolution_result(args, "speculative_num_steps"), 3) + self.assertEqual(resolution_result(args, "speculative_num_draft_tokens"), 4) class TestWaterfillArgs(CustomTestCase): @@ -1552,10 +1641,9 @@ class TestWaterfillArgs(CustomTestCase): from sglang.srt.arg_groups.overrides import resolved_view - # dual-apply retired: the fields stay pristine, the declarations win self.assertTrue(server_args.disable_shared_experts_fusion) self.assertFalse(resolved_view(server_args).disable_shared_experts_fusion) - self.assertTrue(server_args.enforce_shared_experts_fusion) + self.assertTrue(resolution_result(server_args, "enforce_shared_experts_fusion")) def test_waterfill_overrides_moe_a2a_backend_to_deepep(self): server_args = ServerArgs( @@ -1570,7 +1658,7 @@ class TestWaterfillArgs(CustomTestCase): self.assertEqual(server_args.moe_a2a_backend, "none") # pristine self.assertEqual(resolved_view(server_args).moe_a2a_backend, "deepep") - self.assertTrue(server_args.enforce_shared_experts_fusion) + self.assertTrue(resolution_result(server_args, "enforce_shared_experts_fusion")) def test_waterfill_keeps_megamoe_backend(self): server_args = ServerArgs( @@ -1586,7 +1674,7 @@ class TestWaterfillArgs(CustomTestCase): self.assertEqual(resolved_view(server_args).moe_a2a_backend, "megamoe") self.assertFalse(resolved_view(server_args).disable_shared_experts_fusion) - self.assertTrue(server_args.enforce_shared_experts_fusion) + self.assertTrue(resolution_result(server_args, "enforce_shared_experts_fusion")) def test_waterfill_supports_deepep_low_latency_mode(self): server_args = ServerArgs( @@ -1598,9 +1686,9 @@ class TestWaterfillArgs(CustomTestCase): # dummy-model path short-circuits __post_init__; invoke the handler directly. server_args._handle_a2a_moe() - self.assertEqual(server_args.deepep_mode, "low_latency") - self.assertFalse(server_args.disable_cuda_graph) - self.assertTrue(server_args.enforce_shared_experts_fusion) + self.assertEqual(resolution_result(server_args, "deepep_mode"), "low_latency") + self.assertFalse(resolution_result(server_args, "disable_cuda_graph")) + self.assertTrue(resolution_result(server_args, "enforce_shared_experts_fusion")) class TestPrefillOnlyDisableKvCache(unittest.TestCase): @@ -1635,7 +1723,7 @@ class TestPrefillOnlyDisableKvCache(unittest.TestCase): def test_valid_minimal_config_constructs(self): sa = self._validate_prefill_only_args() - self.assertTrue(sa.prefill_only_disable_kv_cache) + self.assertTrue(resolution_result(sa, "prefill_only_disable_kv_cache")) def test_rejects_when_not_embedding(self): with self.assertRaisesRegex(ValueError, "requires --is-embedding"): @@ -1725,15 +1813,27 @@ class TestCudaGraphDisaggregationRoles(CustomTestCase): def test_cuda_graph_prefill_role_defaults_disable_decode_graph(self): args = self._handled_args(disaggregation_mode="prefill") - self.assertFalse(args.disable_cuda_graph) - self.assertEqual(args.cuda_graph_config.decode.backend, Backend.DISABLED) - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) + self.assertFalse(resolution_result(args, "disable_cuda_graph")) + self.assertEqual( + resolution_result(args, "cuda_graph_config").decode.backend, + Backend.DISABLED, + ) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.BREAKABLE, + ) def test_cuda_graph_decode_role_defaults_disable_prefill_graph(self): args = self._handled_args(disaggregation_mode="decode") - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.DISABLED) - self.assertNotEqual(args.cuda_graph_config.decode.backend, Backend.DISABLED) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.DISABLED, + ) + self.assertNotEqual( + resolution_result(args, "cuda_graph_config").decode.backend, + Backend.DISABLED, + ) def test_cuda_graph_global_disable_still_disables_both_phases_for_all_roles(self): for disaggregation_mode in ("prefill", "decode", "null"): @@ -1744,10 +1844,12 @@ class TestCudaGraphDisaggregationRoles(CustomTestCase): ) self.assertEqual( - args.cuda_graph_config.decode.backend, Backend.DISABLED + resolution_result(args, "cuda_graph_config").decode.backend, + Backend.DISABLED, ) self.assertEqual( - args.cuda_graph_config.prefill.backend, Backend.DISABLED + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.DISABLED, ) def test_cuda_graph_explicit_decode_backend_survives_prefill_role(self): @@ -1756,7 +1858,9 @@ class TestCudaGraphDisaggregationRoles(CustomTestCase): cuda_graph_backend_decode=Backend.FULL, ) - self.assertEqual(args.cuda_graph_config.decode.backend, Backend.FULL) + self.assertEqual( + resolution_result(args, "cuda_graph_config").decode.backend, Backend.FULL + ) self.assertIn((Phase.DECODE, "backend"), args._cuda_graph_config_locked) @@ -1782,12 +1886,18 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase): def test_enable_lora_keeps_breakable_prefill_graph(self): args = self._handled_args(enable_lora=True) - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.BREAKABLE, + ) def test_lora_paths_keep_breakable_prefill_graph(self): args = self._handled_args(lora_paths=["dummy/lora-adapter"]) - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.BREAKABLE, + ) def test_lora_still_disables_tc_piecewise_prefill_graph(self): # Pin the tc_piecewise LoRA rule itself, with the hardware rule @@ -1811,7 +1921,10 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase): ): args._disable_tc_piecewise_cudagraph_if_incompatible() - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.DISABLED) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.DISABLED, + ) class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase): @@ -1840,7 +1953,10 @@ class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase): is_multimodal=True, allowlisted=False, ) - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.DISABLED) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.DISABLED, + ) def test_allowlisted_multimodal_arch_keeps_prefill_breakable(self): args = self._handled_args( @@ -1848,7 +1964,10 @@ class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase): is_multimodal=True, allowlisted=True, ) - self.assertEqual(args.cuda_graph_config.prefill.backend, Backend.BREAKABLE) + self.assertEqual( + resolution_result(args, "cuda_graph_config").prefill.backend, + Backend.BREAKABLE, + ) def test_allowlist_membership(self): from sglang.srt.configs.model_config import ( @@ -2046,20 +2165,20 @@ class TestGrpcServerArgs(CustomTestCase): def test_http_only_high_port_does_not_derive_grpc_port(self): sa = self._args(port=56000) sa._handle_deprecated_args() - self.assertIsNone(sa.grpc_port) + self.assertIsNone(resolution_result(sa, "grpc_port")) def test_grpc_port_enables_native_and_env_knobs(self): sa = self._args(grpc_port=50051) with envs.SGLANG_GRPC_WORKER_THREADS.override(8): sa._handle_deprecated_args() - self.assertEqual(sa.grpc_port, 50051) + self.assertEqual(resolution_result(sa, "grpc_port"), 50051) self.assertEqual(sa.grpc_worker_threads, 8) def test_env_grpc_port_enables_native(self): sa = self._args(port=30000) with envs.SGLANG_GRPC_PORT.override(45000): sa._handle_deprecated_args() - self.assertEqual(sa.grpc_port, 45000) + self.assertEqual(resolution_result(sa, "grpc_port"), 45000) @staticmethod def _sidecar_parser(): @@ -2196,20 +2315,20 @@ class TestGrpcServerArgs(CustomTestCase): def test_legacy_smg_derives_grpc_port_from_http_port(self): sa = self._args(port=30000, smg_grpc_mode=True) sa._handle_deprecated_args() - self.assertEqual(sa.grpc_port, 40000) + self.assertEqual(resolution_result(sa, "grpc_port"), 40000) def test_grpc_mode_is_deprecated_alias_for_smg_grpc_mode(self): sa = self._args(grpc_mode=True) with self.assertLogs(server_args_module.logger, level="WARNING") as cm: sa._handle_deprecated_args() - self.assertTrue(sa.smg_grpc_mode) + self.assertTrue(resolution_result(sa, "smg_grpc_mode")) self.assertTrue(any("--grpc-mode is deprecated" in line for line in cm.output)) def test_legacy_smg_takes_precedence_over_grpc_port(self): sa = self._args(grpc_port=50051, smg_grpc_mode=True) sa._handle_deprecated_args() - self.assertTrue(sa.smg_grpc_mode) - self.assertEqual(sa.grpc_port, 50051) + self.assertTrue(resolution_result(sa, "smg_grpc_mode")) + self.assertEqual(resolution_result(sa, "grpc_port"), 50051) def test_native_grpc_rejects_multi_tokenizer(self): sa = self._args(grpc_port=40000, tokenizer_worker_num=2) @@ -2254,7 +2373,7 @@ class TestGrpcServerArgs(CustomTestCase): tokenizer_manager=MagicMock(), template_manager=MagicMock(), scheduler_info={}, - grpc_port=server_args.grpc_port, + grpc_port=resolution_result(server_args, "grpc_port"), ) self.assertEqual(handle, "handle") diff --git a/test/registered/unit/spec/test_spec_cpu_overlap_constraint.py b/test/registered/unit/spec/test_spec_cpu_overlap_constraint.py index 6e21a69be..68ef5eac9 100644 --- a/test/registered/unit/spec/test_spec_cpu_overlap_constraint.py +++ b/test/registered/unit/spec/test_spec_cpu_overlap_constraint.py @@ -1,6 +1,7 @@ import unittest from types import SimpleNamespace +from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci @@ -33,18 +34,18 @@ def _make_spec_args(device: str, algorithm: str = "EAGLE", **overrides) -> Serve class TestSpecCPUOverlapConstraint(CustomTestCase): def test_cpu_eagle_forces_disable_overlap_schedule(self): args = _make_spec_args(device="cpu") - self.assertFalse(args.disable_overlap_schedule) + self.assertFalse(resolution_result(args, "disable_overlap_schedule")) handle_speculative_decoding(args) - self.assertTrue(args.disable_overlap_schedule) + self.assertTrue(resolution_result(args, "disable_overlap_schedule")) def test_cpu_eagle3_forces_disable_overlap_schedule(self): args = _make_spec_args(device="cpu", algorithm="EAGLE3") handle_speculative_decoding(args) - self.assertTrue(args.disable_overlap_schedule) + self.assertTrue(resolution_result(args, "disable_overlap_schedule")) def test_cpu_explicit_disable_overlap_is_preserved(self): args = _make_spec_args(device="cpu", disable_overlap_schedule=True) @@ -56,7 +57,7 @@ class TestSpecCPUOverlapConstraint(CustomTestCase): ) as logs: handle_speculative_decoding(args) - self.assertTrue(args.disable_overlap_schedule) + self.assertTrue(resolution_result(args, "disable_overlap_schedule")) self.assertFalse( any("Overlap schedule" in message for message in logs.output), f"hook warned about overriding an already-disabled overlap: {logs.output}", @@ -68,7 +69,7 @@ class TestSpecCPUOverlapConstraint(CustomTestCase): handle_speculative_decoding(args) - self.assertFalse(args.disable_overlap_schedule) + self.assertFalse(resolution_result(args, "disable_overlap_schedule")) if __name__ == "__main__": diff --git a/test/registered/unit/spec/test_spec_registry.py b/test/registered/unit/spec/test_spec_registry.py index 141b854c4..047a8b717 100644 --- a/test/registered/unit/spec/test_spec_registry.py +++ b/test/registered/unit/spec/test_spec_registry.py @@ -4,6 +4,7 @@ import unittest from types import SimpleNamespace from unittest.mock import MagicMock +from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.speculative.spec_registry import ( @@ -237,7 +238,9 @@ class TestServerArgsHook(_RegistryIsolated): handle_speculative_decoding(server_args) - self.assertEqual(server_args.speculative_algorithm, "MY_HANDLE_ARGS") + self.assertEqual( + resolution_result(server_args, "speculative_algorithm"), "MY_HANDLE_ARGS" + ) self.assertEqual(server_args.custom_spec_handle_seen, "MY_HANDLE_ARGS") self.assertEqual(server_args.speculative_num_draft_tokens, 7) diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index c7d748f6a..fbb40f2b4 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -280,10 +280,16 @@ class TestPublishInstallsSlot(_IsolatedPublish): set_global_server_args_for_scheduler(sa) self.assertIs(get_server_args(), sa) # Publishing is what resolved it; the handlers ahead of the dummy - # short-circuit still declare. + # short-circuit still declare. What they decided is the projection -- + # the fields keep what the caller passed. + from sglang.srt.arg_groups.overrides import resolution_result + + self.assertTrue(sa._resolved_overrides, "publishing declared nothing") for source, declared in sa._resolved_overrides: for field, value in declared.items(): - self.assertEqual(getattr(sa, field), value, f"{source}: {field}") + self.assertEqual( + resolution_result(sa, field), value, f"{source}: {field}" + ) class TestGoldenModelOverrides(_IsolatedPublish): diff --git a/test/registered/unit/test_runtime_context_override.py b/test/registered/unit/test_runtime_context_override.py index e7dfd144e..623125a00 100644 --- a/test/registered/unit/test_runtime_context_override.py +++ b/test/registered/unit/test_runtime_context_override.py @@ -31,11 +31,15 @@ class TestContextOverride(CustomTestCase): def test_override_writes_bag_not_server_args(self): sa = self._publish() - before = sa.hicache_ratio + # The published leaf, not the field: `hicache_ratio` is resolved by + # declaration, so the field still holds what the caller passed. + before = rc.get_memory().hicache_ratio + pristine = sa.hicache_ratio rc.get_context().override("test", hicache_ratio=before + 1.0) self.assertEqual(rc.get_memory().hicache_ratio, before + 1.0) - # server_args stays the pristine startup record. - self.assertEqual(sa.hicache_ratio, before) + # server_args stays the pristine startup record: the override does not + # touch it, and neither did resolution. + self.assertEqual(sa.hicache_ratio, pristine) def test_override_routes_across_namespaces(self): self._publish() diff --git a/test/registered/unit/test_server_args_migration.py b/test/registered/unit/test_server_args_migration.py index cac5d7cc6..67b2d9eb7 100644 --- a/test/registered/unit/test_server_args_migration.py +++ b/test/registered/unit/test_server_args_migration.py @@ -7,6 +7,7 @@ translates field annotations into argparse arguments. import argparse import unittest +from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.server_args import ServerArgs from sglang.srt.utils.common import configure_media_url_security from sglang.test.ci.ci_register import register_cpu_ci @@ -95,8 +96,10 @@ class TestServerArgsAnnotatedCli(CustomTestCase): "32", ] ) + # The normalization is a declaration. self.assertEqual( - sa.allowed_media_domains, ["127.0.0.1", "media.example.com"] + resolution_result(sa, "allowed_media_domains"), + ["127.0.0.1", "media.example.com"], ) self.assertEqual(sa.media_url_max_file_size_mb, 32) finally: diff --git a/test/registered/unit/test_supplied_instance_exposure_ratchet.py b/test/registered/unit/test_supplied_instance_exposure_ratchet.py index 3c366806f..23e863670 100644 --- a/test/registered/unit/test_supplied_instance_exposure_ratchet.py +++ b/test/registered/unit/test_supplied_instance_exposure_ratchet.py @@ -134,10 +134,10 @@ _ENV_MATRIX = (({}, {"SGLANG_IS_IN_CI": "true"}),) # are step-12 exposure like any other pair. _PASSED = frozenset({"model_path", "device", "random_seed"}) +# The reads that still take a value off the supplied instance. `initialize_moe_config` +# is handed the record until the replay goes away; the rest are pre-publish launcher +# reads. _EXPOSED = { - ("dllm/config.py", "max_running_requests"), - ("dllm/config.py", "model_path"), - ("speculative/spec_registry.py", "disable_overlap_schedule"), ("disaggregation/encoder/server.py", "model_loader_extra_config"), ("layers/moe/utils.py", "deepep_mode"), ("layers/moe/utils.py", "disable_shared_experts_fusion"), @@ -150,40 +150,16 @@ _EXPOSED = { ("configs/embedding_model_spec.py", "disable_radix_cache"), ("configs/embedding_model_spec.py", "is_embedding"), ("configs/embedding_model_spec.py", "prefill_only_disable_kv_cache"), - ("configs/model_config.py", "_speculative_draft_quantization_explicitly_set"), - ("configs/model_config.py", "disable_hybrid_swa_memory"), - ("configs/model_config.py", "dtype"), - ("configs/model_config.py", "enable_multi_layer_eagle"), - ("configs/model_config.py", "is_embedding"), - ("configs/model_config.py", "model_path"), - ("configs/model_config.py", "quantization"), - ("configs/model_config.py", "speculative_algorithm"), - ("configs/model_config.py", "speculative_draft_model_quantization"), - ("dllm/config.py", "max_running_requests"), - ("dllm/config.py", "model_path"), ("entrypoints/engine.py", "enable_symm_mem"), ("entrypoints/engine.py", "reasoning_parser"), ("entrypoints/engine.py", "tool_call_parser"), - ("layers/cp/base.py", "attn_cp_size"), - ("layers/cp/base.py", "cp_strategy"), - ("layers/cp/base.py", "enable_prefill_cp"), - ("layers/cp/bcg.py", "cp_strategy"), - ("layers/cp/bcg.py", "enable_prefill_cp"), ("layers/flashinfer_comm_fusion.py", "flashinfer_allreduce_fusion_backend"), ("layers/moe/utils.py", "deepep_mode"), ("layers/moe/utils.py", "moe_a2a_backend"), ("layers/moe/utils.py", "moe_runner_backend"), ("layers/moe/utils.py", "quantization"), ("layers/moe/utils.py", "speculative_moe_runner_backend"), - ("lora/marlin_lora_temp/policy.py", "lora_paths"), - ("model_loader/expert_pack_runtime.py", "model_path"), - ("model_loader/expert_pack_runtime.py", "tokenizer_path"), - ("parser/template_detection.py", "model_path"), - ("speculative/adaptive_spec_params.py", "speculative_algorithm"), - ("speculative/adaptive_spec_params.py", "speculative_eagle_topk"), ("speculative/draft_worker_common.py", "speculative_draft_attention_backend"), - ("speculative/spec_info.py", "enable_multi_layer_eagle"), - ("speculative/spec_registry.py", "disable_overlap_schedule"), ("utils/common.py", "speculative_num_draft_tokens"), ("utils/common.py", "speculative_num_steps"), ("utils/hf_transformers/processor.py", "image_processor_backend"), @@ -216,26 +192,19 @@ _EXPOSED_CUDA_ONLY: frozenset = frozenset() # some code overrides post-publish. Each needs an ordering judgment, not a blanket # conversion; the list exists so a new one is a decision made when it is written. _OVERRIDDEN_AND_READ = { - ("configs/model_config.py", "dtype"), - ("configs/model_config.py", "model_path"), - ("dllm/config.py", "model_path"), ("entrypoints/engine.py", "reasoning_parser"), ("entrypoints/engine.py", "tool_call_parser"), - ("model_loader/expert_pack_runtime.py", "model_path"), ("weight_cache/daemon.py", "dp_size"), ("weight_cache/daemon.py", "dtype"), ("weight_cache/daemon.py", "ep_size"), ("weight_cache/daemon.py", "load_format"), ("weight_cache/daemon.py", "model_path"), - ("configs/model_config.py", "dtype"), - ("configs/model_config.py", "model_path"), ("mem_cache/pool_host/common.py", "hicache_storage_backend"), ("mem_cache/pool_host/common.py", "hicache_storage_backend_extra_config"), ("mem_cache/unified_radix_cache.py", "hicache_storage_backend"), ("mem_cache/unified_radix_cache.py", "hicache_storage_backend_extra_config"), ("mem_cache/unified_radix_cache.py", "hicache_storage_prefetch_policy"), ("mem_cache/unified_radix_cache.py", "hicache_write_policy"), - ("parser/template_detection.py", "model_path"), ("utils/common.py", "speculative_num_draft_tokens"), ("utils/common.py", "speculative_num_steps"), ("weight_cache/daemon.py", "dp_size"),