From be32c575983a52ba7c536538ede0d4c567f5d5c7 Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:28:34 -0700 Subject: [PATCH] [refactor] Read resolved config from server_args fields; retire the flags mirror tier (#30346) --- python/sglang/srt/arg_groups/arg_utils.py | 4 +- python/sglang/srt/arg_groups/overrides.py | 127 ++------- .../srt/batch_overlap/two_batch_overlap.py | 4 +- python/sglang/srt/layers/logits_processor.py | 4 +- .../srt/layers/rotary_embedding/mrope.py | 5 +- python/sglang/srt/layers/sampler.py | 6 +- python/sglang/srt/managers/scheduler.py | 8 - python/sglang/srt/mem_cache/common.py | 6 +- .../sglang/srt/model_executor/model_runner.py | 4 +- python/sglang/srt/models/apertus.py | 4 +- python/sglang/srt/models/arcee.py | 4 +- python/sglang/srt/models/bailing_moe.py | 4 +- .../sglang/srt/models/bailing_moe_linear.py | 4 +- python/sglang/srt/models/bailing_moe_nextn.py | 4 +- python/sglang/srt/models/deepseek_nextn.py | 6 +- python/sglang/srt/models/deepseek_v2.py | 8 +- python/sglang/srt/models/deepseek_v4.py | 6 +- python/sglang/srt/models/deepseek_v4_nextn.py | 4 +- python/sglang/srt/models/exaone4.py | 4 +- python/sglang/srt/models/exaone_moe.py | 4 +- python/sglang/srt/models/exaone_moe_mtp.py | 4 +- python/sglang/srt/models/falcon_h1.py | 4 +- python/sglang/srt/models/glm4_moe.py | 10 +- python/sglang/srt/models/glm4_moe_lite.py | 10 +- .../sglang/srt/models/glm4_moe_lite_nextn.py | 6 +- python/sglang/srt/models/glm4_moe_nextn.py | 6 +- python/sglang/srt/models/glm4v_moe.py | 6 +- python/sglang/srt/models/glm_ocr_nextn.py | 6 +- python/sglang/srt/models/gpt_oss.py | 6 +- python/sglang/srt/models/laguna.py | 4 +- python/sglang/srt/models/llada2.py | 4 +- python/sglang/srt/models/llama.py | 4 +- python/sglang/srt/models/longcat_flash.py | 4 +- python/sglang/srt/models/mimo_v2.py | 4 +- python/sglang/srt/models/mimo_v2_nextn.py | 4 +- python/sglang/srt/models/nemotron_h.py | 4 +- python/sglang/srt/models/nemotron_h_mtp.py | 4 +- python/sglang/srt/models/qwen2_moe.py | 6 +- python/sglang/srt/models/qwen3.py | 4 +- python/sglang/srt/models/qwen3_5.py | 6 +- python/sglang/srt/models/qwen3_5_mtp.py | 4 +- python/sglang/srt/models/qwen3_moe.py | 4 +- python/sglang/srt/models/qwen3_moe_mtp.py | 4 +- python/sglang/srt/models/qwen3_next.py | 4 +- python/sglang/srt/models/qwen3_next_mtp.py | 6 +- python/sglang/srt/models/qwen3_vl.py | 4 +- python/sglang/srt/models/sarvam_moe.py | 4 +- python/sglang/srt/models/sdar.py | 4 +- python/sglang/srt/models/sdar_moe.py | 4 +- python/sglang/srt/models/step3p5.py | 4 +- python/sglang/srt/runtime_context.py | 249 ++---------------- .../mock_model/test_self_unit_install.py | 4 - .../ops/test_aiter_greedy_sample_amd.py | 15 +- test/registered/rl/test_fp32_lm_head.py | 2 - .../test_tbo_filter_batch_marker.py | 4 +- .../model_executor/test_pool_configurator.py | 4 - .../test_deepseek_v4_shared_expert_fusion.py | 9 +- test/registered/unit/test_model_overrides.py | 136 ++-------- test/registered/unit/test_runtime_context.py | 238 ++++------------- .../unit/test_server_args_mutation_ratchet.py | 4 +- 60 files changed, 238 insertions(+), 795 deletions(-) diff --git a/python/sglang/srt/arg_groups/arg_utils.py b/python/sglang/srt/arg_groups/arg_utils.py index aa8738e5c..8cda59fb4 100644 --- a/python/sglang/srt/arg_groups/arg_utils.py +++ b/python/sglang/srt/arg_groups/arg_utils.py @@ -77,8 +77,8 @@ class Arg: no_cli: bool = False # When True, this field may be written by config resolution (model # overrides and post-process passes): it is part of the whitelist accepted - # by the apply_model_overrides gate, and its resolved value lives on the - # flags tier (the server_args field itself stays the pristine user input). + # by the declaration stash, and its resolved value materializes onto the + # field at the end of __post_init__. resolvable: bool = False diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index cb33fd5f8..864dc52de 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -14,8 +14,9 @@ """Declarative model-override registry. Model-identity adjustments to the server configuration are DECLARED here and -resolved into the flags tier through the ``apply_model_overrides`` gate — -model code never mutates ``ServerArgs``, which stays the pristine user input. +materialized onto ``server_args`` at the end of ``__post_init__`` (gate +order, last writer wins) — model code never mutates ``ServerArgs`` fields +imperatively. Two declaration forms, keyed on ``hf_config.architectures[0]``: @@ -30,11 +31,10 @@ from __future__ import annotations import dataclasses import logging -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple from sglang.srt.arg_groups.arg_utils import resolvable_fields from sglang.srt.model_executor.cuda_graph_config import Backend -from sglang.srt.runtime_context import resolve_flag_leaf from sglang.srt.utils.common import ( cpu_has_amx_support, get_device_capability, @@ -259,16 +259,19 @@ def mamba_extra_buffer_of(cfg: Any) -> bool: def declare_load_time_override(source: str, declared: Dict[str, Any]) -> None: """Declare a load-time resolved field (model-file config overrides, - weight-resolved dtypes): apply it onto the published ``server_args`` — - resolution has already materialized, so post-init declarations write - through — and record it into the flags tier through the runtime gate.""" + weight-resolved dtypes) on the published ``server_args``: resolution has + already materialized, so the declaration writes through, joining the + declaration stash for provenance and republish consistency.""" from sglang.srt.runtime_context import get_context - ctx = get_context() - entry = (source, dict(declared)) - validate_declarations(ctx.server_args, [entry]) - _apply_fields(ctx.server_args, declared) - ctx.record_runtime_overrides([entry]) + server_args = get_context().server_args + validate_declarations(server_args, [(source, dict(declared))]) + override = getattr(server_args, "override", None) + if override is not None: + override(source, **declared) + else: + # Config-shaped fixtures without the mutation entry point. + _apply_fields(server_args, declared) def collect_model_override_declarations( @@ -2054,84 +2057,6 @@ def _dllm_page_size(view: Any) -> dict: return {} -@dataclasses.dataclass(frozen=True) -class OverrideRecord: - """Provenance of one resolved write: ``base`` is the value before this - declaration applied (the pristine value for the first writer).""" - - source: str - field: str - base: Any - resolved: Any - - -def apply_model_overrides( - flags: Any, - server_args: Any, - declarations: Sequence[Tuple[str, Dict[str, Any]]], - *, - terminal: Sequence[Tuple[str, Dict[str, Any]]] = (), - whitelist: Optional[Iterable[str]] = None, - leaf_map: Optional[Dict[str, str]] = None, -) -> List[OverrideRecord]: - """Resolve model-override declarations into the flags tier. - - - **Transactional**: every declaration (``terminal`` included) is - validated against the whitelist and the flag-leaf layout BEFORE any - write; on error nothing is applied. - - **Ordering**: ``declarations`` apply in order (last writer wins), then - ``terminal`` (the enforce-disable pass) applies after everything. - - **Materialization**: every whitelisted field becomes a flag leaf — - declared fields carry the resolved value, undeclared ones the pristine - ``server_args`` value — so readers only ever read flags, never a - "flag or fallback to config" combination. - - ``server_args`` is read-only here: resolution output lives on flags. - - Returns the provenance log, one record per declared write. - """ - if whitelist is None: - whitelist = resolvable_fields(type(server_args)) - whitelist = frozenset(whitelist) - - ordered = list(declarations) + list(terminal) - - problems = [ - f"{source}: {sorted(set(decl) - whitelist)} not model-overridable" - for source, decl in ordered - if set(decl) - whitelist - ] - if problems: - raise ValueError( - "model override validation failed (nothing was applied): " - + "; ".join(problems) - ) - for field in sorted(whitelist): - owner, leaf = resolve_flag_leaf(flags, field, leaf_map=leaf_map) - if leaf not in type(owner).__dataclass_fields__: - raise ValueError( - f"flag leaf for '{field}' is not declared on " - f"{type(owner).__name__} (declare the dataclass field and map " - "it in FLAG_LEAF_MAP); nothing was applied" - ) - if getattr(owner, "_frozen", False): - raise RuntimeError( - f"cannot resolve '{field}': {type(owner).__name__} is frozen; " - "nothing was applied" - ) - - resolved = {field: getattr(server_args, field) for field in whitelist} - records: List[OverrideRecord] = [] - for source, decl in ordered: - for field, value in decl.items(): - records.append(OverrideRecord(source, field, resolved[field], value)) - resolved[field] = value - - for field, value in resolved.items(): - owner, leaf = resolve_flag_leaf(flags, field, leaf_map=leaf_map) - setattr(owner, leaf, value) - return records - - def validate_declarations( server_args: Any, declarations: Sequence[Tuple[str, Dict[str, Any]]], @@ -2168,25 +2093,3 @@ def _hrm_text_attention_force(view: Any) -> dict: "attention." ) return {"attention_backend": "triton"} - - -def assert_flag_parity( - flags: Any, - server_args: Any, - fields: Iterable[str], - *, - leaf_map: Optional[Dict[str, str]] = None, -) -> None: - """Drift guard: each declared field's flag leaf must equal the - (materialized) ``server_args`` value.""" - mismatches = [] - for field in fields: - owner, leaf = resolve_flag_leaf(flags, field, leaf_map=leaf_map) - flag_value = getattr(owner, leaf) - args_value = getattr(server_args, field) - if flag_value != args_value: - mismatches.append( - f"{field}: flags={flag_value!r} server_args={args_value!r}" - ) - if mismatches: - raise AssertionError("flag/server_args parity broken: " + "; ".join(mismatches)) diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index fc5dac537..ee755d373 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -39,7 +39,7 @@ from sglang.srt.model_executor.forward_batch_info import ( compute_position, ) from sglang.srt.model_executor.forward_context import get_attn_backend -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.speculative.spec_info import SpecInput from sglang.srt.utils import BumpAllocator, empty_context, get_bool_env_var, is_hip @@ -634,7 +634,7 @@ class TboForwardBatchPreparer: sum_field=None, ) _, child_b.extend_start_loc = compute_position( - get_flags().attn.backend, + get_server_args().attention_backend, child_b.extend_prefix_lens, child_b.extend_seq_lens, child_b.extend_num_tokens, diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index 45dc18463..e3dde55cc 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -47,7 +47,7 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardBatch, ForwardMode, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils.common import ( is_cpu, @@ -335,7 +335,7 @@ class LogitsProcessor(nn.Module): self.config = config self.vocab_size = config.vocab_size self.logit_scale = logit_scale - self.use_attn_tp_group = get_flags().enable_dp_lm_head + self.use_attn_tp_group = get_server_args().enable_dp_lm_head self.use_fp32_lm_head = get_global_server_args().enable_fp32_lm_head if self.use_attn_tp_group: self.attn_tp_size = get_parallel().attn_tp_size diff --git a/python/sglang/srt/layers/rotary_embedding/mrope.py b/python/sglang/srt/layers/rotary_embedding/mrope.py index 4ddeebeff..bc9f29c5f 100644 --- a/python/sglang/srt/layers/rotary_embedding/mrope.py +++ b/python/sglang/srt/layers/rotary_embedding/mrope.py @@ -18,7 +18,6 @@ from sglang.srt.layers.rotary_embedding.yarn import ( yarn_get_mscale_simple, yarn_linear_ramp_mask, ) -from sglang.srt.runtime_context import get_flags from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( cpu_has_amx_support, @@ -45,6 +44,8 @@ if _is_xpu: import triton import triton.language as tl +from sglang.srt.runtime_context import get_server_args + @triton.jit def apply_interleaved_rope_kernel( @@ -227,7 +228,7 @@ class MRotaryEmbedding(RotaryEmbedding): last_dim = cos_sin.size()[-1] cos, sin = cos_sin.chunk(2, dim=-1) if self.mrope_interleaved: - if support_triton(get_flags().attn.backend): + if support_triton(get_server_args().attention_backend): cos = apply_interleaved_rope_triton(cos, self.mrope_section) sin = apply_interleaved_rope_triton(sin, self.mrope_section) else: diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py index e83e4157f..bce846a98 100644 --- a/python/sglang/srt/layers/sampler.py +++ b/python/sglang/srt/layers/sampler.py @@ -13,7 +13,7 @@ from sglang.srt.layers.dp_attention import ( from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.layers.utils.hash import murmur_hash32 from sglang.srt.layers.utils.logprob import get_token_ids_logprobs, get_top_logprobs -from sglang.srt.runtime_context import get_flags +from sglang.srt.runtime_context import get_server_args from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo from sglang.srt.sampling.sampling_params import TOP_K_ALL from sglang.srt.server_args import get_global_server_args @@ -80,7 +80,7 @@ class Sampler(nn.Module): ) # In RL on-policy mode, we use log_softmax to compute logprobs to match the trainer. self.use_log_softmax_logprob = self.rl_on_policy_target is not None - self.use_ascend_backend = get_flags().sampling_backend == "ascend" + self.use_ascend_backend = get_server_args().sampling_backend == "ascend" def _preprocess_logits( self, logits: torch.Tensor, sampling_info: SamplingBatchInfo @@ -231,7 +231,7 @@ class Sampler(nn.Module): positions=positions, ) else: - backend = get_flags().sampling_backend + backend = get_server_args().sampling_backend if backend == "flashinfer": assert ( sampling_info.sampling_seed is None diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 9061c619d..58500a66b 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -554,14 +554,6 @@ class Scheduler( self.init_batch_result_processor() - # The config-resolution lifecycle of this scheduler process ends - # here: every load-time stage has run (target and draft model init, - # weight-resolved kv-cache dtype), so lock the static flag groups. - # flags.capture stays writable; late resolution writes now raise. - from sglang.srt.runtime_context import get_context - - get_context().freeze_flags() - self.is_initializing = False def init_zbal_on_npu(self): diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 233a6ef29..461a795e0 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -25,7 +25,7 @@ from sglang.srt.mem_cache.triton_ops.common import ( get_last_loc_triton_safe, write_req_to_token_pool_triton, ) -from sglang.srt.runtime_context import get_flags +from sglang.srt.runtime_context import get_server_args from sglang.srt.server_args import ServerArgs, get_global_server_args from sglang.srt.utils import is_cuda, is_hip, is_npu, support_triton from sglang.srt.utils.common import ceil_align, is_pin_memory_available @@ -134,7 +134,7 @@ def write_cache_indices( prefix_tensors: list[torch.Tensor], req_to_token_pool: ReqToTokenPool, ): - if support_triton(get_flags().attn.backend): + if support_triton(get_server_args().attention_backend): prefix_pointers = torch.tensor( [t.data_ptr() for t in prefix_tensors], dtype=torch.uint64, @@ -175,7 +175,7 @@ def get_last_loc( req_pool_indices_tensor: torch.Tensor, prefix_lens_tensor: torch.Tensor, ) -> torch.Tensor: - attn_backend = get_flags().attn.backend + attn_backend = get_server_args().attention_backend uses_triton_dispatch = attn_backend not in ("ascend", "torch_native") if _is_hip and uses_triton_dispatch: diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index ab770cf33..b72ed6c33 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -182,7 +182,7 @@ from sglang.srt.model_loader.remote_instance_weight_loader_utils import ( from sglang.srt.model_loader.utils import set_default_torch_dtype from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_flags +from sglang.srt.runtime_context import get_flags, get_server_args from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo from sglang.srt.server_args import ( # noqa: F401 (re-export) CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS, @@ -541,7 +541,7 @@ class ModelRunner(ModelRunnerKVCacheMixin): self.init_threads_binding() # Set float32 matmul precision - if get_flags().enable_tf32_matmul: + if get_server_args().enable_tf32_matmul: torch.set_float32_matmul_precision("high") # Get available memory before model loading. diff --git a/python/sglang/srt/models/apertus.py b/python/sglang/srt/models/apertus.py index 6d8daf25e..2ba7f63d7 100644 --- a/python/sglang/srt/models/apertus.py +++ b/python/sglang/srt/models/apertus.py @@ -52,7 +52,7 @@ from sglang.srt.model_loader.weight_utils import ( kv_cache_scales_loader, maybe_remap_kv_scale_name, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, make_layers logger = logging.getLogger(__name__) @@ -446,7 +446,7 @@ class ApertusForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) diff --git a/python/sglang/srt/models/arcee.py b/python/sglang/srt/models/arcee.py index 0bc11476a..20d0ecc7c 100644 --- a/python/sglang/srt/models/arcee.py +++ b/python/sglang/srt/models/arcee.py @@ -46,7 +46,7 @@ from sglang.srt.model_loader.weight_utils import ( kv_cache_scales_loader, maybe_remap_kv_scale_name, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, make_layers logger = logging.getLogger(__name__) @@ -405,7 +405,7 @@ class ArceeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index b86fd5eef..019656306 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -77,7 +77,7 @@ from sglang.srt.models.utils import ( create_fused_set_kv_buffer_arg, enable_fused_set_kv_buffer, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix, is_cuda, is_non_idle_and_non_empty, make_layers @@ -832,7 +832,7 @@ class BailingMoEForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 4e2dbdfbe..3a983f0ab 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -58,7 +58,7 @@ from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA, DeepseekV2MLP, _is_hip from sglang.srt.models.utils import WeightsMapper -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( BumpAllocator, @@ -1089,7 +1089,7 @@ class BailingMoELinearForCausalLM(nn.Module): config.hidden_size, params_dtype=torch.float32, quant_config=quant_config, - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py index 489fa58c1..5741f81c4 100644 --- a/python/sglang/srt/models/bailing_moe_nextn.py +++ b/python/sglang/srt/models/bailing_moe_nextn.py @@ -42,7 +42,7 @@ from sglang.srt.models.bailing_moe_linear import ( BailingMoeV2_5ForCausalLM, ) from sglang.srt.models.utils import WeightsMapper -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import BumpAllocator, add_prefix LoraConfig = None @@ -208,7 +208,7 @@ class BailingMoeForCausalLMNextN(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) if hasattr(self.config, "model_type") and config.model_type == "bailing_hybrid": diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index a51c5b51d..c5b4939c8 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -57,7 +57,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.deepseek_common.utils import enable_nextn_moe_bf16_cast_to_fp8 from sglang.srt.models.deepseek_v2 import DeepseekV2DecoderLayer, DeepseekV3ForCausalLM from sglang.srt.models.utils import WeightsMapper -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import BumpAllocator, add_prefix, is_cuda, is_npu @@ -172,7 +172,7 @@ class DeepseekModelNextN(nn.Module): if ( _is_npu and self.quant_config is None - and get_flags().quantization is not None + and get_server_args().quantization is not None ): # ascend mtp unquant exit_stack.enter_context(envs.SGLANG_DEEPEP_BF16_DISPATCH.override(True)) @@ -330,7 +330,7 @@ class DeepseekV3ForCausalLMNextN(DeepseekV3ForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 4e342f448..ed7022727 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -176,7 +176,7 @@ from sglang.srt.models.deepseek_common.utils import ( _use_aiter_bpreshuffle_gfx95, _use_aiter_gfx95, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_flags, get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils import ( @@ -541,7 +541,7 @@ class DeepseekV2MoE(nn.Module): n_shared_experts = ( 0 if config.n_shared_experts is None else int(config.n_shared_experts) ) - _fusion_disabled = get_flags().disable_shared_experts_fusion + _fusion_disabled = get_server_args().disable_shared_experts_fusion # num_fused_shared_experts drives weight remapping in deepseek_weight_loader: # mlp.shared_experts → mlp.experts.256 when > 0. @@ -2703,7 +2703,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: # ranks other than the last rank will have a placeholder layer @@ -2742,7 +2742,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): self.num_fused_shared_experts = 0 server_args = get_global_server_args() - if get_flags().disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 33a26896d..e66508ea4 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -125,7 +125,7 @@ from sglang.srt.models.deepseek_v2 import ( _is_npu, _is_xpu, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args if not _is_hip: from sglang.srt.layers.utils.cp_utils import ( @@ -2171,7 +2171,7 @@ class DeepseekV4ForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() @@ -2209,7 +2209,7 @@ class DeepseekV4ForCausalLM(nn.Module): def determine_num_fused_shared_experts(self): self.num_fused_shared_experts = 0 - if get_flags().disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None diff --git a/python/sglang/srt/models/deepseek_v4_nextn.py b/python/sglang/srt/models/deepseek_v4_nextn.py index e80c1ec6e..fe9d9b197 100644 --- a/python/sglang/srt/models/deepseek_v4_nextn.py +++ b/python/sglang/srt/models/deepseek_v4_nextn.py @@ -38,7 +38,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer, DeepseekV4ForCausalLM -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -233,7 +233,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/exaone4.py b/python/sglang/srt/models/exaone4.py index 8465a5d6e..7957c0dca 100644 --- a/python/sglang/srt/models/exaone4.py +++ b/python/sglang/srt/models/exaone4.py @@ -31,7 +31,7 @@ from sglang.srt.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, make_layers from sglang.utils import get_exception_traceback, logger @@ -443,7 +443,7 @@ class Exaone4ForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index fef7487c7..582568a10 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -62,7 +62,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_layers @@ -652,7 +652,7 @@ class ExaoneMoEForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # For EAGLE3 support diff --git a/python/sglang/srt/models/exaone_moe_mtp.py b/python/sglang/srt/models/exaone_moe_mtp.py index 5975087ee..439a4c354 100644 --- a/python/sglang/srt/models/exaone_moe_mtp.py +++ b/python/sglang/srt/models/exaone_moe_mtp.py @@ -30,7 +30,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.exaone_moe import ExaoneMoEForCausalLM, ExaoneMoEModel -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -63,7 +63,7 @@ class ExaoneMoEForCausalLMMTP(ExaoneMoEForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 780631250..3bd029a92 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -33,7 +33,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_cuda, make_layers logger = logging.getLogger(__name__) @@ -477,7 +477,7 @@ class FalconH1ForCausalLM(nn.Module): quant_config=quant_config, org_num_embeddings=config.vocab_size, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.lm_head = self.lm_head.float() self.lm_head_multiplier = config.lm_head_multiplier diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 558a83b38..97f20d3e3 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -82,7 +82,7 @@ from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.deepseek_v2 import DeepseekV2ForCausalLM from sglang.srt.models.utils import apply_qk_norm -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( add_prefix, @@ -404,7 +404,9 @@ class Glm4MoeSparseMoeBlock(nn.Module): self.routed_scaling_factor = config.routed_scaling_factor self.n_shared_experts = config.n_shared_experts self.num_fused_shared_experts = ( - 0 if get_flags().disable_shared_experts_fusion else config.n_shared_experts + 0 + if get_server_args().disable_shared_experts_fusion + else config.n_shared_experts ) self.config = config @@ -1184,7 +1186,7 @@ class Glm4MoeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) @@ -1192,7 +1194,7 @@ class Glm4MoeForCausalLM(nn.Module): self.capture_aux_hidden_states = False def determine_num_fused_shared_experts(self): - if get_flags().disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index f97bd3e48..b0176a250 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -74,7 +74,7 @@ from sglang.srt.models.deepseek_common.deepseek_weight_loader import ( ) from sglang.srt.models.deepseek_common.utils import _is_cuda, _use_aiter from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( BumpAllocator, @@ -188,7 +188,9 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): self.routed_scaling_factor = config.routed_scaling_factor self.n_shared_experts = config.n_shared_experts self.num_fused_shared_experts = ( - 0 if get_flags().disable_shared_experts_fusion else config.n_shared_experts + 0 + if get_server_args().disable_shared_experts_fusion + else config.n_shared_experts ) self.config = config self.layer_id = layer_id @@ -918,7 +920,7 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) @@ -939,7 +941,7 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): self, architecture: str = "Glm4MoeLiteForCausalLM" ): self.num_fused_shared_experts = 0 - if get_flags().disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index 50a644e38..9cb52ff9b 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -35,7 +35,7 @@ from sglang.srt.models.glm4_moe_lite import ( Glm4MoeLiteDecoderLayer, Glm4MoeLiteForCausalLM, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import BumpAllocator, add_prefix, is_npu @@ -155,12 +155,12 @@ class Glm4MoeLiteForCausalLMNextN(Glm4MoeLiteForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.num_fused_shared_experts = ( - 0 if get_flags().disable_shared_experts_fusion else 1 + 0 if get_server_args().disable_shared_experts_fusion else 1 ) @torch.no_grad() diff --git a/python/sglang/srt/models/glm4_moe_nextn.py b/python/sglang/srt/models/glm4_moe_nextn.py index 4fa88ba09..732c20aa3 100644 --- a/python/sglang/srt/models/glm4_moe_nextn.py +++ b/python/sglang/srt/models/glm4_moe_nextn.py @@ -32,7 +32,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.glm4_moe import Glm4MoeDecoderLayer, Glm4MoeForCausalLM -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix, is_npu @@ -141,12 +141,12 @@ class Glm4MoeForCausalLMNextN(Glm4MoeForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.num_fused_shared_experts = ( - 0 if get_flags().disable_shared_experts_fusion else 1 + 0 if get_server_args().disable_shared_experts_fusion else 1 ) @torch.no_grad() diff --git a/python/sglang/srt/models/glm4v_moe.py b/python/sglang/srt/models/glm4v_moe.py index 83d55414e..bc1d931a2 100644 --- a/python/sglang/srt/models/glm4v_moe.py +++ b/python/sglang/srt/models/glm4v_moe.py @@ -18,7 +18,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.glm4_moe import Glm4MoeModel from sglang.srt.models.glm4v import Glm4vForConditionalGeneration, Glm4vVisionModel -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix, get_device_sm, is_cuda, log_info_on_rank0 from sglang.srt.utils.hf_transformers_utils import get_processor @@ -70,7 +70,7 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: # ranks other than the last rank will have a placeholder layer @@ -84,7 +84,7 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): self.capture_aux_hidden_states = False def determine_num_fused_shared_experts(self): - if get_flags().disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None diff --git a/python/sglang/srt/models/glm_ocr_nextn.py b/python/sglang/srt/models/glm_ocr_nextn.py index 674509412..07a2bb245 100644 --- a/python/sglang/srt/models/glm_ocr_nextn.py +++ b/python/sglang/srt/models/glm_ocr_nextn.py @@ -33,7 +33,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.glm4 import Glm4DecoderLayer from sglang.srt.models.glm_ocr import GlmOcrForConditionalGeneration -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -134,12 +134,12 @@ class GlmOcrForConditionalGenerationNextN(GlmOcrForConditionalGeneration): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.num_fused_shared_experts = ( - 0 if get_flags().disable_shared_experts_fusion else 1 + 0 if get_server_args().disable_shared_experts_fusion else 1 ) @torch.no_grad() diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index c7dadfc43..b6dc2994a 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -68,7 +68,7 @@ from sglang.srt.models.utils import ( create_fused_set_kv_buffer_arg, enable_fused_set_kv_buffer, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( LazyValue, @@ -390,7 +390,7 @@ class GptOssAttention(nn.Module): # Choose dtype of sinks based on attention backend: trtllm_mha requires float32, # others can use bfloat16 - attn_backend = get_flags().attn.backend + attn_backend = get_server_args().attention_backend sinks_dtype = torch.float32 if attn_backend == "trtllm_mha" else torch.bfloat16 self.sinks = nn.Parameter( torch.empty(self.num_heads, dtype=sinks_dtype), requires_grad=False @@ -745,7 +745,7 @@ class GptOssForCausalLM(nn.Module): config.hidden_size, # quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 81b5e29af..ef7d9c041 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -53,7 +53,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.utils import apply_qk_norm -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import LazyValue, add_prefix, make_layers @@ -649,7 +649,7 @@ class LagunaForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index c795061c8..e77017606 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -76,7 +76,7 @@ from sglang.srt.models.utils import ( create_fused_set_kv_buffer_arg, enable_fused_set_kv_buffer, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( add_prefix, @@ -796,7 +796,7 @@ class LLaDA2MoeModelLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config, return_full_logits=True) diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index 12721be02..105a96441 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -52,7 +52,7 @@ from sglang.srt.model_loader.weight_utils import ( kv_cache_scales_loader, maybe_remap_kv_scale_name, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_cuda, is_npu, is_xpu, make_layers from sglang.utils import get_exception_traceback @@ -501,7 +501,7 @@ class LlamaForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 7e91b416f..69215d076 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -86,7 +86,7 @@ from sglang.srt.model_loader.utils import ( ) from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import ( BumpAllocator, add_prefix, @@ -644,7 +644,7 @@ class LongcatFlashForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 174f6fd69..d74b9ecfe 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -76,7 +76,7 @@ from sglang.srt.model_loader.weight_utils import ( ) from sglang.srt.models.mimo_audio import AudioEncoderMixin, MiMoAudioEncoderConfig from sglang.srt.models.mimo_vl import MiMoVisionTransformer, MiMoVLVisionConfig -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( LazyValue, @@ -1041,7 +1041,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index 7de4e20d3..49d32b66f 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -44,7 +44,7 @@ from sglang.srt.models.mimo_v2 import ( MiMoV2MLP, load_mimo_v2_qkv_proj_weight, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix MiMoV2Config = None @@ -259,7 +259,7 @@ class MiMoV2MTP(MiMoV2ForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 1fc5fe2f0..15362b00e 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -87,7 +87,7 @@ from sglang.srt.models.nemotron_h_utils import ( pad_to_original_num_tokens, ) from sglang.srt.models.utils import WeightsMapper -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( add_prefix, @@ -919,7 +919,7 @@ class NemotronHForCausalLM(nn.Module): else lora_config.lora_vocab_padding_size ), quant_config=quant_config, - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py index 189e364b9..c8c89e20c 100644 --- a/python/sglang/srt/models/nemotron_h_mtp.py +++ b/python/sglang/srt/models/nemotron_h_mtp.py @@ -39,7 +39,7 @@ from sglang.srt.models.nemotron_h import ( NemotronHMoEDecoderLayer, ) from sglang.srt.models.nemotron_h_utils import is_attn_layer -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix @@ -339,7 +339,7 @@ class NemotronHForCausalLMMTP(NemotronHForCausalLM): self.config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index d5aac381d..123277bdd 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -93,7 +93,7 @@ from sglang.srt.model_executor.cuda_graph_config import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( add_prefix, @@ -148,7 +148,7 @@ def can_fuse_shared_expert( Caller must still gate on the model/backend support flag. """ if ( - get_flags().disable_shared_experts_fusion is True + get_server_args().disable_shared_experts_fusion is True or getattr(config, "shared_expert_intermediate_size", 0) <= 0 or config.shared_expert_intermediate_size != config.moe_intermediate_size or get_moe_a2a_backend().is_deepep() @@ -1003,7 +1003,7 @@ class Qwen2MoeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # For EAGLE3 support diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 2dde70c36..05fb5372a 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -33,7 +33,7 @@ from sglang.srt.model_loader.weight_utils import ( from sglang.srt.models.qwen2 import Qwen2MLP as Qwen3MLP from sglang.srt.models.qwen2 import Qwen2Model from sglang.srt.models.utils import apply_qk_norm -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix, get_bool_env_var, is_cuda, is_hip, is_npu @@ -493,7 +493,7 @@ class Qwen3ForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index fed78cf88..283820957 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -91,7 +91,7 @@ from sglang.srt.models.utils import ( fused_qk_gemma_rmsnorm, fused_qk_gemma_rmsnorm_with_gate, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args # Utils from sglang.srt.utils import ( @@ -133,7 +133,7 @@ cached_get_processor = lru_cache(get_processor) def _disable_shared_experts_fusion() -> bool: # Resolved lazily: the global server args is not set at module import time # (e.g. when this module is imported by unit tests). - return get_flags().disable_shared_experts_fusion + return get_server_args().disable_shared_experts_fusion if _is_cuda: @@ -1177,7 +1177,7 @@ class Qwen3_5ForCausalLM(nn.Module): # so the model still gets the #25885 multi-streaming path. ROCm-only. if ( config.model_type == "qwen3_5_moe_text" - and not get_flags().disable_shared_experts_fusion + and not get_server_args().disable_shared_experts_fusion and not can_fuse_shared_expert(config, quant_config) ): from sglang.srt.arg_groups.overrides import declare_load_time_override diff --git a/python/sglang/srt/models/qwen3_5_mtp.py b/python/sglang/srt/models/qwen3_5_mtp.py index 5cc5b6f98..d08ed72e6 100644 --- a/python/sglang/srt/models/qwen3_5_mtp.py +++ b/python/sglang/srt/models/qwen3_5_mtp.py @@ -34,7 +34,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.qwen3_5 import Qwen3_5ForCausalLM -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix, is_npu @@ -157,7 +157,7 @@ class Qwen3_5ForCausalLMMTP(nn.Module): if ( is_npu() and self.quant_config is None - and get_flags().quantization is not None + and get_server_args().quantization is not None ): # ascend mtp unquant exit_stack.enter_context(envs.SGLANG_DEEPEP_BF16_DISPATCH.override(True)) diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index a7d0ae45e..26c7f4964 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -72,7 +72,7 @@ from sglang.srt.models.utils import ( create_fused_set_kv_buffer_arg, enable_fused_set_kv_buffer, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( LazyValue, @@ -960,7 +960,7 @@ class Qwen3MoeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/qwen3_moe_mtp.py b/python/sglang/srt/models/qwen3_moe_mtp.py index 48d9e4888..6f6ec6091 100644 --- a/python/sglang/srt/models/qwen3_moe_mtp.py +++ b/python/sglang/srt/models/qwen3_moe_mtp.py @@ -30,7 +30,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.qwen3_moe import Qwen3MoeForCausalLM, Qwen3MoeModel -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -63,7 +63,7 @@ class Qwen3MoeForCausalLMMTP(Qwen3MoeForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index e86855cce..233462d3f 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -47,7 +47,7 @@ from sglang.srt.model_loader.weight_utils import ( sharded_weight_loader, ) from sglang.srt.models.qwen2_moe import Qwen2MoeMLP, Qwen2MoeSparseMoeBlock -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import ( LazyValue, add_prefix, @@ -1027,7 +1027,7 @@ class Qwen3NextForCausalLM(nn.Module): quant_config=quant_config, org_num_embeddings=config.vocab_size, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # For EAGLE3 support diff --git a/python/sglang/srt/models/qwen3_next_mtp.py b/python/sglang/srt/models/qwen3_next_mtp.py index bc2525e34..50becff5e 100644 --- a/python/sglang/srt/models/qwen3_next_mtp.py +++ b/python/sglang/srt/models/qwen3_next_mtp.py @@ -32,7 +32,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.qwen3_next import Qwen3NextForCausalLM, Qwen3NextModel -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix, is_npu @@ -84,7 +84,7 @@ class Qwen3NextForCausalLMMTP(Qwen3NextForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # Mirror Qwen3NextForCausalLM.__init__'s shared-expert fusion setup so @@ -114,7 +114,7 @@ class Qwen3NextForCausalLMMTP(Qwen3NextForCausalLM): if ( is_npu() and self.quant_config is None - and get_flags().quantization is not None + and get_server_args().quantization is not None ): # ascend mtp unquant exit_stack.enter_context(envs.SGLANG_DEEPEP_BF16_DISPATCH.override(True)) diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 75c807a5d..b21a63a7d 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -68,7 +68,7 @@ from sglang.srt.models.utils import ( ) from sglang.srt.multimodal.mm_utils import run_dp_sharded_mrope_vision_model from sglang.srt.multimodal.vit_cuda_graph_runner import ViTCudaGraphRunner -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( add_prefix, @@ -1278,7 +1278,7 @@ class Qwen3VLForConditionalGeneration(nn.Module): self.config.vocab_size, self.config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index a56f78438..2eba09d64 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -60,7 +60,7 @@ from sglang.srt.models.bailing_moe import BailingMoEForCausalLM from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mha import ( DeepseekMHAForwardMixin, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import ( BumpAllocator, @@ -1241,7 +1241,7 @@ class SarvamMLAForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 2ac243443..ed52aec69 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -41,7 +41,7 @@ from sglang.srt.models.utils import ( create_fused_set_kv_buffer_arg, enable_fused_set_kv_buffer, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix, is_cuda, make_layers @@ -471,7 +471,7 @@ class SDARForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index 0bb16250a..858991d67 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -57,7 +57,7 @@ from sglang.srt.models.utils import ( create_fused_set_kv_buffer_arg, enable_fused_set_kv_buffer, ) -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_layers @@ -566,7 +566,7 @@ class SDARMoeForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index a2a5a53f3..643f7f0c0 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -46,7 +46,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import get_flags, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.server_args import get_global_server_args from sglang.srt.utils import add_prefix, is_cuda, is_non_idle_and_non_empty, make_layers @@ -826,7 +826,7 @@ class Step3p5ForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_flags().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 6659a2f18..da78c3df0 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -28,13 +28,13 @@ tier). The context owns the storage: publishing goes through ``server_args.py`` are thin shims over this slot), and the object is returned by reference — the same live instance everywhere, never a copy. -``get_flags()`` returns the resolved-flags tier: what the system *resolved* -the configuration to (``server_args`` stays the pristine user input). Flags -live in typed dataclass groups (``flags.attn`` / ``flags.moe`` / flat generic -leaves on ``flags`` itself); reads and writes are plain attribute access. -Static groups are writable during resolution and locked by ``freeze()``; -``flags.capture`` stays writable (capture-time state). Each group offers a -transactional, test-only ``override(**kw)`` that also works on frozen groups. +``get_flags()`` returns the runtime-flags tier. Resolved configuration lives +on ``server_args`` fields (declarations materialize at the end of +``__post_init__``), so this tier only carries genuine runtime state that is +not a function of the configuration alone — today the capture lifecycle +(``flags.capture``). Flags live in typed dataclass groups; reads and writes +are plain attribute access, and each group offers a transactional, test-only +``override(**kw)``. """ from __future__ import annotations @@ -238,18 +238,13 @@ class _FlagGroupBase: f"{type(self).__name__} has no flag '{name}' (leaves are " "declared as dataclass fields; check for typos)" ) - if getattr(self, "_frozen", False): - raise RuntimeError( - f"{type(self).__name__} is frozen; cannot write '{name}'. " - "Test-scoped changes go through override()." - ) object.__setattr__(self, name, value) @contextmanager def override(self, **kwargs): """Temporarily force flag values, restoring on exit. Transactional - (keys validated before any write) and usable on frozen groups — this - is the test-only injection primitive.""" + (keys validated before any write) — the test-only injection + primitive.""" fields = type(self).__dataclass_fields__ unknown = set(kwargs) - set(fields) if unknown: @@ -266,37 +261,6 @@ class _FlagGroupBase: object.__setattr__(self, name, value) -class _StaticFlags(_FlagGroupBase): - """Static flag-group: writable during resolution, locked by ``freeze()``.""" - - def freeze(self) -> None: - object.__setattr__(self, "_frozen", True) - - @property - def frozen(self) -> bool: - return getattr(self, "_frozen", False) - - -@dataclasses.dataclass -class AttnFlags(_StaticFlags): - """Attention-family resolved flags (leaves arrive with the V3 sweeps).""" - - # Resolved attention backend; the pristine user request stays on - # server_args.attention_backend. - backend: str | None = None - prefill_backend: str | None = None - decode_backend: str | None = None - - -@dataclasses.dataclass -class MoeFlags(_StaticFlags): - """MoE-family resolved flags (leaves arrive with the V3 sweeps).""" - - # Resolved MoE runner backend; the pristine user request stays on - # server_args.moe_runner_backend. - runner_backend: str = "auto" - - @dataclasses.dataclass class CaptureFlags(_FlagGroupBase): """Capture-time flags; never frozen (written during cuda-graph capture).""" @@ -307,96 +271,28 @@ class CaptureFlags(_FlagGroupBase): @dataclasses.dataclass -class Flags(_StaticFlags): - """Root of the resolved-flags tier. +class Flags(_FlagGroupBase): + """Root of the runtime-flags tier. - Family groups hang off it (``flags.attn`` / ``flags.moe`` / ``flags.capture``); - single generic flags live flat on this container, declared as fields here. - ``freeze()`` locks the container and every static sub-group; ``capture`` - stays writable. + Resolved configuration lives on ``server_args`` fields (materialized at + the end of ``__post_init__``) — this tier only carries genuine runtime + state whose value is not a function of the configuration alone, grouped + by lifecycle (today: ``capture``). """ - attn: AttnFlags = dataclasses.field(default_factory=AttnFlags) - moe: MoeFlags = dataclasses.field(default_factory=MoeFlags) capture: CaptureFlags = dataclasses.field(default_factory=CaptureFlags) - # -- resolved config leaves (flat; materialized at publish) -------------- - # Pristine user requests stay on the matching server_args fields; these - # leaves carry the model-resolved values. - dtype: str = "auto" - enable_tf32_matmul: bool = False - enable_multi_layer_eagle: bool = False - swa_full_tokens_ratio: float = 0.8 - disable_hybrid_swa_memory: bool = False - sampling_backend: str | None = None - page_size: int | None = None - quantization: str | None = None - fp8_gemm_runner_backend: str = "auto" - disable_overlap_schedule: bool = False - uses_mamba_radix_cache: bool = False - mamba_radix_cache_strategy: str = "auto" - speculative_moe_runner_backend: str | None = None - speculative_moe_a2a_backend: str | None = None - disable_shared_experts_fusion: bool = False - kv_cache_dtype: str = "auto" - dsa_prefill_backend: str | None = None - dsa_decode_backend: str | None = None - flashinfer_allreduce_fusion_backend: str | None = None - # Parallel-request fields: flat transitional home, to be re-homed by the - # Parallel Parameters Clarification module. - enable_dp_attention: bool = False - enable_dp_lm_head: bool = False - moe_a2a_backend: str = "none" - ep_size: int = 1 - moe_dense_tp_size: int | None = None - attn_cp_size: int = 1 - - def freeze(self) -> None: - for field in dataclasses.fields(self): - value = getattr(self, field.name) - if isinstance(value, _StaticFlags): - value.freeze() - super().freeze() - - -# Resolved-config field name → dotted flag-leaf path (e.g. a V3 sweep adds -# "use_mla_backend": "attn.use_mla_backend"). Fields not listed default to a -# flat leaf of the same name on the Flags container. Populated per field -# family as readers migrate. -FLAG_LEAF_MAP: dict[str, str] = { - "attention_backend": "attn.backend", - "prefill_attention_backend": "attn.prefill_backend", - "decode_attention_backend": "attn.decode_backend", - "moe_runner_backend": "moe.runner_backend", -} - - -def resolve_flag_leaf( - flags: Flags, field: str, *, leaf_map: dict[str, str] | None = None -) -> tuple[Any, str]: - """Return ``(owning group, leaf attribute name)`` for a resolved-config field.""" - path = (FLAG_LEAF_MAP if leaf_map is None else leaf_map).get(field, field) - owner: Any = flags - *groups, leaf = path.split(".") - for part in groups: - owner = getattr(owner, part) - return owner, leaf - class RuntimeContext: """Container for the structured runtime accessors; exposes ``parallel``, ``server_args``, and ``flags``.""" - __slots__ = ("parallel", "_server_args", "flags", "_runtime_overrides") + __slots__ = ("parallel", "_server_args", "flags") def __init__(self, parallel: ParallelContext): self.parallel = parallel self._server_args: ServerArgs | None = None self.flags = Flags() - # Post-publish resolution declarations (runner- and load-time - # resolved fields), replayed after the publish-time stash on - # every re-resolve. Cleared on (re-)publish and reset. - self._runtime_overrides: list[tuple[str, dict]] = [] @property def server_args(self) -> ServerArgs: @@ -412,29 +308,10 @@ class RuntimeContext: Overwrite-allowed: a re-publish replaces the slot (test kits re-publish per test; production ordering discipline lives at the call-sites, e.g. - the draft-worker guard in ``ModelRunner.__init__``). - - Publishing also resolves the stashed model-override declarations - into the flags tier (skipped for objects without the stash — dummy / - "none" fixture ServerArgs and test-kit mocks never compute it). - Resolution runs first: if it fails, the previous publish stays intact. - A publish after ``freeze_flags()`` is an ordering violation and raises. + the draft-worker guard in ``ModelRunner.__init__``). The published + object already carries the resolved configuration (declarations + materialize at the end of ``__post_init__``). """ - if self.flags.frozen: - raise RuntimeError( - "set_server_args() after freeze_flags(): the flags tier is " - "frozen for this process; use reset_context() in tests." - ) - # A (re-)publish starts a fresh resolution lifecycle; a failed - # resolve keeps the previous lifecycle (including its recorded - # runtime overrides) intact. - saved_runtime_overrides = self._runtime_overrides - self._runtime_overrides = [] - try: - self._resolve_flags(server_args) - except BaseException: - self._runtime_overrides = saved_runtime_overrides - raise # Seed the capture tier for the new lifecycle (defaults for sentinel # and mock publishes, which carry no config). self.flags.capture.enable_torch_compile = getattr( @@ -442,91 +319,6 @@ class RuntimeContext: ) self._server_args = server_args - def record_runtime_overrides( - self, entries: list[tuple[str, dict]] - ) -> list[tuple[str, dict]]: - """Append post-publish resolution declarations (the runner- and - load-time stages) and - atomically re-resolve the flags tier. - - Target-worker only, and only before ``freeze_flags()``. During the - dual-apply transition the call sites keep their imperative - ``server_args`` writes; the recorded declarations must match them — - parity is re-asserted on every declared field. On failure the - recorded entries are rolled back and the previous flags stay - installed. - """ - server_args = self._server_args - if server_args is None: - raise ValueError("Global server args is not set yet!") - if self.flags.frozen: - raise RuntimeError( - "record_runtime_overrides() after freeze_flags(): runtime " - "resolution stages must complete before the flags tier " - "freezes." - ) - entries = [(source, dict(declared)) for source, declared in entries] - self._runtime_overrides.extend(entries) - try: - self._resolve_flags(server_args) - except BaseException: - del self._runtime_overrides[len(self._runtime_overrides) - len(entries) :] - raise - return entries - - def freeze_flags(self) -> None: - """Lock every static flag group (the resolution end point: after the - load-time stages, before serving). ``flags.capture`` stays writable.""" - self.flags.freeze() - - def _resolve_flags(self, server_args: ServerArgs) -> None: - declarations = getattr(server_args, "_resolved_overrides", None) - if declarations is None and not self._runtime_overrides: - # Stash-less publish. For a config-shaped object (a dataclass: - # mock ServerArgs fixtures, dummy-path instances that skipped the - # monolith) still materialize the whitelist from its own fields, - # so flag reads match legacy server_args reads. Skip only for - # field-less sentinels (tests publishing object()). - if not dataclasses.is_dataclass(server_args): - return - from sglang.srt.arg_groups.arg_utils import resolvable_fields - - if any( - field not in vars(server_args) - for field in resolvable_fields(type(server_args)) - ): - # Bare object.__new__ fixtures: dataclass defaults live on - # the class, not the instance — nothing was populated, so - # treat it as a sentinel (hasattr would see the class - # defaults and materialize them, clobbering resolved flags). - return - declarations = () - declarations = list(declarations or ()) + self._runtime_overrides - from sglang.srt.arg_groups.overrides import ( - apply_model_overrides, - assert_flag_parity, - ) - - # Resolve into a fresh container and only install it once everything - # passed: a failed resolution (gate validation or the parity assert) - # must not leave the process-global flags half-written for callers - # that catch the error or republish (same install-fresh semantics as - # reset_context()). - flags = Flags() - apply_model_overrides(flags, server_args, declarations) - # Transition-period drift guard: dual-apply keeps the declared fields - # on server_args byte-identical to the resolved flag leaves. - assert_flag_parity( - flags, - server_args, - {field for _source, decl in declarations for field in decl}, - ) - # The capture tier is not part of the static resolution: carry it - # across re-resolves so runtime-stage recording cannot clobber a - # capture-time write (set_server_args re-seeds it per lifecycle). - flags.capture = self.flags.capture - self.flags = flags - _PARALLEL = ParallelContext() _CONTEXT = RuntimeContext(parallel=_PARALLEL) @@ -550,10 +342,9 @@ def get_flags() -> Flags: def reset_context() -> None: """Clear the context-owned store (unit-test teardown): drop the published - ``server_args`` and install a fresh, unfrozen ``Flags``. + ``server_args`` and install a fresh ``Flags``. Wrapper subsystems (``parallel``) hold no state and are unaffected. """ _CONTEXT._server_args = None _CONTEXT.flags = Flags() - _CONTEXT._runtime_overrides = [] diff --git a/test/registered/mock_model/test_self_unit_install.py b/test/registered/mock_model/test_self_unit_install.py index 9f32cf0e2..0bcc35feb 100644 --- a/test/registered/mock_model/test_self_unit_install.py +++ b/test/registered/mock_model/test_self_unit_install.py @@ -17,10 +17,6 @@ register_amd_ci(est_time=60, suite="extra-a-test-1-gpu-small-amd") def _make_server_args(*, sampling_backend: str) -> SimpleNamespace: - # The install gate reads the resolved backend from the flags tier. - from sglang.srt.runtime_context import get_flags - - get_flags().sampling_backend = sampling_backend return SimpleNamespace(sampling_backend=sampling_backend) diff --git a/test/registered/ops/test_aiter_greedy_sample_amd.py b/test/registered/ops/test_aiter_greedy_sample_amd.py index 4dd688685..3c17f07f7 100644 --- a/test/registered/ops/test_aiter_greedy_sample_amd.py +++ b/test/registered/ops/test_aiter_greedy_sample_amd.py @@ -23,15 +23,16 @@ register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd") def _mock_global_server_args(backend="pytorch"): from sglang.srt.layers import sampler as sampler_mod - from sglang.srt.runtime_context import get_flags - from sglang.srt.server_args import ServerArgs + from sglang.srt.server_args import ( + ServerArgs, + set_global_server_args_for_scheduler, + ) - sampler_mod.get_global_server_args = lambda: ServerArgs( - model_path="dummy", - sampling_backend=backend, + # Publish for real: the sampler reads the context slot through + # get_server_args(), which a module-attribute rebinding cannot intercept. + set_global_server_args_for_scheduler( + ServerArgs(model_path="dummy", sampling_backend=backend) ) - # The sampler reads the resolved backend from the flags tier. - get_flags().sampling_backend = backend class _DummyTPGroup: device_group = None diff --git a/test/registered/rl/test_fp32_lm_head.py b/test/registered/rl/test_fp32_lm_head.py index cc7fba620..d952974f6 100644 --- a/test/registered/rl/test_fp32_lm_head.py +++ b/test/registered/rl/test_fp32_lm_head.py @@ -7,7 +7,6 @@ import torch.nn as nn import torch.nn.functional as F from sglang.srt.layers.logits_processor import LogitsProcessor -from sglang.srt.runtime_context import get_flags from sglang.srt.server_args import ( ServerArgs, get_global_server_args, @@ -45,7 +44,6 @@ class TestLMHeadFP32(unittest.TestCase): def _make_logprocessor(self, vocab_size, enable_fp32): set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) - get_flags().enable_dp_lm_head = False get_global_server_args().enable_fp32_lm_head = enable_fp32 cfg = SimpleNamespace(vocab_size=vocab_size, final_logit_softcapping=None) return LogitsProcessor(cfg, skip_all_gather=True, logit_scale=None) diff --git a/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py b/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py index 9d9e66474..a3cd88203 100644 --- a/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py +++ b/test/registered/unit/batch_overlap/test_tbo_filter_batch_marker.py @@ -38,11 +38,9 @@ def _make_target_verify_batch(bs: int) -> ForwardBatch: def _filter(batch: ForwardBatch, *, lo: int, hi: int) -> ForwardBatch: fake_args = SimpleNamespace(moe_dense_tp_size=None, attention_backend="fa3") - from sglang.srt.runtime_context import get_flags - with get_parallel().override(attn_tp_size=1), patch.object( tbo, "get_global_server_args", lambda: fake_args - ), get_flags().attn.override(backend="fa3"): + ): return TboForwardBatchPreparer.filter_batch( batch, start_token_index=lo, diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py index 287c4fd54..dbd9c24a4 100644 --- a/test/registered/unit/model_executor/test_pool_configurator.py +++ b/test/registered/unit/model_executor/test_pool_configurator.py @@ -102,10 +102,6 @@ def _make_model_runner( sa = SimpleNamespace() sa.swa_full_tokens_ratio = swa_full_tokens_ratio - # The configurator reads the resolved ratio from the flags tier. - from sglang.srt.runtime_context import get_flags - - get_flags().swa_full_tokens_ratio = swa_full_tokens_ratio sa.page_size = page_size sa.disable_radix_cache = disable_radix_cache sa.chunked_prefill_size = chunked_prefill_size diff --git a/test/registered/unit/models/test_deepseek_v4_shared_expert_fusion.py b/test/registered/unit/models/test_deepseek_v4_shared_expert_fusion.py index 07dcc9fbb..356039064 100644 --- a/test/registered/unit/models/test_deepseek_v4_shared_expert_fusion.py +++ b/test/registered/unit/models/test_deepseek_v4_shared_expert_fusion.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM -from sglang.srt.runtime_context import get_context, get_flags, reset_context +from sglang.srt.runtime_context import get_context, reset_context from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_cpu_ci @@ -10,9 +10,8 @@ register_cpu_ci(est_time=4, suite="base-a-test-cpu") class TestDeepseekV4SharedExpertFusionPolicy(unittest.TestCase): - """The disable decision is a load-time resolution: it lands on the flags - tier through declare_load_time_override (dual-applied onto the published - config during the transition).""" + """The disable decision is a load-time resolution: it writes through to + the published config via declare_load_time_override.""" def setUp(self): self._saved_server_args = get_context()._server_args @@ -41,7 +40,6 @@ class TestDeepseekV4SharedExpertFusionPolicy(unittest.TestCase): DeepseekV4ForCausalLM.determine_num_fused_shared_experts(model) self.assertEqual(model.num_fused_shared_experts, 0) - self.assertTrue(get_flags().disable_shared_experts_fusion) # post-init declaration writes through to the published config self.assertTrue(server_args.disable_shared_experts_fusion) @@ -52,7 +50,6 @@ class TestDeepseekV4SharedExpertFusionPolicy(unittest.TestCase): DeepseekV4ForCausalLM.determine_num_fused_shared_experts(model) self.assertEqual(model.num_fused_shared_experts, 1) - self.assertFalse(get_flags().disable_shared_experts_fusion) self.assertFalse(server_args.disable_shared_experts_fusion) diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index b09173964..3567b5c79 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -18,14 +18,11 @@ from unittest.mock import patch from sglang.srt.arg_groups import overrides as overrides_module from sglang.srt.arg_groups.arg_utils import A, Arg, resolvable_fields from sglang.srt.arg_groups.overrides import ( - OverrideRecord, - apply_model_overrides, collect_model_override_declarations, register_model_override, validate_declarations, ) from sglang.srt.runtime_context import ( - _StaticFlags, get_context, get_server_args, reset_context, @@ -233,89 +230,6 @@ class TestResolvedViewAndPasses(CustomTestCase): ) -@dataclasses.dataclass -class _FakeAttnGroup(_StaticFlags): - backend: str = "unset" - - -@dataclasses.dataclass -class _FakeFlags(_StaticFlags): - attn: _FakeAttnGroup = dataclasses.field(default_factory=_FakeAttnGroup) - resolved_by_model: str = "unset" - also_resolved: Optional[int] = None - - -class TestApplyModelOverridesGate(CustomTestCase): - def _fresh(self): - return _FakeFlags(), _FakeArgs() - - def test_materializes_declared_and_pristine_leaves(self): - flags, args = self._fresh() - records = apply_model_overrides( - flags, args, [("src", {"resolved_by_model": "dsv4"})] - ) - self.assertEqual(flags.resolved_by_model, "dsv4") # declared - self.assertIsNone(flags.also_resolved) # undeclared -> pristine value - self.assertEqual(args.resolved_by_model, "auto") # server_args untouched - self.assertEqual( - records, [OverrideRecord("src", "resolved_by_model", "auto", "dsv4")] - ) - - def test_last_writer_wins_then_terminal_wins_last(self): - flags, args = self._fresh() - records = apply_model_overrides( - flags, - args, - [ - ("first", {"resolved_by_model": "a"}), - ("second", {"resolved_by_model": "b"}), - ], - terminal=[("enforce_disable", {"resolved_by_model": "off"})], - ) - self.assertEqual(flags.resolved_by_model, "off") - self.assertEqual([r.resolved for r in records], ["a", "b", "off"]) - self.assertEqual(records[1].base, "a") # provenance chains the writers - - def test_non_whitelisted_field_rejected_before_any_write(self): - flags, args = self._fresh() - with self.assertRaises(ValueError): - apply_model_overrides( - flags, - args, - [("ok", {"resolved_by_model": "x"}), ("bad", {"plain": 1})], - ) - self.assertEqual(flags.resolved_by_model, "unset") # transactional - - def test_missing_leaf_rejected_before_any_write(self): - flags, args = self._fresh() - with self.assertRaises(ValueError): - apply_model_overrides( - flags, - args, - [("src", {"resolved_by_model": "x"})], - whitelist={"resolved_by_model", "field_without_leaf"}, - ) - self.assertEqual(flags.resolved_by_model, "unset") - - def test_frozen_flags_rejected(self): - flags, args = self._fresh() - flags.freeze() - with self.assertRaises(RuntimeError): - apply_model_overrides(flags, args, [("src", {"resolved_by_model": "x"})]) - - def test_leaf_map_routes_to_group_leaf(self): - flags, args = self._fresh() - apply_model_overrides( - flags, - args, - [("src", {"resolved_by_model": "fa3"})], - whitelist={"resolved_by_model"}, - leaf_map={"resolved_by_model": "attn.backend"}, - ) - self.assertEqual(flags.attn.backend, "fa3") - self.assertEqual(flags.resolved_by_model, "unset") # flat leaf untouched - - class _IsolatedPublish(CustomTestCase): """Publishing writes the process-global context; save/restore around it.""" @@ -335,9 +249,9 @@ class _NoOverridableArgs: x: int = 1 -class TestPublishResolvesFlags(_IsolatedPublish): - """Publish wiring: stash-carrying publishes resolve into flags via the - gate; publishes without the stash skip resolution.""" +class TestPublishInstallsSlot(_IsolatedPublish): + """Publish wiring: set_server_args installs the already-resolved object + into the context-owned slot (no transformation at publish time).""" def test_dummy_fixture_has_empty_stash_and_publishes_cleanly(self): from sglang.srt.server_args import ( @@ -357,23 +271,12 @@ class TestPublishResolvesFlags(_IsolatedPublish): get_context().set_server_args(sa) self.assertIs(get_server_args(), sa) - def test_non_whitelisted_declaration_fails_at_publish(self): - from sglang.srt.runtime_context import get_flags - - flags_before = get_flags() - sa = _NoOverridableArgs() - sa._resolved_overrides = [("rogue", {"x": 2})] - with self.assertRaises(ValueError): - get_context().set_server_args(sa) - # a failed publish must leave BOTH the slot and the flags untouched - self.assertIs(get_flags(), flags_before) - class TestGoldenModelOverrides(_IsolatedPublish): """Per-arch golden diff for migrated families: the declarative path must - reproduce the legacy imperative writes byte-identically on server_args - (dual-apply) and materialize the same values on the flags tier at - publish.""" + reproduce the legacy imperative writes byte-identically on the + materialized server_args fields; the publish round-trip returns the same + object.""" _MINI_CONFIG = { "hidden_size": 64, @@ -408,11 +311,13 @@ class TestGoldenModelOverrides(_IsolatedPublish): return ServerArgs(model_path=config_dir, **server_kwargs) def _publish(self, server_args): - from sglang.srt.runtime_context import get_flags - from sglang.srt.server_args import set_global_server_args_for_scheduler + from sglang.srt.server_args import ( + get_global_server_args, + set_global_server_args_for_scheduler, + ) set_global_server_args_for_scheduler(server_args) - return get_flags() + return get_global_server_args() def test_mistral_large3_forces_bfloat16(self): sa = self._construct("MistralLarge3ForCausalLM", "mistral") @@ -601,7 +506,7 @@ class TestGoldenModelOverrides(_IsolatedPublish): ] self.assertEqual(len(deterministic_fills), 1) self.assertEqual(sa.attention_backend, deterministic_fills[0]) - self.assertEqual(flags.attn.backend, deterministic_fills[0]) + self.assertEqual(flags.attention_backend, deterministic_fills[0]) def test_deterministic_incompatible_backend_raises(self): from sglang.srt.arg_groups.overrides import ( @@ -645,8 +550,8 @@ class TestGoldenModelOverrides(_IsolatedPublish): ("_dllm_attention_backend", {"attention_backend": "flashinfer"}), sa._resolved_overrides, ) - # first MAPPED leaf: attention_backend routes to flags.attn.backend - self.assertEqual(self._publish(sa).attn.backend, "flashinfer") + # the deterministic fill lands on the attention_backend field + self.assertEqual(self._publish(sa).attention_backend, "flashinfer") def test_attention_backend_leaf_materializes_end_state(self): # The default-fill pass declares the platform-selected backend; the @@ -660,7 +565,7 @@ class TestGoldenModelOverrides(_IsolatedPublish): ] self.assertTrue(declared_values) # default fill declared self.assertEqual(sa.attention_backend, declared_values[-1]) # materialized - self.assertEqual(self._publish(sa).attn.backend, declared_values[-1]) + self.assertEqual(self._publish(sa).attention_backend, declared_values[-1]) def test_post_materialize_pass_writes_through(self): from sglang.srt.arg_groups.overrides import run_post_process_pass @@ -679,12 +584,12 @@ class TestGoldenModelOverrides(_IsolatedPublish): run_post_process_pass(sa, _force_triton) if resolved_before != "triton": self.assertEqual(sa.attention_backend, "triton") - self.assertEqual(self._publish(sa).attn.backend, sa.attention_backend) + self.assertEqual(self._publish(sa).attention_backend, sa.attention_backend) def test_attention_backend_user_choice_declares_nothing_extra(self): sa = self._construct("LlamaForCausalLM", "llama", attention_backend="triton") self.assertEqual(sa.attention_backend, "triton") - self.assertEqual(self._publish(sa).attn.backend, "triton") + self.assertEqual(self._publish(sa).attention_backend, "triton") def test_compatibility_passes_at_callable_level(self): from sglang.srt.arg_groups.overrides import ( @@ -2158,13 +2063,10 @@ class TestGoldenModelOverrides(_IsolatedPublish): class TestDeclarationValidation(CustomTestCase): def test_declarations_never_mutate_server_args(self): - flags, args = _FakeFlags(), _FakeArgs() + args = _FakeArgs() declarations = [("src", {"resolved_by_model": "dsv4", "also_resolved": 7})] - apply_model_overrides(flags, args, declarations) validate_declarations(args, declarations) - # the leaves carry the declared values; the fields stay pristine - self.assertEqual(flags.resolved_by_model, "dsv4") - self.assertEqual(flags.also_resolved, 7) + # validation is a pure whitelist check: the fields stay untouched self.assertEqual(args.resolved_by_model, _FakeArgs.resolved_by_model) self.assertEqual(args.also_resolved, _FakeArgs.also_resolved) diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index f1f132fa4..b9c836e21 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -15,13 +15,11 @@ from sglang.srt.runtime_context import ( ParallelContext, RuntimeContext, _FlagGroupBase, - _StaticFlags, get_context, get_flags, get_parallel, get_server_args, reset_context, - resolve_flag_leaf, ) from sglang.test.test_utils import CustomTestCase @@ -215,91 +213,47 @@ class TestServerArgsOwnership(_IsolatedServerArgs): self.assertFalse(hasattr(server_args_module, "_global_server_args")) -@dataclasses.dataclass -class _FakeStaticGroup(_StaticFlags): - alpha: int = 1 - beta: str = "b" - - @dataclasses.dataclass class _FakeCaptureGroup(_FlagGroupBase): gamma: int = 0 class TestFlagsTier(_IsolatedServerArgs): - """V3a skeleton: typed dataclass groups, freeze guard, override primitive.""" + """Runtime-flags tier: typed groups, typo-safe writes, override primitive. + + Resolved configuration lives on server_args fields (materialized at the + end of __post_init__); the flags tier only carries runtime state + (today: the capture lifecycle).""" def test_wiring_and_groups(self): flags = get_flags() self.assertIs(flags, get_context().flags) self.assertIsInstance(flags, Flags) - for group in ("attn", "moe", "capture"): - self.assertTrue(hasattr(flags, group)) - self.assertFalse(flags.frozen) + self.assertTrue(hasattr(flags, "capture")) def test_typo_safety(self): - group = _FakeStaticGroup() + group = _FakeCaptureGroup() with self.assertRaises(AttributeError): - group.alpha_misspelled = 2 # undeclared leaf + group.gamma_misspelled = 2 # undeclared leaf with self.assertRaises(AttributeError): get_flags().not_a_flag = 1 - def test_static_group_writable_until_freeze(self): - group = _FakeStaticGroup() - group.alpha = 5 - self.assertEqual(group.alpha, 5) - group.freeze() - with self.assertRaises(RuntimeError): - group.alpha = 6 - self.assertEqual(group.alpha, 5) - - def test_override_is_transactional_and_works_on_frozen(self): - group = _FakeStaticGroup() - group.freeze() - with group.override(alpha=99, beta="x"): - self.assertEqual(group.alpha, 99) - self.assertEqual(group.beta, "x") - self.assertEqual(group.alpha, 1) - self.assertEqual(group.beta, "b") - with self.assertRaises(ValueError): - with group.override(alpha=2, gamma=3): # gamma undeclared - pass - self.assertEqual(group.alpha, 1) # validated before any write - - def test_non_static_group_has_no_freeze(self): + def test_override_is_transactional(self): group = _FakeCaptureGroup() - group.gamma = 42 - self.assertEqual(group.gamma, 42) - self.assertFalse(hasattr(group, "freeze")) + with group.override(gamma=99): + self.assertEqual(group.gamma, 99) + self.assertEqual(group.gamma, 0) + with self.assertRaises(ValueError): + with group.override(gamma=2, delta=3): # delta undeclared + pass + self.assertEqual(group.gamma, 0) # validated before any write - def test_container_freeze_cascades_except_capture(self): - flags = Flags() # fresh container, not the process singleton - flags.freeze() - self.assertTrue(flags.frozen) - self.assertTrue(flags.attn.frozen) - self.assertTrue(flags.moe.frozen) - with self.assertRaises(RuntimeError): - flags.attn = flags.attn # container leaves lock too - self.assertFalse(getattr(flags.capture, "_frozen", False)) - - def test_resolve_flag_leaf_flat_default_and_mapped(self): - flags = Flags() - owner, leaf = resolve_flag_leaf(flags, "some_field") - self.assertIs(owner, flags) - self.assertEqual(leaf, "some_field") - owner, leaf = resolve_flag_leaf(flags, "x", leaf_map={"x": "attn.x"}) - self.assertIs(owner, flags.attn) - self.assertEqual(leaf, "x") - - def test_reset_context_installs_fresh_unfrozen_flags(self): - try: - old = get_flags() - old.freeze() - reset_context() - self.assertIsNot(get_flags(), old) - self.assertFalse(get_flags().frozen) - finally: - reset_context() # never leave the singleton frozen for other tests + def test_reset_context_installs_fresh_flags(self): + old = get_flags() + old.capture.enable_torch_compile = True + reset_context() + self.assertIsNot(get_flags(), old) + self.assertFalse(get_flags().capture.enable_torch_compile) @dataclasses.dataclass @@ -311,84 +265,15 @@ class _FakeResolvedArgs: _resolved_overrides: list = dataclasses.field(default_factory=list) -class TestRuntimeResolutionStages(_IsolatedServerArgs): - """Runtime stages: post-publish declarations re-resolve the flags tier - atomically; freeze_flags() ends the resolution lifecycle.""" +class TestPublishLifecycle(_IsolatedServerArgs): + """Publish installs the resolved server_args and seeds the capture tier.""" def _publish(self, **kw): args = _FakeResolvedArgs(**kw) get_context().set_server_args(args) return args - def test_record_before_publish_raises(self): - reset_context() - with self.assertRaises(ValueError): - get_context().record_runtime_overrides([("stage", {"page_size": 64})]) - - def test_record_updates_leaves_and_accumulates_stages(self): - args = self._publish(page_size=1, sampling_backend="flashinfer") - self.assertEqual(get_flags().page_size, 1) # publish-time materialize - # dual-apply transition: the call site keeps its imperative write - args.page_size = 64 - get_context().record_runtime_overrides([("stage.runner", {"page_size": 64})]) - self.assertEqual(get_flags().page_size, 64) - args.sampling_backend = "pytorch" - get_context().record_runtime_overrides( - [("stage.load", {"sampling_backend": "pytorch"})] - ) - self.assertEqual(get_flags().sampling_backend, "pytorch") - self.assertEqual(get_flags().page_size, 64) # earlier stage survives - - def test_record_whitelist_violation_rolls_back(self): - self._publish() - with self.assertRaises(ValueError): - get_context().record_runtime_overrides([("bad", {"nope": 1})]) - self.assertEqual(get_context()._runtime_overrides, []) - - def test_freeze_ends_the_resolution_lifecycle(self): - args = self._publish(page_size=1) - try: - get_context().freeze_flags() - self.assertTrue(get_flags().frozen) - with self.assertRaises(RuntimeError): - get_context().record_runtime_overrides([("late", {"page_size": 64})]) - with self.assertRaises(RuntimeError): - get_context().set_server_args(args) - finally: - reset_context() - - def test_declare_load_time_override_applies_and_records(self): - from sglang.srt.arg_groups.overrides import declare_load_time_override - - args = self._publish(page_size=1) - declare_load_time_override("model.load_time", {"page_size": 64}) - # post-init declaration: written through to the field and resolved - # into the leaf - self.assertEqual(args.page_size, 64) - self.assertEqual(get_flags().page_size, 64) - self.assertEqual( - get_context()._runtime_overrides, - [("model.load_time", {"page_size": 64})], - ) - - def test_failed_republish_keeps_previous_lifecycle(self): - args = self._publish(page_size=1) - args.page_size = 64 - get_context().record_runtime_overrides([("stage", {"page_size": 64})]) - flags_before = get_flags() - bad = _FakeResolvedArgs(page_size=1) - bad._resolved_overrides = [("bad", {"nope": 1})] # gate rejects - with self.assertRaises(ValueError): - get_context().set_server_args(bad) - # previous publish fully intact: slot, flags, and the recorded stages - self.assertIs(get_context()._server_args, args) - self.assertIs(get_flags(), flags_before) - self.assertEqual( - get_context()._runtime_overrides, [("stage", {"page_size": 64})] - ) - - def test_capture_tier_seeded_at_publish_and_survives_stages(self): - # seeded from the published config + def test_capture_tier_seeded_at_publish(self): args = self._publish(page_size=1) args.enable_torch_compile = True get_context().set_server_args(args) # re-publish picks up the value @@ -396,59 +281,38 @@ class TestRuntimeResolutionStages(_IsolatedServerArgs): # capture-time write (B4) targets the capture leaf get_flags().capture.enable_torch_compile = False self.assertFalse(get_flags().capture.enable_torch_compile) - # a runtime-stage re-resolve must not clobber the capture write - args.page_size = 64 - get_context().record_runtime_overrides([("stage", {"page_size": 64})]) - self.assertFalse(get_flags().capture.enable_torch_compile) - # capture stays writable after freeze - try: - get_context().freeze_flags() - get_flags().capture.enable_torch_compile = True - self.assertTrue(get_flags().capture.enable_torch_compile) - finally: - reset_context() - - def test_declared_leaf_wins_over_stale_field(self): - # A stash entry always drives the leaf at publish, even if the field - # value diverged (e.g. a fixture that skipped materialization). - @dataclasses.dataclass - class _Args: - enable_dp_lm_head: A[bool, Arg(help="d", resolvable=True)] = True - _resolved_overrides: list = dataclasses.field(default_factory=list) - - args = _Args() - args._resolved_overrides = [("dp", {"enable_dp_lm_head": False})] - args._declarations_materialized = True - args.enable_dp_lm_head = False - get_context().set_server_args(args) - self.assertFalse(get_flags().enable_dp_lm_head) - - def test_bare_dataclass_publish_skips_materialization(self): - # object.__new__(ServerArgs) fixtures (no __init__, no field values) - # must publish without touching the flags tier — dataclass defaults - # live on the class, so materializing from them would clobber - # previously resolved flags with defaults. - from sglang.srt.server_args import ServerArgs - - self._publish(page_size=64) - self.assertEqual(get_flags().page_size, 64) - bare = object.__new__(ServerArgs) - get_context().set_server_args(bare) - self.assertIs(get_server_args(), bare) - self.assertEqual(get_flags().page_size, 64) # not clobbered def test_capture_tier_defaults_for_sentinel_publish(self): get_context().set_server_args(object()) self.assertFalse(get_flags().capture.enable_torch_compile) - def test_republish_clears_runtime_overrides(self): + def test_declare_load_time_override_writes_through(self): + from sglang.srt.arg_groups.overrides import declare_load_time_override + args = self._publish(page_size=1) - args.page_size = 64 - get_context().record_runtime_overrides([("stage", {"page_size": 64})]) - self.assertEqual(get_flags().page_size, 64) - self._publish(page_size=1) # fresh lifecycle - self.assertEqual(get_flags().page_size, 1) - self.assertEqual(get_context()._runtime_overrides, []) + declare_load_time_override("model.load_time", {"page_size": 64}) + self.assertEqual(args.page_size, 64) + + def test_declare_load_time_override_validates_whitelist(self): + from sglang.srt.arg_groups.overrides import declare_load_time_override + + args = self._publish(page_size=1) + with self.assertRaises(ValueError): + declare_load_time_override("bad", {"nope": 1}) + self.assertEqual(args.page_size, 1) + + def test_declare_load_time_override_records_provenance(self): + from sglang.srt.arg_groups.overrides import declare_load_time_override + from sglang.srt.server_args import ServerArgs + + class _Args(_FakeResolvedArgs): + override = ServerArgs.override + + args = _Args(page_size=1) + get_context().set_server_args(args) + declare_load_time_override("model.load_time", {"page_size": 64}) + self.assertEqual(args.page_size, 64) + self.assertIn(("model.load_time", {"page_size": 64}), args._resolved_overrides) if __name__ == "__main__": diff --git a/test/registered/unit/test_server_args_mutation_ratchet.py b/test/registered/unit/test_server_args_mutation_ratchet.py index 90d27a001..c8711a632 100644 --- a/test/registered/unit/test_server_args_mutation_ratchet.py +++ b/test/registered/unit/test_server_args_mutation_ratchet.py @@ -68,8 +68,8 @@ class TestServerArgsMutationRatchet(CustomTestCase): f"server_args mutations outside the resolution pipeline grew: " f"{count} > baseline {_BASELINE}. Configuration is resolved in " "ServerArgs.__post_init__; declare through the pipeline " - "(passes / declare_load_time_override / " - "record_runtime_overrides) instead of assigning fields." + "(passes / declare_load_time_override) or go through " + "ServerArgs.override(source, ...) instead of assigning fields." ) if count < _BASELINE: self.fail(