config: route runtime config adjustments through the namespace bags (#31812)

python/sglang/srt/disaggregation/common/conn.py

@@ -36,7 +36,7 @@ from sglang.srt.layers.dp_attention import (
     get_attention_dp_rank,
     get_attention_dp_size,
 )
-from sglang.srt.runtime_context import get_parallel
+from sglang.srt.runtime_context import get_model, get_parallel
 from sglang.srt.server_args import ServerArgs
 from sglang.srt.utils.network import (
     NetworkAddress,
@@ -469,11 +469,11 @@ class CommonKVManager(BaseKVManager):
 
         if (
             info.kv_cache_dtype is not None
-            and info.kv_cache_dtype != self.server_args.kv_cache_dtype
+            and info.kv_cache_dtype != get_model().kv_cache_dtype
         ):
             raise RuntimeError(
                 f"KV cache dtype mismatch: prefill server has kv_cache_dtype={info.kv_cache_dtype}, "
-                f"but decode server has kv_cache_dtype={self.server_args.kv_cache_dtype}. "
+                f"but decode server has kv_cache_dtype={get_model().kv_cache_dtype}. "
                 f"Both servers must use the same --kv-cache-dtype value."
             )
 
@@ -626,7 +626,7 @@ class CommonKVManager(BaseKVManager):
             "rank_ip": self.local_ip,
             "rank_port": self.rank_port,
             "page_size": self.kv_args.page_size,
-            "kv_cache_dtype": self.server_args.kv_cache_dtype,
+            "kv_cache_dtype": get_model().kv_cache_dtype,
             "load_balance_method": self.server_args.load_balance_method,
             "enable_dsa_cache_layer_split": getattr(
                 self.server_args, "enable_dsa_cache_layer_split", False

python/sglang/srt/layers/attention/dual_chunk_flashattention_backend.py

@@ -125,7 +125,12 @@ class DualChunkFlashAttentionBackend(AttentionBackend):
         self.token_to_kv_pool = model_runner.token_to_kv_pool
         self.req_to_token = model_runner.req_to_token_pool.req_to_token
         self.kv_cache_dtype = model_runner.kv_cache_dtype
-        self.kv_cache_dtype_str = model_runner.server_args.kv_cache_dtype
+
+        self.kv_cache_dtype_str = getattr(
+            model_runner,
+            "kv_cache_dtype_str",
+            model_runner.server_args.kv_cache_dtype,
+        )
         self.page_size = model_runner.page_size
 
         assert self.num_heads % self.num_kv_heads == 0

python/sglang/srt/layers/attention/flashattention_backend.py

@@ -166,7 +166,9 @@ class FlashAttentionBackend(AttentionBackend):
         self.token_to_kv_pool = model_runner.token_to_kv_pool
         self.req_to_token = model_runner.req_to_token_pool.req_to_token
         self.kv_cache_dtype = model_runner.kv_cache_dtype
-        self.kv_cache_dtype_str = model_runner.server_args.kv_cache_dtype
+        from sglang.srt.runtime_context import get_model
+
+        self.kv_cache_dtype_str = get_model().kv_cache_dtype
         self.kv_cache_is_mxfp8 = self.kv_cache_dtype_str == "mxfp8"
         self.page_size = model_runner.page_size
         # Static page-table width (upper bound). The device-side page-table build

python/sglang/srt/layers/attention/linear/lightning_backend.py

@@ -64,7 +64,12 @@ class LightningAttentionBackend(MambaAttnBackendBase):
         self.device = model_runner.device
         self.decode_cuda_graph_metadata = {}
         self.kv_cache_dtype = model_runner.kv_cache_dtype
-        self.kv_cache_dtype_str = model_runner.server_args.kv_cache_dtype
+
+        self.kv_cache_dtype_str = getattr(
+            model_runner,
+            "kv_cache_dtype_str",
+            model_runner.server_args.kv_cache_dtype,
+        )
         self.BLOCK = (
             model_runner.model_config.block
             if hasattr(model_runner.model_config, "block")

python/sglang/srt/layers/attention/xpu_backend.py

@@ -69,7 +69,9 @@ class XPUAttentionBackend(AttentionBackend):
         self.token_to_kv_pool = model_runner.token_to_kv_pool
         self.req_to_token = model_runner.req_to_token_pool.req_to_token
         self.kv_cache_dtype = model_runner.kv_cache_dtype
-        self.kv_cache_dtype_str = model_runner.server_args.kv_cache_dtype
+        from sglang.srt.runtime_context import get_model
+
+        self.kv_cache_dtype_str = get_model().kv_cache_dtype
         self.page_size = model_runner.page_size
         self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA
         self.skip_prefill = skip_prefill

python/sglang/srt/managers/scheduler.py

@@ -241,7 +241,7 @@ from sglang.srt.observability.trace import process_tracing_init, trace_set_threa
 from sglang.srt.parser.reasoning_parser import ReasoningParser
 from sglang.srt.platforms import current_platform
 from sglang.srt.plugins import load_plugins
-from sglang.srt.runtime_context import get_parallel, get_server_args
+from sglang.srt.runtime_context import get_context, get_parallel
 from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
 from sglang.srt.sampling.sampling_params import TOP_K_ALL
 from sglang.srt.server_args import PortArgs, ServerArgs
@@ -786,6 +786,11 @@ class Scheduler(
         )
 
         if self.server_args.speculative_draft_load_format is not None:
+            # Write the draft load_format onto server_args (not just the bag):
+            # the draft worker is built from a copy of self.server_args and
+            # build_load_config reads server_args.load_format, so a bag-only
+            # override would be ignored and the draft would load in the target's
+            # format.
             self.server_args.override(
                 "scheduler.draft_load_format",
                 load_format=self.server_args.speculative_draft_load_format,
@@ -890,8 +895,8 @@ class Scheduler(
             self.min_free_slots_delayer = MinFreeSlotsDelayer(
                 min_free_slots=min_free_slots
             )
-        if not get_server_args().pp_max_micro_batch_size:
-            get_server_args().override(
+        if not get_parallel().pp_max_micro_batch_size:
+            get_context().override(
                 "scheduler.pp_max_micro_batch_size_default",
                 pp_max_micro_batch_size=max(
                     self.max_running_requests // self.ps.pp_size, 1
@@ -2837,7 +2842,7 @@ class Scheduler(
         return NextBatchPlan(batch_to_run=ret, running_batch=running_batch)
 
     def get_num_allocatable_reqs(self, running_bs):
-        res = get_server_args().pp_max_micro_batch_size - running_bs
+        res = get_parallel().pp_max_micro_batch_size - running_bs
         res = min(res, self.req_to_token_pool.available_size())
         return res
 
@@ -3893,7 +3898,9 @@ class Scheduler(
         return success
 
     def get_internal_state(self, recv_req: GetInternalStateReq):
-        ret = dict(vars(get_server_args()))  # vars returns a ref to obj.__dict__
+        # Resolved config (pristine server_args + post-publish overrides) so a
+        # readback reflects values changed via /set_internal_state, not startup.
+        ret = get_context().resolved_server_args_dict()
         ret["last_gen_throughput"] = self.metrics_reporter.last_gen_throughput
         ret["memory_usage"] = {
             "weight": round(self.tp_worker.model_runner.weight_load_mem_usage, 2),
@@ -4013,8 +4020,8 @@ class Scheduler(
             if remaining.pop("dspark_clear_info_records", None):
                 self.draft_worker.clear_info_records()
             if remaining:
-                get_server_args().override(source="update_server_args", **remaining)
-            logger.info(f"Global server args updated! {get_server_args()=}")
+                get_context().override(source="update_server_args", **remaining)
+            logger.info(f"Config updated via context override: {remaining}")
 
         return SetInternalStateReqOutput(updated=if_success)
 

python/sglang/srt/mem_cache/kv_cache_configurator.py

@@ -58,7 +58,7 @@ from sglang.srt.mem_cache.memory_pool import (
 )
 from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
 from sglang.srt.platforms import current_platform
-from sglang.srt.runtime_context import get_parallel
+from sglang.srt.runtime_context import get_model, get_parallel
 from sglang.srt.server_args import ServerArgs
 from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
 from sglang.srt.utils.common import (
@@ -205,7 +205,7 @@ class KVCacheConfigurator:
     def _build_fp4_quant_method(self, *, num_layers: int):
         if not is_float4_e2m1fn_x2(self.kv_cache_dtype):
             return None
-        quant_name = resolve_kv_cache_quant(self.server_args.kv_cache_dtype)
+        quant_name = resolve_kv_cache_quant(get_model().kv_cache_dtype)
         if quant_name is None:
             return None
         quant_method = get_kv_cache_quant_method(
@@ -1191,7 +1191,7 @@ class KVCacheConfigurator:
             }
         swa_pool_class = (
             MHATokenToKVPoolMXFP8
-            if self.server_args.kv_cache_dtype == "mxfp8"
+            if get_model().kv_cache_dtype == "mxfp8"
             else mha_pool_class
         )
         swa_attention_layer_ids = self.model_config.swa_attention_layer_ids
@@ -1289,7 +1289,7 @@ class KVCacheConfigurator:
         # buffers) for the full-attention layers, same as the SWA branch.
         full_pool_class = (
             MHATokenToKVPoolMXFP8
-            if self.server_args.kv_cache_dtype == "mxfp8" and not self.use_mla_backend
+            if get_model().kv_cache_dtype == "mxfp8" and not self.use_mla_backend
             else mha_pool_class
         )
         token_to_kv_pool = HybridLinearKVPool(
@@ -1334,7 +1334,7 @@ class KVCacheConfigurator:
     def _build_mha_kv_pool(
         self, *, max_total_num_tokens: int, mha_pool_class: type, quant_method=None
     ) -> KVCache:
-        if self.server_args.kv_cache_dtype == "mxfp8":
+        if get_model().kv_cache_dtype == "mxfp8":
             pool_cls = MHATokenToKVPoolMXFP8
         else:
             pool_cls = (

python/sglang/srt/model_executor/model_runner.py

@@ -1093,18 +1093,16 @@ class ModelRunner:
             return self.max_total_num_tokens
 
     def _record_kv_cache_dtype(self, resolved: str) -> None:
-        # Load-time resolution transition: the weight-resolved kv-cache dtype
-        # is declared into the flags tier; the dual-apply inside the helper
-        # replaces the legacy in-place write. Mock runners whose server_args
-        # is not the published object keep the plain write.
+        # the weight-resolved kv-cache dtype is written to the config
+        # bags via get_context().override, so get_model().kv_cache_dtype readers
+        # see it. server_args stays the pristine RAW record -- configure_kv_cache
+        # _dtype reads it as the resolver INPUT. A draft / mock runner whose
+        # server_args is not the published object keeps the private-bag write.
         from sglang.srt.runtime_context import get_context
 
         if get_context()._server_args is self.server_args:
-            from sglang.srt.arg_groups.overrides import declare_load_time_override
-
-            declare_load_time_override(
-                "ModelRunner.configure_kv_cache_dtype",
-                {"kv_cache_dtype": resolved},
+            get_context().override(
+                "ModelRunner.configure_kv_cache_dtype", kv_cache_dtype=resolved
             )
         else:
             self.server_args.override(
@@ -1115,6 +1113,8 @@ class ModelRunner:
         spec_algorithm = getattr(self, "spec_algorithm", None)
         resolved_kv_cache_dtype, self.kv_cache_dtype = (
             kv_cache_dtype.configure_kv_cache_dtype(
+                # RAW user intent = resolver INPUT; server_args stays pristine
+                # so read it here -- not the resolved get_model() bag.
                 server_args_kv_cache_dtype=self.server_args.kv_cache_dtype,
                 model=getattr(self, "model", None),
                 model_dtype=getattr(self, "dtype", torch.bfloat16),
@@ -1127,6 +1127,15 @@ class ModelRunner:
                 ),
             )
         )
+        # This runner's OWN resolved dtype string (target or draft). Attention
+        # backends read it directly instead of the process-global get_model()
+        # bag: a draft runner does not publish its args, so the bag would carry
+        # the target's dtype and mis-drive the draft's FP8 cast/descale paths.
+        self.kv_cache_dtype_str = (
+            resolved_kv_cache_dtype
+            if resolved_kv_cache_dtype is not None
+            else self.server_args.kv_cache_dtype
+        )
         if resolved_kv_cache_dtype is not None:
             self._record_kv_cache_dtype(resolved_kv_cache_dtype)
 

python/sglang/srt/model_executor/pool_configurator.py

@@ -33,7 +33,7 @@ from sglang.srt.environ import envs
 from sglang.srt.mem_cache.allocation_sizing import get_alloc_len_per_decode
 from sglang.srt.mem_cache.deepseek_v4_memory_pool import get_compress_state_ring_size
 from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
-from sglang.srt.runtime_context import get_parallel
+from sglang.srt.runtime_context import get_model, get_parallel
 from sglang.srt.utils.common import (
     ceil_align,
     ceil_div,
@@ -275,7 +275,7 @@ class DefaultPoolConfigurator(MemoryPoolConfigurator):
                 )
                 # FP4 prefill uses one shared FP8 dequant workspace across layers.
                 cell_size += n * k * 2 * kv_size
-            elif kvc.server_args.kv_cache_dtype == "mxfp8":
+            elif get_model().kv_cache_dtype == "mxfp8":
                 scale_block_size = 32
                 n = model_config.get_num_kv_heads(tp_size)
                 cell_size += (
@@ -335,7 +335,7 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
             * kv_size
         )
 
-        if kvc.server_args.kv_cache_dtype == "mxfp8":
+        if get_model().kv_cache_dtype == "mxfp8":
             scale_block_size = 32
             self._full_per_token += (
                 model_config.get_num_kv_heads(tp_size)

python/sglang/srt/models/deepseek_v2.py

@@ -185,6 +185,7 @@ from sglang.srt.models.deepseek_common.utils import (
 from sglang.srt.runtime_context import (
     get_flags,
     get_forward,
+    get_model,
     get_parallel,
     get_server_args,
 )
@@ -1605,7 +1606,7 @@ class DeepseekV2AttentionMLA(
         self.scaling = self.qk_head_dim**-0.5
         self.rope_theta = rope_theta
         self.max_position_embeddings = max_position_embeddings
-        self.kv_cache_dtype = get_server_args().kv_cache_dtype
+        self.kv_cache_dtype = get_model().kv_cache_dtype
 
         # NOTE modification to rope_scaling must be done early enough, b/c e.g. Indexer needs it
         if rope_scaling:

python/sglang/srt/models/inkling.py

@@ -72,7 +72,7 @@ from sglang.srt.models.inkling_common.util import (
     trtllm_bf16_weight_prep_enabled,
     use_inkling_shared_fused_moe,
 )
-from sglang.srt.runtime_context import get_parallel, get_server_args
+from sglang.srt.runtime_context import get_model, get_parallel, get_server_args
 from sglang.srt.utils import add_prefix, is_cuda, make_layers
 
 logger = logging.getLogger(__name__)
@@ -649,7 +649,7 @@ class InklingCausalLLM(nn.Module):
             )
 
             warmed: set = set()
-            warm_mxfp8 = get_server_args().kv_cache_dtype == "mxfp8"
+            warm_mxfp8 = get_model().kv_cache_dtype == "mxfp8"
             for layer in self.layers:
                 attn = layer.attn
                 ks = attn.k_sconv

python/sglang/srt/models/sarvam_moe.py

@@ -62,6 +62,7 @@ from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mha imp
 )
 from sglang.srt.runtime_context import (
     get_forward,
+    get_model,
     get_parallel,
     get_server_args,
     get_stream,
@@ -450,7 +451,7 @@ class SarvamMoEMLAAttention(nn.Module):
         self.scaling = self.qk_head_dim**-0.5
         self.rope_theta = rope_theta
         self.max_position_embeddings = max_position_embeddings
-        self.kv_cache_dtype = get_server_args().kv_cache_dtype
+        self.kv_cache_dtype = get_model().kv_cache_dtype
 
         self._server_args = None
         self.current_attention_backend = None

python/sglang/srt/runtime_context.py

@@ -107,12 +107,37 @@ _PARALLEL_FIELDS = frozenset(
 
 
 class ParallelContext:
-    """Parallel-topology namespace; the only instance state is ``_overrides``."""
+    """Parallel-topology namespace.
 
-    __slots__ = ("_overrides",)
+    Live topology (size / rank / group) is read-through via ``@property`` (the
+    canonical getters). Parallel **config** leaves (``nccl_port``,
+    ``pp_max_micro_batch_size``, ``enable_dp_attention``, …) come from the
+    published ``parallel`` config bag via ``__getattr__``. Where a config leaf
+    shares a name with a live property (``tp_size`` …), the property (the live
+    fact) wins; the same-name==same-value invariant holds once dist is up.
+    """
+
+    __slots__ = ("_overrides", "_config")
 
     def __init__(self):
         self._overrides = {}
+        self._config = None  # parallel config bag, wired at publish
+
+    def __getattr__(self, name):
+        # Reached only for names that are neither a live @property nor a slot:
+        # serve parallel config leaves from the published bag.
+        try:
+            config = object.__getattribute__(self, "_config")
+        except AttributeError:
+            config = None
+        if config is not None and name in config:
+            return getattr(config, name)
+        detail = (
+            "not a published parallel config leaf"
+            if config is not None
+            else "config not published"
+        )
+        raise AttributeError(f"ParallelContext has no {name!r} ({detail})")
 
     def _v(self, name, getter):
         overrides = self._overrides
@@ -727,6 +752,10 @@ class RuntimeContext:
         # truth for config reads). Driven by NS(...) metadata; a mock/partial
         # config with no NS markers yields an empty tree (no bags projected).
         self._config_bags = _build_config_bags(server_args)
+        # Wire the parallel config leaves onto the live wrapper (config-only
+        # leaves like pp_max_micro_batch_size are served via ParallelContext
+        # __getattr__; live topology properties still win by name).
+        self.parallel._config = self._config_bags.get("parallel")
         # Fresh config lifecycle: prior override provenance no longer applies.
         self._overrides_log = []
 
@@ -789,6 +818,27 @@ class RuntimeContext:
         """Provenance of post-publish ``override`` calls: ``[(source, {field: value})]``."""
         return list(self._overrides_log)
 
+    def resolved_server_args_dict(self, base: dict | None = None) -> dict:
+        """Serialize the *resolved* config: the pristine ``server_args`` fields
+        with every post-publish ``override`` overlaid.
+
+        Reporting endpoints (``/server_info``, ``get_internal_state``) surface
+        the config the process is *currently* running, not the startup record,
+        so they read this rather than serializing ``server_args`` directly —
+        otherwise runtime updates (weight version, model path, tunables set via
+        ``/set_internal_state``) never show up in the readback.
+
+        ``base`` defaults to ``dict(vars(server_args))`` (matching the legacy
+        ``vars`` dump); pass ``dataclasses.asdict(server_args)`` when nested
+        dataclass fields must be expanded first (``/server_info``). Override
+        leaves are flat ``ServerArgs`` field names, so overlaying them onto the
+        top level of either base is exact.
+        """
+        d = dict(vars(self.server_args)) if base is None else dict(base)
+        for _source, fields in self._overrides_log:
+            d.update(fields)
+        return d
+
     def override_server_args(self, **fields) -> _ServerArgsOverride:
         """Test-only scoped override for the config tier — the sibling of
         ``get_parallel().override()`` and the flag groups' ``override()``:
@@ -981,6 +1031,7 @@ def reset_context() -> None:
     _CONTEXT._server_args = None
     _CONTEXT._config_bags = None
     _CONTEXT._overrides_log = []
+    _CONTEXT.parallel._config = None
     _CONTEXT.flags = Flags()
     _CONTEXT.resources = Resources()
     _CONTEXT.forward = ForwardFlags()

python/sglang/srt/speculative/dflash_utils.py

@@ -634,13 +634,13 @@ def compute_dflash_sampling_correct_drafts_and_bonus(
         )
 
     if threshold_single is None:
-        from sglang.srt.runtime_context import get_server_args
+        from sglang.srt.runtime_context import get_spec
 
-        threshold_single = get_server_args().speculative_accept_threshold_single
+        threshold_single = get_spec().speculative_accept_threshold_single
     if threshold_acc is None:
-        from sglang.srt.runtime_context import get_server_args
+        from sglang.srt.runtime_context import get_spec
 
-        threshold_acc = get_server_args().speculative_accept_threshold_acc
+        threshold_acc = get_spec().speculative_accept_threshold_acc
     threshold_single = float(threshold_single)
     threshold_acc = max(float(threshold_acc), 1e-9)
 

python/sglang/srt/speculative/eagle_utils.py

@@ -16,7 +16,7 @@ from sglang.srt.hardware_backend.npu.dsv4.dsv4_common_hooks import (
 )
 from sglang.srt.mem_cache.allocation import alloc_for_spec_decode
 from sglang.srt.mem_cache.allocation_sizing import get_alloc_reserve_per_decode
-from sglang.srt.runtime_context import get_parallel
+from sglang.srt.runtime_context import get_parallel, get_spec
 from sglang.srt.utils import (
     is_cpu,
     is_cuda,
@@ -740,8 +740,8 @@ def eagle_sample(
             uniform_samples_for_final_sampling=coins_for_final_sampling,
             target_probs=target_probs,
             draft_probs=draft_probs,
-            threshold_single=get_server_args().speculative_accept_threshold_single,
-            threshold_acc=get_server_args().speculative_accept_threshold_acc,
+            threshold_single=get_spec().speculative_accept_threshold_single,
+            threshold_acc=get_spec().speculative_accept_threshold_acc,
             deterministic=True,
         )