# Copyright 2023-2026 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Process-wide configuration, parallel state, flags, and resources. ``get_server_args()`` retains the raw input for diagnostics. Namespace accessors such as ``get_exec()`` return resolved configuration published from declarations; ``RuntimeContext.override`` changes those bags. ``get_parallel()`` also exposes ranks and process groups, with scoped and permanent overrides taking precedence over published values. ``get_flags()`` holds mutable runtime state; ``get_resources()`` owns process handles, and ``get_forward()`` holds per-forward state. """ from __future__ import annotations import functools import logging import math import os import sys from contextlib import contextmanager from enum import Enum, auto from typing import TYPE_CHECKING, Any, Dict, Optional import msgspec from sglang.srt.arg_groups.prefill_buffer_ceiling import prefill_buffer_ceiling_of if TYPE_CHECKING: from sglang.srt.model_executor.runner_utils.pool import GraphPoolBorrowState from sglang.srt.server_args import ServerArgs logger = logging.getLogger(__name__) @functools.lru_cache(maxsize=1) def _parallel_config_leaves() -> frozenset: """Return configured parallel field names, including before publication.""" from sglang.srt.arg_groups.arg_utils import namespace_of from sglang.srt.server_args import ServerArgs return frozenset( field for field, path in namespace_of(ServerArgs).items() if path.split(".")[0] == "parallel" ) @functools.lru_cache(maxsize=1) def _parallel_fields() -> frozenset: """Return configured and derived parallel names accepted by overrides.""" from sglang.srt.arg_groups.arg_utils import Derived from sglang.srt.arg_groups.fields.parallel import Parallel derived = { name for name, decl in vars(Parallel).items() if isinstance(decl, Derived) } return frozenset(_parallel_config_leaves() | derived) def derive_attn_tp_size(*, tp_size: int, attn_cp_size: int, attn_dp_size: int) -> int: """Return the attention-TP width: what is left of the TP group after the attention-DP and attention-CP dimensions.""" return tp_size // attn_dp_size // attn_cp_size def derive_attention_ranks( *, tp_rank: int, attn_tp_size: int, attn_cp_size: int ) -> tuple: """Return (attn_tp_rank, attn_dp_rank) for a process at ``tp_rank``. The rank layout is (dp, cp, tp), with tp changing fastest:: tp_rank = (attn_dp_rank * attn_cp_size + attn_cp_rank) * attn_tp_size + attn_tp_rank Without attention DP the TP group is one attention-DP group, so ``attn_dp_rank`` is zero. """ return tp_rank % attn_tp_size, tp_rank // (attn_tp_size * attn_cp_size) def spawn_world_rank(server_args, *, tp_rank: int, pp_rank: int) -> int: """Compute the WORLD rank from launcher TP and PP ranks. Uses a resolving view because callers may run before ``publish``. """ from sglang.srt.arg_groups.model_override_base import resolving_view cfg = resolving_view(server_args) return cfg.ep_join_rank_offset + cfg.tp_size * pp_rank + tp_rank def derive_spawn_ranks( *, world_rank: int, tp_size: int, ep_join_rank_offset: int, attn_cp_size: int, attn_tp_size: int, moe_dp_size: int, moe_ep_size: int, ) -> dict: """Derive TP, PP, attention, and MoE ranks without process groups. WORLD uses ``rank = ep_join_rank_offset + tp_size * pp_rank + tp_rank``. TP groups are contiguous blocks; PP groups are strided by ``tp_size``. """ local = world_rank - ep_join_rank_offset tp_rank = local % tp_size return { "tp_rank": tp_rank, "pp_rank": local // tp_size, "attn_cp_rank": (tp_rank // attn_tp_size) % attn_cp_size, "moe_dp_rank": tp_rank // (tp_size // moe_dp_size), "moe_ep_rank": ( tp_rank % (tp_size // moe_dp_size) // (tp_size // moe_dp_size // moe_ep_size) ), "moe_tp_rank": tp_rank % (tp_size // moe_dp_size // moe_ep_size), } def derive_parallel_widths( *, tp_size: int, attn_cp_size: int, attn_dp_size: int, moe_ep_size: int, moe_dp_size: int, dcp_size: int, dcp_enabled: bool, ) -> dict: """Derive attention and MoE widths and DCP settings from configuration.""" return { "attn_dp_size": attn_dp_size, "attn_tp_size": derive_attn_tp_size( tp_size=tp_size, attn_cp_size=attn_cp_size, attn_dp_size=attn_dp_size ), "moe_ep_size": moe_ep_size, "moe_tp_size": tp_size // moe_ep_size // moe_dp_size, "dcp_enabled": dcp_enabled, "attn_dcp_size": dcp_size if dcp_enabled else 1, } def parallel_widths_of(cfg: Any) -> dict: """Return derived parallel settings from resolved configuration.""" return derive_parallel_widths( tp_size=cfg.tp_size, attn_cp_size=cfg.attn_cp_size, attn_dp_size=cfg.attn_dp_size, moe_ep_size=cfg.ep_size, moe_dp_size=cfg.moe_dp_size, dcp_size=cfg.dcp_size, dcp_enabled=cfg.dcp_size > 1, ) def launch_world_size_of(cfg: Any): """Return the initial WORLD width, including existing ranks below a joiner. This value remains fixed after elastic scale-up. """ return cfg.ep_join_rank_offset + cfg.tp_size * cfg.pp_size def max_world_size_of(cfg: Any): """Return the WORLD capacity: ``max_ep_size`` or the launch width.""" return cfg.max_ep_size or launch_world_size_of(cfg) def attn_tp_size_of(cfg: Any): """`attn_tp_size`, computed at publish. See `parallel_widths_of`.""" return parallel_widths_of(cfg)["attn_tp_size"] def attn_dp_enabled_of(cfg: Any) -> bool: """`attn_dp_enabled`, computed at publish. A scale joiner's own group can be one rank wide, but it joins an attention-DP deployment and runs its DP-attention paths. """ from sglang.srt.arg_groups.model_override_base import ep_scale_joiner_of return cfg.attn_dp_size > 1 or ep_scale_joiner_of(cfg) def num_dp_ranks_of(cfg: Any) -> int: """`num_dp_ranks`, computed at publish: replicas times attention-DP groups. DP attention does not run inside replicas, so one of the two is one. """ return cfg.dp_size * cfg.attn_dp_size def attn_dcp_size_of(cfg: Any): """`attn_dcp_size`, computed at publish. See `parallel_widths_of`.""" return parallel_widths_of(cfg)["attn_dcp_size"] def moe_ep_size_of(cfg: Any): """`moe_ep_size`, computed at publish. See `parallel_widths_of`.""" return parallel_widths_of(cfg)["moe_ep_size"] def moe_tp_size_of(cfg: Any): """`moe_tp_size`, computed at publish. See `parallel_widths_of`.""" return parallel_widths_of(cfg)["moe_tp_size"] def dcp_enabled_of(cfg: Any): """`dcp_enabled`, computed at publish. See `parallel_widths_of`.""" return parallel_widths_of(cfg)["dcp_enabled"] class SpawnRanks(msgspec.Struct, frozen=True): """Process placement supplied by the launcher. ``world_rank`` determines TP, PP, attention, and MoE ranks from the configured widths. ``dp_rank`` identifies the replica across separate WORLD groups; ``None`` means no data-parallel controller. ``gpu_id`` is the assigned device index, or ``None`` for a process without a device. """ world_rank: int dp_rank: Optional[int] = None gpu_id: Optional[int] = None _RANK_AND_WIDTH = ( ("tp_rank", "tp_size"), ("pp_rank", "pp_size"), ("attn_tp_rank", "attn_tp_size"), ("attn_dp_rank", "attn_dp_size"), ("attn_cp_rank", "attn_cp_size"), ("moe_ep_rank", "moe_ep_size"), ) # MoE-DP may alias a wider attention-CP group, so its configured width # need not match the group width. _WIDTH_AND_GROUP = ( ("tp_size", "tp_group"), ("pp_size", "pp_group"), ("attn_tp_size", "attn_tp_group"), ("attn_cp_size", "attn_cp_group"), ("moe_ep_size", "moe_ep_group"), ) _UNREADABLE = object() def _validate_parallel(parallel, source: str) -> None: """Check rank bounds, topology factorizations, and group widths. Skip unavailable or non-integer values so partially initialized contexts can be validated. """ def read(name): """Return an integer topology value, or ``_UNREADABLE``; exclude booleans.""" try: value = getattr(parallel, name) except Exception: return _UNREADABLE if isinstance(value, bool) or not isinstance(value, int): return _UNREADABLE return value problems = [] for rank_name, size_name in _RANK_AND_WIDTH: rank, size = read(rank_name), read(size_name) if _UNREADABLE in (rank, size): continue if not 0 <= rank < size: problems.append( f"0 <= {rank_name} < {size_name}\n {rank} is not a rank of {size}" ) terms = ("tp_size", "attn_tp_size", "attn_dp_size", "attn_cp_size") tp_size, a_tp, a_dp, a_cp = (read(n) for n in terms) if _UNREADABLE not in (tp_size, a_tp, a_dp, a_cp): if tp_size != a_tp * a_dp * a_cp: problems.append( "tp_size == attn_tp_size * attn_dp_size * attn_cp_size\n" f" {tp_size} != {a_tp} * {a_dp} * {a_cp} (= {a_tp * a_dp * a_cp})" ) moe_terms = ("tp_size", "moe_ep_size", "moe_dp_size", "moe_tp_size") tp_size, m_ep, m_dp, m_tp = (read(n) for n in moe_terms) if _UNREADABLE not in (tp_size, m_ep, m_dp, m_tp): if tp_size != m_ep * m_dp * m_tp: problems.append( "tp_size == moe_ep_size * moe_dp_size * moe_tp_size\n" f" {tp_size} != {m_ep} * {m_dp} * {m_tp} (= {m_ep * m_dp * m_tp})" ) layout_terms = ( "tp_rank", "attn_dp_rank", "attn_cp_rank", "attn_tp_rank", "attn_cp_size", "attn_tp_size", ) tp_rank, r_dp, r_cp, r_tp, w_cp, w_tp = (read(n) for n in layout_terms) if _UNREADABLE not in (tp_rank, r_dp, r_cp, r_tp, w_cp, w_tp): laid_out = (r_dp * w_cp + r_cp) * w_tp + r_tp if tp_rank != laid_out: problems.append( "tp_rank == (attn_dp_rank * attn_cp_size + attn_cp_rank)" " * attn_tp_size + attn_tp_rank\n" f" {tp_rank} != ({r_dp} * {w_cp} + {r_cp})" f" * {w_tp} + {r_tp} (= {laid_out})" ) for size_name, group_name in _WIDTH_AND_GROUP: size = read(size_name) if size is _UNREADABLE: continue try: group = getattr(parallel, group_name) except Exception: continue built = getattr(group, "world_size", _UNREADABLE) if isinstance(built, int) and not isinstance(built, bool) and built != size: problems.append( f"{group_name}.world_size == {size_name}\n" f" built {built}, configured {size}" ) if problems: raise ValueError( f"parallel topology is inconsistent (set by {source}):\n" + "\n".join(problems) ) class ParallelContext: """Parallel configuration, process ranks, and group handles. Configured and derived widths come from the published configuration. ``publish`` records ranks from the launcher; distributed initialization records group handles. Scoped overrides take precedence over permanent overrides and configuration. Uninitialized runtime fields raise on read. """ __slots__ = ("_overrides", "_stamp", "_config") def __init__(self): self._overrides = {} # scoped, restored when the `with` block exits self._stamp = {} # permanent for the process, dropped by clear_stamp self._config = None # parallel config bag, wired at publish def __getattr__(self, name): if name.startswith("_"): # This also breaks the recursion when the ``_config`` slot itself is # still unset (pickle/copy protocols probe attributes before # __init__ runs). raise AttributeError(name) return self._read(name) def _read(self, name): """Read a scoped override, permanent override, or published value, in order.""" overrides = self._overrides if name in overrides: return overrides[name] stamp = self._stamp if name in stamp: return stamp[name] config = self._config if config is not None and name in config._fields: return getattr(config, name) if config is None and name in _parallel_config_leaves(): raise ValueError("config namespace 'parallel' not published") declared = _derived_widths().get(name) if declared is not None and not declared.fn: raise RuntimeError( f"parallel name {name!r} has not been written in this process. " + declared.doc + f" Write it by publishing a rank bundle or building the groups, " f"or state it with get_parallel().override({name}=...)" ) if declared is not None: raise RuntimeError( f"derived parallel width {name!r} is not available: it is computed " "from the configured leaves at publish, and permanently corrected " "when the process groups are built. Nothing is published and " "nothing has been set with override_permanently -- publish a " f"parallel config, or state the width with get_parallel().override({name}=...)" ) raise AttributeError(f"ParallelContext has no {name!r}") def override_permanently(self, **values) -> None: """Set parallel values until ``clear_stamp`` or ``reset_context``. Works without published configuration. Validate the combined topology and restore the previous values if validation fails. """ unknown = set(values) - _parallel_fields() if unknown: raise ValueError(f"unknown parallel field(s): {sorted(unknown)}") saved = dict(self._stamp) self._stamp.update(values) try: _validate_parallel(self, "override_permanently") except Exception: self._stamp = saved raise def clear_stamp(self) -> None: """Drop every stamped name, ranks included.""" self._stamp.clear() @contextmanager def override(self, **kwargs): """Temporarily force parallel values, restoring on exit. Validates keys and supports nesting.""" unknown = set(kwargs) - _parallel_fields() if unknown: raise ValueError(f"unknown parallel field(s): {sorted(unknown)}") saved = dict(self._overrides) self._overrides.update(kwargs) try: _validate_parallel(self, "override") except Exception: self._overrides = saved raise try: yield self finally: self._overrides = saved def _derived_widths() -> dict: """Return parallel ``Derived`` declarations, including ranks and groups.""" from sglang.srt.arg_groups.arg_utils import Derived from sglang.srt.arg_groups.fields.parallel import Parallel return { name: decl for name, decl in vars(Parallel).items() if isinstance(decl, Derived) } def _install_parallel_properties() -> None: """Expose declared parallel fields as documented properties. Properties support class-level introspection; all reads use ``_read``. """ for name, decl in _derived_widths().items(): def getter(self, _name=name): return self._read(_name) getter.__name__ = name getter.__doc__ = decl.doc setattr(ParallelContext, name, property(getter)) _install_parallel_properties() class _FlagGroupBase(msgspec.Struct): """Shared flag-group behavior: typo-safe writes + transactional ``override()``. ``__struct_fields__`` is the single source of truth for which leaves exist, so a mistyped name fails loudly instead of creating a stray attribute. The write goes through ``super().__setattr__``: a ``Struct`` keeps its fields in its own layout, so ``object.__setattr__`` does not reach them. """ def __setattr__(self, name: str, value: Any) -> None: if name not in type(self).__struct_fields__: raise AttributeError( f"{type(self).__name__} has no flag '{name}' (leaves are " "declared as struct fields; check for typos)" ) super().__setattr__(name, value) @contextmanager def override(self, **kwargs): """Temporarily force flag values, restoring on exit. Transactional (keys validated before any write) — the test-only injection primitive.""" fields = type(self).__struct_fields__ unknown = set(kwargs) - set(fields) if unknown: raise ValueError( f"unknown flag(s) for {type(self).__name__}: {sorted(unknown)}" ) saved = {name: getattr(self, name) for name in kwargs} for name, value in kwargs.items(): setattr(self, name, value) try: yield self finally: for name, value in saved.items(): setattr(self, name, value) class CaptureFlags(_FlagGroupBase): """Capture-time flags; never frozen (written during cuda-graph capture).""" # Seeded from server_args at publish; a model whose _can_torch_compile is # False clears it during warmup (the only post-publish writer). enable_torch_compile: bool = False # Set for the duration of decode/spec graph capture (model_capture_mode). # While set, dispose_tensor() is a no-op so deep_gemm's pre-permute does not # free hidden_states that the dual-stream MoE shared expert reads afterward. disable_dispose_tensor: bool = False class MoeFlags(_FlagGroupBase): """MoE runtime flags, materialized by ``initialize_moe_config`` (scheduler init, after distributed setup). ``a2a_backend`` / ``runner_backend`` / ``disable_fp4_allgather`` are the ACTIVE values: the speculative contexts in ``layers.moe.utils`` swap them around draft-model forwards. Values are the parsed enums from ``layers.moe.utils``; ``None`` means "not initialized yet" and the accessors fall back lazily. """ a2a_backend: Any = None runner_backend: Any = None speculative_runner_backend: Any = None speculative_a2a_backend: Any = None deepep_mode: Any = None deepep_config: str | None = None tbo_enabled: bool | None = None sbo_enabled: bool | None = None tbo_token_distribution_threshold: float | None = None disable_fp4_allgather: bool | None = None quantization: str | None = None # The shared-experts-fusion decision, per runner — the runner_backend / # speculative_runner_backend shape. Both leaves are seeded from the config # intent by ``initialize_moe_config``; each MoE model's gate # (determine_num_fused_shared_experts) refines the ACTIVE leaf, both ways, # before its layers build and read it. ``speculative_moe_backend_context`` # brackets a draft's build: on exit the draft's effective decision is # persisted onto the speculative leaf (inspectable afterwards) and the # target's ACTIVE value returns. disable_shared_experts_fusion: bool | None = None speculative_disable_shared_experts_fusion: bool | None = None # Lifecycle marker (the capture.disable_dispose_tensor family): set while # speculative_moe_backend_context is active, so a draft gate's write also # lands on the speculative leaf. in_speculative_scope: bool = False # Draft construction/execution uses a separate one-sided A2A workspace from # the target model's concurrently live CUDA graphs. speculative_context: bool = False class DpFlags(_FlagGroupBase): """DP-attention runtime flags set by ``initialize_dp_attention``. Attention-DP width and rank are stored on ``get_parallel()``. """ enabled: bool = False use_world_group_for_gather: bool = False joiner_skip_all_gather: bool = False # Hybrid-SSM models materialize idle ranks via the MAX_LEN fabricated-row # conversion (set when hf_config has hybrid_override_pattern). max_len_with_idle: bool = False # Set while the prefill CUDA graph runner captures; latched by the DP # gather/scatter helpers, whose captured geometry needs one shared bucket. capturing_prefill_graph: bool = False prefill_graph_has_dp_gather: bool = False # This process's slot in the target's DP sync while a draft scope narrows # attention; dp_gather_width() also reads it as the in-scope marker. scoped_gather_slot: Optional[int] = None # DP gathered-buffer allocation metadata (model hidden size / dtype / # device), set by initialize_dp_attention alongside the flags above. buffer_hidden_size: Any = None buffer_dtype: Any = None buffer_device: Any = None class SpFlags(_FlagGroupBase): """LayerNorm sequence-parallelism flags, materialized by ``initialize_layernorm_sp`` (after distributed setup; reads the model config). See ``layers.layernorm_sp``.""" enabled: bool = False class Flags(_FlagGroupBase): """Mutable runtime state; resolved configuration lives in the config bags.""" capture: CaptureFlags = msgspec.field(default_factory=CaptureFlags) moe: MoeFlags = msgspec.field(default_factory=MoeFlags) dp: DpFlags = msgspec.field(default_factory=DpFlags) sp: SpFlags = msgspec.field(default_factory=SpFlags) class Resources(_FlagGroupBase): """Process-level resource handles: named slots with one reset lifecycle, scoped test injection via ``override()``, and the creation/publish semantics kept in the owning modules' accessors (which are thin shims over these slots).""" # CUDA graph memory pool shared across the prefill and decode graph # backends (created lazily by model_executor.runner_utils.pool). graph_memory_pool: Any = None graph_pool_borrow: GraphPoolBorrowState | None = None # EPLB: per-process recorder and the publish-once location metadata # (owning accessors live in sglang.srt.eplb). expert_distribution_recorder: Any = None expert_location_metadata: Any = None # LPLB: layer_id -> solver. lplb_solvers: dict = msgspec.field(default_factory=dict) # Named side streams (see RuntimeContext.get_stream): name -> stream. streams: dict = msgspec.field(default_factory=dict) # Named persistent buffers (see RuntimeContext.get_buffer): name -> tensor. # Accessors with bespoke semantics (grow-only, per-device keys) manage # their entries directly. buffers: dict = msgspec.field(default_factory=dict) # Persistent reusable CUDA events for non-EP DP TBO, keyed by # (kind, subbatch) — see dp_attention._tbo_event for why reuse matters. tbo_event_pool: dict = msgspec.field(default_factory=dict) flashinfer_megamoe_workspaces: dict = msgspec.field(default_factory=dict) # State capturers (installed by their subsystems when capture is on). indexer_capturer: Any = None experts_capturer: Any = None # The shared TCPStore created during distributed initialization. tcp_store: Any = None # Trace verbosity; the accessor seeds it lazily from SGLANG_TRACE_LEVEL. trace_level: Any = None class LoRABatchLayout(Enum): """Token layout used by LoRA kernels in the current model section.""" DP_LOCAL = auto() TP_GLOBAL = auto() class ForwardFlags: """Scoped per-forward flags. Eager-only flags use ContextVars for thread and nested-scope isolation. ``_GRAPH_VISIBLE`` flags use plain dict slots because Dynamo cannot trace ContextVar access. Their readers and writers run on one thread per process. ``scoped`` restores both backings on exit; ``set`` supports unscoped setters. """ _DEFAULTS = { "multi_stream": False, "moe_output_buffer": None, # Attention-TP input-scattering (set per forward by # AttnTpContext.maybe_input_scattered / set_attn_inputs). "attn_input_scattered": False, "attn_inputs": None, # Sticky across forwards: every ForwardBatch construction writes it; # graph runners force False around capture. "is_extend_in_batch": False, # Per-layer MLP collective control (set by decoder via scoped() # around the MLP / MoE / hybrid mixer call). # mlp_reduce_scatter: postprocess will reduce-scatter (skip MLP AR). # flashinfer_trtllm_bypass: deepseek dual-stream graph topk bypass. # defer_moe_finalize: next layer's fused collective absorbs the finalize. "mlp_reduce_scatter": False, "flashinfer_trtllm_bypass": False, "defer_moe_finalize": False, # Attention runs on a DP-local token batch. The MLP runs after the DP # gather and therefore uses the TP-global token batch. "lora_batch_layout": LoRABatchLayout.DP_LOCAL, # LayerNorm sequence parallelism region; see layers/layernorm_sp.py. "sp_active": False, } # Read/written inside compiled graphs (vocab embedding, layer boundaries, # EP dispatch, DP gather/scatter, MLP/MoE skip-AR): plain-slot backed. # Before moving a flag out of this set, prove no read/write site sits # under torch.compile. _GRAPH_VISIBLE = frozenset( { "attn_input_scattered", "attn_inputs", "is_extend_in_batch", "mlp_reduce_scatter", "flashinfer_trtllm_bypass", "defer_moe_finalize", "lora_batch_layout", "sp_active", } ) __slots__ = ("_vars", "_plain") def __init__(self): import contextvars object.__setattr__( self, "_plain", { name: default for name, default in self._DEFAULTS.items() if name in self._GRAPH_VISIBLE }, ) object.__setattr__( self, "_vars", { name: contextvars.ContextVar(f"forward.{name}", default=default) for name, default in self._DEFAULTS.items() if name not in self._GRAPH_VISIBLE }, ) def __getattr__(self, name: str) -> Any: plain = self._plain if name in plain: return plain[name] try: return self._vars[name].get() except KeyError: raise AttributeError( f"ForwardFlags has no flag '{name}' (flags are declared in " "ForwardFlags._DEFAULTS; check for typos)" ) from None def __setattr__(self, name: str, value: Any) -> None: raise AttributeError( "ForwardFlags is written through scoped(**kw) (or the legacy " "set() shim), never by attribute assignment" ) def set(self, name: str, value: Any) -> None: """Unscoped write for legacy setter shims; persists until the next write (current context only, for contextvar-backed flags).""" if name in self._plain: self._plain[name] = value else: self._vars[name].set(value) @contextmanager def scoped(self, **kwargs): """Set flags for the current scope, restoring on exit. Transactional (keys validated before any write) and exception-safe.""" unknown = set(kwargs) - set(self._DEFAULTS) if unknown: raise ValueError(f"unknown forward flag(s): {sorted(unknown)}") plain_saved = [ (name, self._plain[name]) for name in kwargs if name in self._plain ] tokens = [] for name, value in kwargs.items(): if name in self._plain: self._plain[name] = value else: tokens.append((self._vars[name], self._vars[name].set(value))) try: yield self finally: for var, token in reversed(tokens): var.reset(token) for name, value in reversed(plain_saved): self._plain[name] = value class _ConfigBag: """Resolved namespace with nested bags and guarded writes. Publication snapshots resolved values; ``RuntimeContext.override`` updates them permanently and ``override`` restores scoped changes on exit. Leaves and sub-bags are real instance attributes so Dynamo can trace reads. ``_fields`` and ``_subs`` track names for routing and restoration. """ def __init__(self, path: str): object.__setattr__(self, "_path", path) object.__setattr__(self, "_fields", {}) # {leaf: value} object.__setattr__(self, "_subs", {}) # {subname: _ConfigBag} def __getattr__(self, name: str) -> Any: # Fallback only: real leaves/sub-bags resolve via __dict__ before this # runs. Uses object.__getattribute__ (not self._fields) to stay safe if # invoked before __init__ populates the bookkeeping dicts. fields = object.__getattribute__(self, "_fields") if name in fields: return fields[name] subs = object.__getattribute__(self, "_subs") if name in subs: return subs[name] path = object.__getattribute__(self, "_path") raise AttributeError(f"config namespace {path!r} has no leaf/subgroup {name!r}") def __setattr__(self, name: str, value: Any) -> None: raise AttributeError( f"config namespace {self._path!r} is read-only; write via " "get_context().override(source, ...) or the scoped .override(**kw)" ) def _set(self, name: str, value: Any) -> None: """Update the leaf map and traceable attribute, bypassing the write guard.""" object.__getattribute__(self, "_fields")[name] = value object.__setattr__(self, name, value) def _set_sub(self, name: str, sub: _ConfigBag) -> None: """Register a nested bag in the lookup map and as a traceable attribute.""" object.__getattribute__(self, "_subs")[name] = sub object.__setattr__(self, name, sub) def __contains__(self, name: str) -> bool: return name in object.__getattribute__(self, "_fields") @contextmanager def override(self, **kwargs): """Temporarily override this bag's leaves, validating all keys before writing. Restores on exit; use ``RuntimeContext.override`` for permanent changes. """ fields = object.__getattribute__(self, "_fields") unknown = set(kwargs) - set(fields) if unknown: path = object.__getattribute__(self, "_path") raise ValueError(f"unknown config leaf for {path!r}: {sorted(unknown)}") saved = {name: fields[name] for name in kwargs} for name, value in kwargs.items(): self._set(name, value) try: yield self finally: for name, value in saved.items(): self._set(name, value) def _build_config_bags(server_args: Any) -> dict: """Project resolved fields by namespace, rejecting leaf/subgroup name collisions.""" from sglang.srt.arg_groups.arg_utils import namespace_of from sglang.srt.arg_groups.overrides import resolution_result _MISSING = object() tops: dict = {} for field, path in namespace_of(type(server_args)).items(): value = resolution_result(server_args, field, _MISSING) if value is _MISSING: raise AttributeError( f"config field {field!r} belongs to namespace {path!r} but is absent from " f"the published {type(server_args).__name__}; cannot project its bag leaf" ) parts = path.split(".") bag = tops.get(parts[0]) if bag is None: bag = tops[parts[0]] = _ConfigBag(parts[0]) for depth in range(1, len(parts)): name = parts[depth] if name in object.__getattribute__(bag, "_fields"): raise ValueError( f"config namespace collision: {'.'.join(parts[: depth + 1])!r} " "is declared as both a leaf and a subgroup" ) subs = object.__getattribute__(bag, "_subs") child = subs.get(name) if child is None: child = _ConfigBag(".".join(parts[: depth + 1])) bag._set_sub(name, child) bag = child if field in object.__getattribute__(bag, "_subs"): raise ValueError( f"config namespace collision: leaf {field!r} under {path!r} " "clashes with a subgroup of the same name" ) bag._set(field, value) _install_derived_leaves(tops, server_args) return tops def _install_derived_leaves(tops: dict, server_args: Any) -> None: """Compute declared derived leaves once at publication. Derivations receive the whole resolved config because they may span namespaces. """ import importlib from sglang.srt.arg_groups.arg_utils import _NO_DEFAULT, Derived from sglang.srt.arg_groups.overrides import resolved_view namespaces = getattr(type(server_args), "_NAMESPACES", None) if not namespaces: return view = resolved_view(server_args) for source in namespaces: path = getattr(source, "_NS_PATH", None) if path is None: continue for name, decl in vars(source).items(): if not isinstance(decl, Derived): continue if not decl.fn: # Initialize runtime-only fields that declare a default. if decl.default is not _NO_DEFAULT: bag = tops.get(path.split(".")[0]) for segment in path.split(".")[1:]: bag = bag and getattr(bag, segment, None) if bag is not None: bag._set(name, decl.default) continue module, _, attr = decl.fn.rpartition(".") bag = tops.get(path.split(".")[0]) for segment in path.split(".")[1:]: bag = bag and getattr(bag, segment, None) if bag is None: continue bag._set(name, getattr(importlib.import_module(module), attr)(view)) def _resolved_or_field(server_args: Any, name: str, default: Any) -> Any: """What resolution decided for `name`, falling back to the field. Publishes that carry no config at all (sentinels, mocks) have neither, and answer with `default`. """ if server_args is None: return default from sglang.srt.arg_groups.overrides import resolution_result decided = resolution_result(server_args, name) if decided is not None: return decided return getattr(server_args, name, default) class RuntimeContext: """Container for the structured runtime accessors; exposes ``parallel``, ``server_args``, the resolved config namespace bags, ``flags``, ``resources``, and ``forward``.""" __slots__ = ( "parallel", "_server_args", "_config_bags", "_overrides_log", "_publish_role", "flags", "resources", "forward", ) def __init__(self, parallel: ParallelContext): self.parallel = parallel self._server_args: ServerArgs | None = None self._config_bags: dict | None = None self._overrides_log: list = [] self._publish_role: str | None = None self.flags = Flags() self.resources = Resources() self.forward = ForwardFlags() def get_stream(self, name: str) -> Any: """Get or create a named process stream. Call before CUDA graph capture.""" from sglang.srt.arg_groups.overrides import resolution_result stream = self.resources.streams.get(name) if stream is None: import torch device = ( resolution_result(self._server_args, "device") if self._server_args else "cuda" ) stream = torch.get_device_module(device).Stream() self.resources.streams[name] = stream return stream def set_stream(self, name: str, stream: Any) -> Any: """Install (or replace) the named stream — explicit injection for tests and backends that bring their own stream.""" self.resources.streams[name] = stream return stream def get_buffer(self, name: str, factory: Any) -> Any: """Get or create a named persistent buffer using ``factory()``.""" buf = self.resources.buffers.get(name) if buf is None: buf = factory() self.resources.buffers[name] = buf return buf @property def server_args(self) -> ServerArgs: """The process-wide ``ServerArgs`` (context-owned slot).""" server_args = self._server_args if server_args is None: # Verbatim legacy message: tests and user scripts may match on it. raise ValueError("Global server args is not set yet!") return server_args def set_server_args(self, server_args: ServerArgs) -> None: """Replace the raw input and rebuild resolved config bags, clearing override provenance.""" self.flags.capture.enable_torch_compile = bool( _resolved_or_field(server_args, "enable_torch_compile", False) ) self._server_args = server_args # Preserve the launcher-assigned device when rebuilding config bags. stated = {} if self._config_bags is not None: device = self._config_bags.get("device") fields = object.__getattribute__(device, "_fields") if device else {} if "gpu_id" in fields: stated["gpu_id"] = fields["gpu_id"] self._config_bags = _build_config_bags(server_args) device = self._config_bags.get("device") if stated and device is not None: device._set("gpu_id", stated["gpu_id"]) spec = self._config_bags.get("spec") if spec is not None: from sglang.srt.arg_groups.overrides import ( max_speculative_num_draft_tokens as max_draft_tokens_of, ) # Keep the launch-time capacity stable while adaptive algorithms # change the active width in this bag. spec._set( "max_speculative_num_draft_tokens", max_draft_tokens_of(server_args), ) self.parallel._config = self._config_bags.get("parallel") # A direct install is roleless; ``publish`` assigns the role afterwards. self._overrides_log = [] self._publish_role = None def config_bag(self, name: str) -> _ConfigBag: """Return a top-level namespace bag; raise if it has not been published.""" bags = self._config_bags if not bags or name not in bags: raise ValueError(f"config namespace {name!r} not published") if _ROLE_NS_MODE != "off": self._check_role_namespace(name) return bags[name] def is_config_namespace_published(self, name: str) -> bool: """Return whether a config namespace exists in the current context.""" bags = self._config_bags return bags is not None and name in bags def _check_role_namespace(self, name: str) -> None: # Out of line so the mode gate above stays one dead-branch-prunable # check under dynamo in the default "off" mode (config_bag runs inside # compiled model forwards). role = self._publish_role if _ROLE_NS_MODE == "record": if not _is_compiling(): _record_namespace_read(role, name) elif _ROLE_NS_MODE == "enforce" and role is not None: if role not in ROLE_NAMESPACE_SETS: raise ValueError( f"publish role {role!r} has no ROLE_NAMESPACE_SETS entry; " "declare its namespace set (None for the full tree)." ) allowed = ROLE_NAMESPACE_SETS[role] if allowed is not None and name not in allowed: raise ValueError( f"config namespace {name!r} is outside the declared set " f"for publish role {role!r} ({sorted(allowed)}). If this " "read is legitimate for the process type, extend " "ROLE_NAMESPACE_SETS; if not, the read belongs in a " "different process or behind a per-instance boundary." ) def override(self, source: str, **fields) -> None: """Update resolved config leaves without changing the raw input. Validate all field names and target bags before writing. Record ``source`` and the changed fields for provenance. """ if not fields: return bags = self._config_bags if bags is None: raise ValueError("config not published; cannot override") from sglang.srt.arg_groups.arg_utils import namespace_of nsmap = namespace_of(type(self._server_args)) targets = [] # (bag, leaf, value) — resolved before any write for name, value in fields.items(): path = nsmap.get(name) if path is None: raise ValueError( f"override: unknown config field {name!r} (no namespace) — " "not a resolved config leaf" ) parts = path.split(".") bag = bags.get(parts[0]) if bag is None: raise ValueError(f"override: namespace {parts[0]!r} not published") for seg in parts[1:]: bag = object.__getattribute__(bag, "_subs").get(seg) if bag is None: raise ValueError( f"override: subgroup {seg!r} missing under {path!r}" ) if name not in bag: raise ValueError(f"override: field {name!r} not projected on {path!r}") targets.append((bag, name, value)) for bag, name, value in targets: bag._set(name, value) self._overrides_log.append((source, dict(fields))) def config_leaf(self, name: str): """Read a resolved config leaf by its flat field name.""" bags = self._config_bags if bags is None: raise ValueError("config not published; cannot read a config leaf") from sglang.srt.arg_groups.arg_utils import namespace_of path = namespace_of(type(self._server_args)).get(name) if path is None: raise ValueError(f"{name!r} is not a config leaf (no namespace)") parts = path.split(".") bag = self.config_bag(parts[0]) for seg in parts[1:]: bag = object.__getattribute__(bag, "_subs").get(seg) if bag is None: raise ValueError(f"subgroup {seg!r} missing under {path!r}") return getattr(bag, name) def overrides_log(self) -> list: """Return ``(source, fields)`` entries with shallow copies of the field dicts.""" return [(source, dict(fields)) for source, fields in self._overrides_log] def resolved_server_args_dict(self, base: dict | None = None) -> dict: """Overlay this process's runtime overrides on the resolved startup config. ``base`` defaults to ``server_args.resolved_dict()``. Scheduler readbacks include these overrides in ``/server_info``'s ``internal_states`` block; the top-level fields describe startup config. TokenizerManager separately overlays its per-instance model identity. """ d = self.server_args.resolved_dict() if base is None else dict(base) for _source, fields in self._overrides_log: d.update(fields) return d def override_server_args(self, **fields) -> _ServerArgsOverride: """Publish a temporary dummy config for tests and restore the previous context. Use as a context manager or call ``install`` / ``restore`` explicitly. Prefer a bag's scoped ``override`` when only existing leaves need changing. """ return _ServerArgsOverride(self, fields) class _ServerArgsOverride: """Scoped config override (see ``RuntimeContext.override_server_args``). Deliberately a plain class rather than a generator context manager: fixtures that live for a whole test case install the override without a ``with`` block, and a suspended generator would run its restore whenever the garbage collector closes it — un-publishing the active config at a nondeterministic point. """ __slots__ = ( "_context", "_fields", "_prev_server_args", "_prev_bags", "_prev_overrides_log", "_prev_publish_role", "_prev_parallel_config", "_prev_capture", "_installed", ) def __init__(self, context: RuntimeContext, fields: dict): self._context = context self._fields = fields self._installed = False def install(self) -> ServerArgs: """Publish a fresh dummy-boundary ``ServerArgs`` carrying the overrides; returns the published instance.""" from sglang.srt.server_args import ServerArgs assert not self._installed, "override_server_args already installed" ctx = self._context self._prev_server_args = ctx._server_args self._prev_bags = ctx._config_bags self._prev_overrides_log = ctx._overrides_log self._prev_publish_role = ctx._publish_role self._prev_parallel_config = ctx.parallel._config self._prev_capture = ctx.flags.capture.enable_torch_compile from sglang.srt.arg_groups.overrides import ( declare_resolution, ) server_args = ServerArgs(model_path="dummy") server_args.resolve_once() # Underscore names seed private property caches (the strict guard # exempts them); everything else must be a real config field. unknown = {name for name in self._fields if not name.startswith("_")} - set( type(server_args).__struct_fields__ ) if unknown: raise ValueError( f"override_server_args: unknown ServerArgs field(s): {sorted(unknown)}" ) # Declare config fields even when underscore-prefixed; only non-fields seed caches. fields = set(type(server_args).__struct_fields__) declared = {n: v for n, v in self._fields.items() if n in fields} if declared: declare_resolution(server_args, "override_server_args", **declared) seeds = {n: v for n, v in self._fields.items() if n not in fields} for name, value in seeds.items(): msgspec.Struct.__setattr__(server_args, name, value) ctx.set_server_args(server_args) self._installed = True return server_args def restore(self) -> None: """Reinstate the exact pre-install lifecycle state (or the empty slot).""" if not self._installed: return self._installed = False ctx = self._context ctx._server_args = self._prev_server_args ctx._config_bags = self._prev_bags ctx._overrides_log = self._prev_overrides_log ctx._publish_role = self._prev_publish_role ctx.parallel._config = self._prev_parallel_config ctx.flags.capture.enable_torch_compile = self._prev_capture self._prev_server_args = None self._prev_bags = None self._prev_overrides_log = None self._prev_parallel_config = None def __enter__(self) -> ServerArgs: return self.install() def __exit__(self, *exc) -> None: self.restore() _PARALLEL = ParallelContext() _CONTEXT = RuntimeContext(parallel=_PARALLEL) def get_context() -> RuntimeContext: return _CONTEXT def get_parallel() -> ParallelContext: return _PARALLEL def get_server_args() -> ServerArgs: return _CONTEXT.server_args def get_flags() -> Flags: return _CONTEXT.flags def get_resources() -> Resources: return _CONTEXT.resources def get_forward() -> ForwardFlags: return _CONTEXT.forward # --- Resolved config namespaces ------------------------- # Each returns the top-level snapshot bag; reads are `get_exec().moe.field` etc. # All fail with ValueError("... not published") until publish has projected them. # ``parallel`` has no bag getter: ``get_parallel()`` answers its leaves # directly, alongside the live topology they belong to. def get_device() -> _ConfigBag: return _CONTEXT.config_bag("device") def get_model() -> _ConfigBag: return _CONTEXT.config_bag("model") def get_exec() -> _ConfigBag: return _CONTEXT.config_bag("exec") def get_schedule() -> _ConfigBag: return _CONTEXT.config_bag("schedule") def get_memory() -> _ConfigBag: return _CONTEXT.config_bag("memory") def get_spec() -> _ConfigBag: return _CONTEXT.config_bag("spec") def get_lora() -> _ConfigBag: return _CONTEXT.config_bag("lora") def get_mm() -> _ConfigBag: return _CONTEXT.config_bag("mm") def get_disagg() -> _ConfigBag: return _CONTEXT.config_bag("disagg") def get_serving() -> _ConfigBag: return _CONTEXT.config_bag("serving") def get_observability() -> _ConfigBag: return _CONTEXT.config_bag("observability") # --- Per-role namespace sets ------------------------------------------- # # ``publish(role=...)`` records which process type installed the config; this # table declares which top-level config namespaces each role reads. ``None`` # means the full tree — either the role genuinely needs everything (scheduler) # or its deployment shape has not been audited yet (restrict only what smoke # coverage can verify). ``parallel`` is served by ``get_parallel()`` and # every process legitimately reads topology config, so it is not in this table. # # ``SGLANG_ROLE_NAMESPACES`` selects the mode (read once at import): # off (default) no bookkeeping, zero overhead; # record audit mode — collect (role, namespace) reads per process and dump # them at exit (the data that seeds this table). Reads made inside # torch.compile-traced code are NOT observed (recording is pruned # under tracing to keep capture legal) — run audits with # compilation disabled before restricting a role. # enforce fail closed — a bag read outside the role's declared set raises. ROLE_NAMESPACE_SETS: dict[str, frozenset[str] | None] = { # Reads (almost) everything by design — the model-executing process. "scheduler": None, "test": None, "dp_controller": frozenset( {"exec", "parallel", "device", "disagg", "observability", "serving"} ), # Keep unrestricted until multimodal, LoRA, disaggregation, and gRPC reads are audited. "tokenizer": None, # Deployment shapes not exercised locally; audit before restricting. "detokenizer": None, "encoder": None, "expert_backup": None, "weight_cache_daemon": None, # The diffusion GPU worker runs a model and publishes a placeholder so # shared SRT reads do not fail closed; declared full for that reason. "diffusion_gpu_worker": None, } def _validated_role_ns_mode(value: str) -> str: mode = value.strip().lower() if mode not in ("off", "record", "enforce"): raise ValueError( f"SGLANG_ROLE_NAMESPACES={value!r} is not one of off / record / " "enforce — refusing to guess (a typo here would silently disable " "enforcement)." ) return mode def _role_ns_mode_from_env() -> str: # Resolved once at import so the config_bag gate stays a dynamo-prunable # constant; validated fail-loud here (EnvField's warn-and-default parse # would silently turn a typo into "off"). from sglang.srt.environ import envs return _validated_role_ns_mode(envs.SGLANG_ROLE_NAMESPACES.get()) _ROLE_NS_MODE = _role_ns_mode_from_env() _RECORDED_NS_READS: set[tuple[str | None, str]] = set() _RECORD_DUMP_REGISTERED = False def _is_compiling() -> bool: # Recording has Python side effects (set mutation, file I/O, atexit) that # must never run under tracing; torch.compiler.is_compiling() is dynamo's # sanctioned probe. The function-level import keeps this module # import-light; a sys.modules lookup here breaks fullgraph tracing (dynamo # enumerates the dict, which other imports mutate mid-trace). import torch return torch.compiler.is_compiling() def _ensure_record_dump_registered() -> None: global _RECORD_DUMP_REGISTERED if not _RECORD_DUMP_REGISTERED: _RECORD_DUMP_REGISTERED = True import atexit atexit.register(_dump_recorded_namespace_reads) def _append_role_ns_out(role: str | None, name: str) -> None: # Persist immediately: worker processes are routinely torn down with # signals that skip atexit, and the audit must survive that. from sglang.srt.environ import envs out = envs.SGLANG_ROLE_NAMESPACES_OUT.get() if not out: return try: with open(out, "a") as f: f.write(f"{role} {name}\n") except OSError as e: # The entry stays in the in-memory set; the exit summary still covers it. print( f"[role-namespaces] pid={os.getpid()} failed to append " f"({role}, {name}) to {out!r}: {e}", file=sys.stderr, flush=True, ) def _record_namespace_read(role: str | None, name: str) -> None: if (role, name) in _RECORDED_NS_READS: return _RECORDED_NS_READS.add((role, name)) _append_role_ns_out(role, name) _ensure_record_dump_registered() def _dump_recorded_namespace_reads() -> None: """Emit the record-mode audit: one line per role with the namespaces its process actually read (multi-process runs dump once per process). The process's own publish role is always included, so a zero-read role emits an (empty) line rather than being indistinguishable from a process where recording never ran.""" by_role: dict = {} own_role = _CONTEXT._publish_role if own_role is not None: by_role.setdefault(own_role, set()) for role, name in _RECORDED_NS_READS: if name == "-": # publish-time marker, not a namespace read by_role.setdefault(role, set()) continue by_role.setdefault(role, set()).add(name) for role in sorted(by_role, key=str): print( f"[role-namespaces] pid={os.getpid()} role={role} " f"read={','.join(sorted(by_role[role]))}", file=sys.stderr, flush=True, ) def publish( server_args, *, role: str, hf_config: Any = None, ranks: SpawnRanks | None = None, ) -> RuntimeContext: """Install process-wide config for this OS process. Records the process ``role`` — one of the ``ROLE_NAMESPACE_SETS`` keys, which is the one place the roles are enumerated — and projects the config bags. Draft workers skip publish (they must not clobber the target). ``role`` is provenance, and — when ``SGLANG_ROLE_NAMESPACES`` is ``enforce`` — the key into ``ROLE_NAMESPACE_SETS`` for fail-closed namespace-read enforcement (``record`` audits the reads instead). ``hf_config`` is accepted for forward-compat and currently unused. ``ranks`` supplies launcher placement for roles that participate in the parallel topology. Without it, rank reads require an explicit override, except for ``attn_dcp_rank=0`` when DCP is disabled. A process holds at most one live config: the bags always describe the engine running now. Re-publish is allowed and is **last-publish-wins** (bags re-projected, provenance reset, role overwritten), which is what lets one process rebuild an engine after shutting the previous one down. """ if _ROLE_NS_MODE == "enforce" and role not in ROLE_NAMESPACE_SETS: # Fail closed at publish time, not at the first stray read. raise ValueError( f"publish role {role!r} has no ROLE_NAMESPACE_SETS entry; declare " "its namespace set (None for the full tree)." ) server_args.resolve_once() discarded = _CONTEXT.overrides_log() _CONTEXT.set_server_args(server_args) if discarded: logger.warning( "publish(role=%s) re-projected the config bags and dropped %d " "override(s) taken since the last publish: %s", role, len(discarded), ", ".join( f"{source}({', '.join(sorted(fields))})" for source, fields in discarded ), ) _CONTEXT._publish_role = role # Disabled DCP has rank zero even in processes without a rank bundle. if not _CONTEXT.parallel.dcp_enabled: _CONTEXT.parallel.override_permanently(attn_dcp_rank=0) # The device is assigned by the launcher; it is not a config field. _CONTEXT.config_bag("device")._set( "gpu_id", ranks.gpu_id if ranks is not None else None ) if ranks is not None: parallel = _CONTEXT.parallel placement = derive_spawn_ranks( world_rank=ranks.world_rank, tp_size=parallel.tp_size, ep_join_rank_offset=parallel.ep_join_rank_offset, attn_cp_size=parallel.attn_cp_size, attn_tp_size=parallel.attn_tp_size, moe_dp_size=parallel.moe_dp_size, moe_ep_size=parallel.moe_ep_size, ) # MoE-DP aliases the attention-CP group when CP is wider. if parallel.moe_dp_size < parallel.attn_cp_size: placement["moe_dp_rank"] = placement["attn_cp_rank"] # `None` means no data-parallel controller. placement["dp_rank"] = ranks.dp_rank placement["launch_world_rank"] = ranks.world_rank placement.update(_attention_ranks(parallel, placement["tp_rank"])) # DCP groups are contiguous slices of a TP group. if parallel.dcp_enabled: placement["dcp_rank"] = placement["tp_rank"] % parallel.dcp_size placement["attn_dcp_rank"] = placement.get("dcp_rank", 0) parallel.override_permanently(**placement) _validate_parallel(parallel, "publish") if _ROLE_NS_MODE == "record": # The '-' marker distinguishes a zero-read role from a process where # recording never ran (signal teardown skips atexit). _record_namespace_read(role, "-") print( f"[role-namespaces] pid={os.getpid()} role={role} recording; note: " "reads inside torch.compile-traced code are not observed — audit " "with compilation disabled before restricting a role.", file=sys.stderr, flush=True, ) return _CONTEXT def _attention_ranks(parallel, tp_rank: int) -> dict: """Derive attention ranks from the configured widths and the TP rank.""" attn_tp_rank, attn_dp_rank = derive_attention_ranks( tp_rank=tp_rank, attn_tp_size=parallel.attn_tp_size, attn_cp_size=parallel.attn_cp_size, ) return {"attn_tp_rank": attn_tp_rank, "attn_dp_rank": attn_dp_rank} def assert_published(server_args, *, role: str) -> RuntimeContext: """Require this record and role to have been published by the process entry. Re-publishing here would discard runtime overrides and their provenance. """ if _CONTEXT._server_args is server_args and _CONTEXT._publish_role == role: return _CONTEXT if _CONTEXT._server_args is None: detail = "nothing is published in this process" elif _CONTEXT._server_args is not server_args: detail = ( "a different record is published " f"(role={_CONTEXT._publish_role!r}); this constructor was handed " "one the process never published" ) else: detail = ( f"this record is published under role " f"{_CONTEXT._publish_role!r}, not {role!r}" ) raise RuntimeError( f"config not published for role {role!r}: {detail}. The process entry " "publishes -- add publish(server_args, role=...) there rather than " "publishing from a constructor." ) def publish_role() -> str | None: """The role recorded by the last ``publish`` (None for a legacy set).""" return _CONTEXT._publish_role def get_stream(name: str) -> Any: return _CONTEXT.get_stream(name) def set_stream(name: str, stream: Any) -> Any: return _CONTEXT.set_stream(name, stream) def get_buffer(name: str, factory: Any) -> Any: return _CONTEXT.get_buffer(name, factory) _GLOBAL_DWDP_MANAGER: Any = None def get_global_dwdp_manager() -> Any: return _GLOBAL_DWDP_MANAGER def set_global_dwdp_manager(manager: Any) -> None: global _GLOBAL_DWDP_MANAGER _GLOBAL_DWDP_MANAGER = manager def _group_leaves(group: _FlagGroupBase) -> dict[str, Any]: """The leaf values of a flag group, recursively.""" leaves: dict[str, Any] = {} for name in type(group).__struct_fields__: value = getattr(group, name) if isinstance(value, _FlagGroupBase): leaves[name] = _group_leaves(value) elif isinstance(value, (dict, list)): leaves[name] = type(value)(value) else: leaves[name] = value return leaves def _restore_leaves(group: _FlagGroupBase, leaves: dict[str, Any]) -> None: for name, value in leaves.items(): current = getattr(group, name) if isinstance(current, _FlagGroupBase): _restore_leaves(current, value) elif isinstance(current, dict): current.clear() current.update(value) elif isinstance(current, list): current[:] = value else: setattr(group, name, value) def snapshot_context() -> dict[str, Any]: """Everything a publish replaces, so a failed launch can put it back. Enumerated from ``__slots__`` rather than listed by hand: a hand-picked copy of context state is one field behind the day a slot is added, and the copy that silently drops one is worse than none. Flag groups are snapshotted by leaf, not by reference: publish writes *into* the same ``Flags`` object (``capture.enable_torch_compile``), so a reference held here would already carry the failed launch's value by the time it is put back. """ state: dict[str, Any] = {} for name in RuntimeContext.__slots__: if name == "parallel": continue value = getattr(_CONTEXT, name) if isinstance(value, _FlagGroupBase): state[name] = (value, _group_leaves(value)) elif isinstance(value, list): state[name] = list(value) else: state[name] = value state["__parallel__"] = { name: getattr(_CONTEXT.parallel, name) for name in type(_CONTEXT.parallel).__slots__ } state["__dwdp__"] = get_global_dwdp_manager() return state def restore_context(state: dict[str, Any]) -> None: """Put back what ``snapshot_context`` captured.""" for name in RuntimeContext.__slots__: if name == "parallel": continue value = state[name] if isinstance(value, tuple) and isinstance(value[0], _FlagGroupBase): group, leaves = value setattr(_CONTEXT, name, group) _restore_leaves(group, leaves) else: setattr(_CONTEXT, name, value) for name, value in state["__parallel__"].items(): setattr(_CONTEXT.parallel, name, value) set_global_dwdp_manager(state["__dwdp__"]) def reset_context() -> None: """Clear published configuration, parallel overrides, flags, and resources. Used for test teardown and runtime lifecycle reset. """ _CONTEXT._server_args = None _CONTEXT._config_bags = None _CONTEXT._overrides_log = [] _CONTEXT._publish_role = None _CONTEXT.parallel._config = None _CONTEXT.parallel.clear_stamp() _CONTEXT.flags = Flags() _CONTEXT.resources = Resources() _CONTEXT.forward = ForwardFlags() set_global_dwdp_manager(None) def remote_instance_transfer_engine_enabled(load_format: str | None = None) -> bool: """Check transfer-engine loading against current model config. ``load_format`` can override the process value for a draft runner. """ from sglang.srt.arg_groups.overrides import remote_instance_transfer_engine_of return remote_instance_transfer_engine_of(get_model(), load_format) def max_prefill_buffer_tokens() -> int: """Return the registered prefill ceiling, or derive it from current config. PP dynamic chunking can grow toward ``max_prefill_tokens`` and probe at 1.25x. """ schedule = get_schedule() chunked = ( schedule.chunked_prefill_size if schedule.chunked_prefill_size and schedule.chunked_prefill_size > 0 else 0 ) tokens = chunked if schedule.enable_dynamic_chunking and get_parallel().pp_size > 1 and chunked: tokens = max( tokens, schedule.max_prefill_tokens or 0, math.ceil(chunked * 1.25) ) return prefill_buffer_ceiling_of(get_server_args(), tokens) def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float: """Return the activation reserve in MB before CUDA graph capture, using current config.""" schedule = get_schedule() if get_disagg().disaggregation_mode == "decode": running_requests = ( schedule.max_running_requests or get_exec().graph.cuda_graph_config.decode.max_bs or 1 ) activation_tokens = max( running_requests * (get_spec().speculative_num_draft_tokens or 1), 2048 ) elif schedule.chunked_prefill_size > 0: activation_tokens = max(schedule.chunked_prefill_size, 2048) else: activation_tokens = max(schedule.max_prefill_tokens, 2048) parallel = get_parallel() reserved_mem = ( 512 + activation_tokens * 1.5 + parallel.tp_size * parallel.pp_size / 8 * 1024 ) if gpu_mem is not None and gpu_mem > 60 * 1024: reserved_mem = max(reserved_mem, 10 * 1024) return reserved_mem # Platform probes work before publication and support scoped overrides. _PLATFORM_PROBES: Dict[str, str] = { "is_cuda": "is_cuda", "is_hip": "is_hip", "is_npu": "is_npu", "is_xpu": "is_xpu", "is_musa": "is_musa", "is_mps": "is_mps", "is_sm90": "is_sm90_supported", "is_sm100": "is_sm100_supported", "is_sm100_or_sm110": "is_sm100_or_sm110_supported", "is_sm120": "is_sm120_supported", "is_blackwell": "is_blackwell_supported", "is_hopper_with_cuda_12_3": "is_hopper_with_cuda_12_3", "has_amx": "cpu_has_amx_support", "has_flashinfer": "is_flashinfer_available", } # Not yes/no facts, same address. _PLATFORM_VALUES: Dict[str, str] = { "device_sm": "get_device_sm", "device_capability": "get_device_capability", } class PlatformContext: """The machine's own facts, with one place to override them. Every name maps to a probe in `utils.common`; the probes are `lru_cache`-d, so reading through here costs a call and a dict lookup (~26 ns) rather than a device query. """ __slots__ = ("_overrides",) def __init__(self) -> None: object.__setattr__(self, "_overrides", {}) def __getattr__(self, name: str) -> Any: probe = _PLATFORM_PROBES.get(name) or _PLATFORM_VALUES.get(name) if probe is None: known = sorted(set(_PLATFORM_PROBES) | set(_PLATFORM_VALUES)) raise AttributeError( f"unknown platform fact {name!r}; known: {', '.join(known)}" ) overrides = object.__getattribute__(self, "_overrides") if name in overrides: return overrides[name] from sglang.srt.utils import common as _common return getattr(_common, probe)() def __setattr__(self, name: str, value: Any) -> None: raise AttributeError( "platform facts are not assigned; use " "`sglang.srt.runtime_context.override_platform(...)` so every " "reader agrees" ) def _install(self, **facts: Any) -> Dict[str, Any]: unknown = set(facts) - set(_PLATFORM_PROBES) - set(_PLATFORM_VALUES) if unknown: raise ValueError(f"unknown platform fact(s): {sorted(unknown)}") overrides = object.__getattribute__(self, "_overrides") previous = {k: overrides[k] for k in facts if k in overrides} missing = [k for k in facts if k not in overrides] overrides.update(facts) return {"previous": previous, "missing": missing} def _restore(self, saved: Dict[str, Any]) -> None: overrides = object.__getattribute__(self, "_overrides") overrides.update(saved["previous"]) for k in saved["missing"]: overrides.pop(k, None) _PLATFORM = PlatformContext() def get_platform() -> PlatformContext: """The machine's facts. Answers before publish, unlike a config bag.""" return _PLATFORM class _PlatformOverride: """Scoped platform override: `with override_platform(is_sm100=True): ...`""" __slots__ = ("_facts", "_saved") def __init__(self, **facts: Any) -> None: self._facts = facts self._saved = None def install(self) -> PlatformContext: self._saved = _PLATFORM._install(**self._facts) return _PLATFORM def restore(self) -> None: if self._saved is not None: _PLATFORM._restore(self._saved) self._saved = None def __enter__(self) -> PlatformContext: return self.install() def __exit__(self, *exc: Any) -> None: self.restore() def __call__(self, fn: Any) -> Any: """Also usable as a decorator, like the `patch` it replaces. A fresh scope per call: the same object decorating two tests must not share one saved state. """ import functools facts = dict(self._facts) @functools.wraps(fn) def wrapper(*args: Any, **kwargs: Any) -> Any: with _PlatformOverride(**facts): return fn(*args, **kwargs) return wrapper def override_platform(**facts: Any) -> _PlatformOverride: """Say what kind of machine this is, once, for every reader.""" return _PlatformOverride(**facts) # Derived config accessors use the process model (the target during speculation). def mamba_cache_chunk_size() -> int: """The caching point granularity for mamba state: ``max(the model's mamba chunk size, page_size)``. Cached on the config after the first call.""" from sglang.srt.arg_groups.overrides import mamba_cache_chunk_size as _of return _of(get_server_args()) def mamba_checkpoint_grid(tree_page: int) -> int: """The granularity a donated mamba checkpoint's depth must land on so the radix tree can name it. Pass the page the tree actually allocates on: DCP widens it past ``mamba_cache_chunk_size``, and deriving that here would be a second copy of a predicate that already lives in the cache builder.""" return math.lcm(mamba_cache_chunk_size(), tree_page) def mamba_track_grid(tree_page: int) -> int: """The same granularity for a decode-donated checkpoint, which additionally has to land on the requested ``mamba_track_interval``.""" return math.lcm( mamba_checkpoint_grid(tree_page), get_exec().mamba.mamba_track_interval ) def max_speculative_num_draft_tokens() -> int | None: """The largest draft-token count speculative decoding may use. Adaptive algorithms may switch to a longer state after the scheduler reserves KV for the next decode batch, so include the capacity captured when the resolved configuration was published. """ spec = get_spec() return max( ( bound for bound in ( spec.speculative_num_draft_tokens, spec.max_speculative_num_draft_tokens, ) if bound is not None ), default=None, ) def uses_mla_backend() -> bool: """Whether this process's model runs the MLA attention path.""" from sglang.srt.arg_groups.overrides import use_mla_backend return use_mla_backend(get_server_args()) def attention_backends() -> tuple: """The configured ``(prefill, decode)`` backend pair, split fields falling back to ``attention_backend``. All three inputs are ``exec.kernel`` leaves, so this derives from the bags and follows a post-publish override; ``overrides.attention_backends_of`` is the pre-publish equivalent the resolution pipeline uses. A built runner stamps its own resolved pair (``ModelRunner.prefill_attention_backend_str``); read that when there is a runner in hand. """ from sglang.srt.arg_groups.overrides import attention_backends_of return attention_backends_of(get_exec().kernel) def process_model_config(): """The process's ``ModelConfig`` (built once from the published config).""" from sglang.srt.arg_groups.overrides import model_config_of return model_config_of(get_server_args()) def reports_expert_balancedness() -> bool: """Whether the expert-balancedness report is on at all. `overrides.should_report_expert_balancedness` is the pre-publish equivalent. """ return get_exec().moe.expert_balancedness_report_mode != "off" def logs_expert_balancedness_to_server_log() -> bool: """Whether the balancedness report goes to the server log.""" return get_exec().moe.expert_balancedness_report_mode in ("server_log", "both") def exports_expert_balancedness_to_prometheus() -> bool: """Whether the balancedness report goes to Prometheus.""" return get_exec().moe.expert_balancedness_report_mode in ("prometheus", "both") def cutedsl_moe_max_num_tokens() -> int: """Largest token count one forward routes through a CuteDSL MoE layer on one DP rank; sizes the standard-allgather wrapper, MegaMoE, and AR fusion buffers. Every input is a published leaf (``spec``, ``schedule``, ``exec.graph``), so this follows a post-publish override. Max over the prefill bound, the piecewise-prefill capture, and the decode/verify bound. """ from sglang.srt.model_executor.cuda_graph_config import Backend spec = get_spec() num_tokens_per_req = ( (spec.speculative_num_draft_tokens or 1) if spec.speculative_algorithm else 1 ) prefill_tokens = get_schedule().max_prefill_tokens cg_config = get_exec().graph.cuda_graph_config if cg_config is not None and cg_config.prefill.backend == Backend.TC_PIECEWISE: prefill_tokens = max(prefill_tokens, cg_config.prefill.max_bs or 0) decode_max_bs = (cg_config.decode.max_bs if cg_config is not None else 0) or 0 return max(prefill_tokens, decode_max_bs * num_tokens_per_req) def describe_kv_events_publisher(server_args: Any) -> Optional[dict]: """Return a structured description of this server's KV-event publisher, or `None` if publishing is disabled / misconfigured. This is the wire contract surfaced under the `kv_events` key on `/server_info` so KV-aware routers (e.g. the SGLang model gateway) can subscribe per-worker without operator-supplied port coordination. The router constructs the per-DP-rank SUB endpoint as tcp://: for every rank reported in dp_size. Returned descriptor shape: { "publisher": "zmq", "endpoint_host": "*", # may be a ZMQ wildcard # ("*", "0.0.0.0", "::"); # subscribers MUST substitute # the worker URL's host when # dialing "endpoint_port_base": 5557, # base TCP port; per-rank # port = base + dp_rank "topic": "", # ZMQ topic prefix on the # SUB filter (empty = # subscribe-all) "block_size": , # subscribers MUST # hash prompts at this size "dp_size": , # number of SUB sockets to # open; not DCP-scaled, as # DCP shards within a rank # rather than adding # publishers "load_endpoint_port_base": , # base TCP port of the load # range (load rank r = base # + r). Consumers MUST read # this key, not re-derive # it; present only when # --load-publish-endpoint # opted in and a range # resolved "load_topic": "load", # SUB filter for the load # socket; present iff # load_endpoint_port_base # is present "replay_endpoint_port_base": 5558, # ROUTER replay port; rank r # = base + r, same host rule # as the SUB endpoints; # present only when # replay_endpoint is tcp } Returns None (i.e. "no publisher to describe") when any of: * --kv-events-config is unset / empty / malformed JSON, * the configured publisher is "null", * page_size is missing or non-positive (a placeholder block_size would cause silent KV-cache misses by hashing prompts at the wrong granularity on the router side), * the endpoint is not a routable TCP address (inproc:// / ipc://, missing port, non-integer port, port outside 1..65535, or a bare unbracketed IPv6 host, which is ambiguous). NOTE for load-socket consumers: pair the load port with the worker's own URL host, as with the KV SUB endpoints — endpoint_host is a wildcard ("*", "0.0.0.0", "::") whenever the default packing applies, so splicing it yields tcp://*:PORT and connects to nothing. Reuses parse_advertisable_tcp and resolve_load_pub_range — the same helpers the scheduler binds through — so the advertisement cannot drift from the sockets. """ from sglang.srt.arg_groups.overrides import kv_event_block_size_of, resolving_view # Lazy import so loading server_args doesn't pull in # disaggregation / msgspec / zmq at module top level. from sglang.srt.disaggregation.kv_events import ( LOAD_TOPIC, KVEventsConfig, parse_advertisable_tcp, resolve_load_pub_range, ) resolved = resolving_view(server_args) raw = resolved.kv_events_config page_size = resolved.page_size if not raw or page_size is None or page_size <= 0: return None try: cfg = KVEventsConfig.from_cli(raw) except Exception: # Malformed JSON / schema mismatch. The publisher would # have failed at server startup; /server_info must # keep working, so just report "no publisher" to consumers. return None if cfg.publisher == "null" or not cfg.endpoint: return None resolved_kv = parse_advertisable_tcp(cfg.endpoint) if resolved_kv is None: return None host, port = resolved_kv descriptor = { "publisher": cfg.publisher, "endpoint_host": host, "endpoint_port_base": port, "topic": cfg.topic, "block_size": kv_event_block_size_of(resolved), # The wire key predates attention DP; it counts DP ranks. "dp_size": num_dp_ranks_of(resolved), } # Load range, from the same resolver SchedulerLoadPublisher binds # with (so the two can't drift). The decline reason is logged once at # startup, not here — this runs per /server_info request. resolved_range, _reason = resolve_load_pub_range( kv_endpoint=cfg.endpoint, replay_endpoint=cfg.replay_endpoint, dp_size=num_dp_ranks_of(resolved), load_publish_endpoint=resolved.load_publish_endpoint, ) if resolved_range is not None: descriptor["load_endpoint_port_base"] = resolved_range[1] descriptor["load_topic"] = LOAD_TOPIC resolved_replay = parse_advertisable_tcp(cfg.replay_endpoint) if resolved_replay is not None: descriptor["replay_endpoint_port_base"] = resolved_replay[1] return descriptor