[Refactor] Trim server configuration and runtime context comments (#40971)

This commit is contained in:
Cheng Wan
2026-09-23 14:47:13 -07:00
committed by GitHub
parent c39a1c0663
commit f766397f2e
53 changed files with 234 additions and 1177 deletions
+18 -73
View File
@@ -72,23 +72,12 @@ class Arg(msgspec.Struct, frozen=True):
action: Any | None = None
action_kwargs: dict | None = None
const: Any | None = None
# When True, this field is skipped by add_cli_args_from_dataclass.
# Use for fields that have no CLI surface (e.g. injected via Python only).
# Skip automatic CLI registration.
no_cli: bool = False
# When True, config resolution (model overrides and post-process passes)
# may decide this field: the declaration stash accepts the name, and
# `resolution_result` and the config bags answer with the decision. The
# field keeps what the operator passed.
# Allow resolution passes to declare a value for this field.
resolvable: bool = False
# What the field means when nobody said anything -- the bottom of the read
# chain: override, decision, input, then this. Not the dataclass default,
# which stays `None` because that is how the record spells "not typed".
# `None` here means the field declares no fallback, which is the same
# answer the read chain gives without one.
#
# Only a value fixed for the life of the configuration belongs here. One
# that depends on the machine, on another field, or on anything impure is a
# decision, and decisions stay in a hook where their order is visible.
# Scalar fallback when neither resolution nor input supplies a value.
# Machine- or config-dependent defaults belong in resolution hooks.
fallback: Any = None
@@ -112,44 +101,25 @@ class Derived(msgspec.Struct, frozen=True):
class NS(msgspec.Struct, frozen=True):
"""Namespace-path marker for a ServerArgs field, attached alongside the
field's metadata in ``Annotated``:
"""Per-field namespace marker for records spanning multiple namespaces.
field: A[int, "help", NS("parallel")] = 1
field: A[str, Arg(help="…"), NS("exec.moe")] = "auto"
``ServerArgs`` no longer uses it: its fields are declared in the
``arg_groups/fields/`` classes, each of which carries the ``_NS_PATH`` it
stands for, so the module a declaration lives in *is* its namespace. What
is left for this marker is the case a class cannot express -- one ad-hoc
dataclass whose fields span several namespaces, which is what the
config-bag tests build."""
Example: ``field: A[int, "help", NS("parallel")] = 1``.
ServerArgs instead collects each namespace class's ``_NS_PATH``.
"""
path: str
@functools.cache
def namespace_of(cls) -> dict:
"""``{field_name: dotted namespace path}``, read from the declaring class.
"""Return ``{field: namespace}`` from collected metadata, class paths, or NS markers.
A field's namespace is where it is declared: each class in
``arg_groups/fields/`` carries the ``_NS_PATH`` it stands for, and
``ServerArgs`` composes them. Walking the MRO therefore answers "which
namespace owns this field" without a per-field marker -- the file the
declaration sits in is the marker.
A class that is not built that way -- an ad-hoc dataclass spanning several
namespaces, which is what the config-bag tests construct -- falls back to
the per-field ``NS`` marker. A field with neither is absent from the map
(the coverage lint flags them). Non-dataclass types yield an empty map.
Unmarked fields are omitted; non-record types yield an empty map.
"""
if not is_record(cls):
return {}
# An assembled record: the collector recorded who declared each field,
# because there are no base classes left to ask.
out = dict(getattr(cls, "_NS_BY_FIELD", None) or {})
# A class that still inherits its namespaces: nearest declaration wins, so
# walk the MRO front to back and keep the first answer.
# Nearest namespace declaration wins.
for base in cls.__mro__:
path = base.__dict__.get("_NS_PATH")
if path is None:
@@ -199,12 +169,7 @@ def resolvable_fields(cls) -> frozenset:
@functools.cache
def fallbacks_of(cls) -> dict:
"""``{field_name: value}`` for every field of ``cls`` that declares one.
Read the same way `resolvable_fields` reads its flag, so a fallback lives
beside the help text of the field it belongs to rather than in whatever
hook used to fill it in.
"""
"""Return declared scalar fallbacks for fields whose input default is None."""
if not is_record(cls):
return {}
hints = get_type_hints(cls, include_extras=True)
@@ -212,11 +177,7 @@ def fallbacks_of(cls) -> dict:
for field in record_fields(cls):
_, arg = _unwrap_annotated(hints.get(field.name, field.type))
if arg is not None and arg.fallback is not None:
# Two things `with_fallback` relies on and cannot check itself,
# asserted where a new declaration passes through. This function is
# cached, so a mutable fallback would hand one shared object to
# every reader; and a field whose dataclass default is not `None`
# can never reach the fallback, which makes the declaration dead.
# Reject shared mutable fallbacks and defaults that make the fallback unreachable.
assert not isinstance(arg.fallback, (list, dict, set)), (
f"{cls.__name__}.{field.name}: a mutable fallback would be "
"shared by every reader -- use a scalar"
@@ -230,16 +191,10 @@ def fallbacks_of(cls) -> dict:
def with_fallback(cls, name: str, value: Any) -> Any:
"""``value``, or the declared fallback when nothing has answered.
"""Apply a declared fallback when ``value`` is None.
`resolution_result` calls this as its last step -- the effective surface,
which the config bags and `/server_info` read through. Deliberately not the
views a pass reads *while deciding*: `model_overrides/inkling.py` branches
on `if cfg.swa_full_tokens_ratio is None`, and a fallback answering there
would make that branch dead. `test_declared_fallbacks.py` pins both halves.
A mutable fallback would need copying per read, for the reason a dataclass
spells this `default_factory`. Every declared one is a scalar.
Used by ``resolution_result``, but not mid-resolution views: handlers must
still distinguish unset inputs. Fallbacks must be immutable scalars.
"""
if value is not None:
return value
@@ -247,12 +202,7 @@ def with_fallback(cls, name: str, value: Any) -> Any:
def record_fields(cls):
"""The declared fields of a record, Struct or dataclass.
`ServerArgs` and the namespace classes are `msgspec.Struct`; the config-bag
tests build ad-hoc dataclasses spanning namespaces, and the helpers here are
driven with both. Anything else yields nothing.
"""
"""Return Struct or dataclass fields, or an empty tuple for other types."""
if isinstance(cls, type) and issubclass(cls, msgspec.Struct):
return msgspec.structs.fields(cls)
if dataclasses.is_dataclass(cls):
@@ -326,12 +276,7 @@ def _infer_type_func(tp):
def _field_default(field):
"""Return the default value for a field, or `_MISSING`.
The two record shapes spell "no default" differently -- a Struct field says
`msgspec.NODEFAULT`, a dataclass field `dataclasses.MISSING` -- so both are
normalized here and every caller below tests against `_MISSING` alone.
"""
"""Return a field default, normalizing Struct and dataclass missing sentinels."""
absent = (_MISSING, msgspec.NODEFAULT)
if field.default not in absent:
return field.default
+1 -7
View File
@@ -1,10 +1,4 @@
"""Enumerated choices shared by the config field declarations.
These lived in ``server_args.py`` beside the fields that name them. The fields
moved to ``arg_groups/fields/``, and ``server_args`` imports the field modules,
so the lists cannot stay there without a cycle. ``server_args`` re-exports them
for the handful of modules that import them from their old home.
"""
"""Shared config choices, re-exported by server_args for plugin compatibility."""
LOAD_FORMAT_CHOICES = [
"auto",
@@ -21,16 +21,7 @@ logger = logging.getLogger(__name__)
def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None:
"""Residual imperative arm of the DeepSeek V4 defaults.
The attention/page/window/MoE-runner declarations moved to the override
registry (arg_groups/overrides.py: _deepseek_v4_overrides) and the
kv-cache dtype default to the resolution pipeline
(_deepseek_v4_kv_cache_dtype, invoked below at its legacy slot). This
keeps, at the legacy slot: the ROCm env fill (env-write policy), the
max_running_requests fill (the speculative hook is a later writer of
that field) and the validations.
"""
"""Apply DeepSeek V4 environment defaults, request limits, and validation."""
cfg = resolving_view(server_args)
# FlashMLA sparse prefill (SGLANG_OPT_FLASHMLA_SPARSE_PREFILL, default on)
+2 -10
View File
@@ -1,14 +1,6 @@
"""The order ``ServerArgs`` presents its fields in, frozen.
"""Field order preserving the ServerArgs positional constructor signature.
A dataclass turns field order into a positional constructor signature, so
``ServerArgs(model_path, tokenizer_path)`` has to keep meaning what it means.
Grouping the declarations by namespace would move the second argument onto
another field, silently.
A compatibility record and nothing else -- a field's namespace is the module it
is declared in, and only ``collect_input_fields`` reads this. A name that is not
here sorts after every name that is, which is the only backward-compatible
position for a new field anyway.
``collect_input_fields`` appends unlisted fields after these entries.
"""
# fmt: off
@@ -1,9 +1,7 @@
"""Config field declarations, one module per top-level namespace.
"""Namespace declarations for input and derived configuration.
Each class carries the ``_NS_PATH`` it stands for, so the module a field is
declared in *is* its namespace. ``ServerArgs`` is assembled from the classes
that declare **operator input**; the derived fields, which nobody can type,
are declared beside them but are not collected into the record.
``ServerArgs`` collects input fields from classes carrying ``_NS_PATH``.
Derived declarations stay on those classes for publication into config bags.
"""
from __future__ import annotations
@@ -20,30 +18,11 @@ from sglang.srt.arg_groups.field_order import POSITIONAL_FIELD_ORDER
def collect_input_fields(
sources: List[type],
) -> Tuple[Dict[str, Any], Dict[str, Any], Dict[str, str]]:
"""The annotations and defaults of every field these classes declare.
"""Return resolved annotations, defaults, and a field-to-namespace map.
Each source's annotations are resolved **in its own module** and handed on
as type objects. Strings never travel: an annotation carried across as text
would be re-evaluated where it lands, and the composing module does not
import the names the declarations use -- that is the whole point of them
living where they do.
Returns the resolved annotations, the defaults, and ``{field: namespace}``
-- the last because the assembled record has no base classes to read a
``_NS_PATH`` off, and the namespace is still the class that declared it.
Which classes are passed here is the record's contract: the record holds
what the operator asked for, so a class of derived fields is simply not in
the list, and the rule is one readable call rather than an invariant spread
across a base-class list.
The result is ordered by ``POSITIONAL_FIELD_ORDER``, not by namespace: a
dataclass turns field order into a positional constructor signature, and
grouping fields that used to be interleaved would silently move
``ServerArgs(model_path, tokenizer_path)``'s second argument onto another
field. Anything the record declares that the frozen order does not name
goes after it, in declaration order -- the only backward-compatible place
for a new field.
Resolve annotations in their declaring module so imported type names remain
available. Preserve ``POSITIONAL_FIELD_ORDER`` for constructor compatibility;
append new fields in declaration order.
"""
annotations: Dict[str, Any] = {}
defaults: Dict[str, Any] = {}
@@ -1,11 +1,4 @@
"""Config fields of the ``device`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``device`` bag, which is what ``get_device()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``device`` namespace."""
from __future__ import annotations
@@ -1,11 +1,4 @@
"""Config fields of the ``disagg`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``disagg`` bag, which is what ``get_disagg()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``disagg`` namespace."""
from __future__ import annotations
+1 -8
View File
@@ -1,11 +1,4 @@
"""Config fields of the ``exec`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``exec`` bag, which is what ``get_exec()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``exec`` namespace."""
from __future__ import annotations
+1 -8
View File
@@ -1,11 +1,4 @@
"""Config fields of the ``lora`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``lora`` bag, which is what ``get_lora()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``lora`` namespace."""
from __future__ import annotations
@@ -1,11 +1,4 @@
"""Config fields of the ``memory`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``memory`` bag, which is what ``get_memory()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``memory`` namespace."""
from __future__ import annotations
+1 -8
View File
@@ -1,11 +1,4 @@
"""Config fields of the ``mm`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``mm`` bag, which is what ``get_mm()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``mm`` namespace."""
from __future__ import annotations
+1 -8
View File
@@ -1,11 +1,4 @@
"""Config fields of the ``model`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``model`` bag, which is what ``get_model()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``model`` namespace."""
from __future__ import annotations
@@ -1,11 +1,4 @@
"""Config fields of the ``observability`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``observability`` bag, which is what ``get_observability()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``observability`` namespace."""
from __future__ import annotations
@@ -1,11 +1,4 @@
"""Config fields of the ``parallel`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``parallel`` bag, which is what ``get_parallel()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``parallel`` namespace."""
from __future__ import annotations
@@ -1,11 +1,4 @@
"""Config fields of the ``schedule`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``schedule`` bag, which is what ``get_schedule()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``schedule`` namespace."""
from __future__ import annotations
@@ -1,11 +1,4 @@
"""Config fields of the ``serving`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``serving`` bag, which is what ``get_serving()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``serving`` namespace."""
from __future__ import annotations
+1 -8
View File
@@ -1,11 +1,4 @@
"""Config fields of the ``spec`` namespace.
One class per namespace. The class *is* the namespace: a field declared here
lands in the ``spec`` bag, which is what ``get_spec()`` returns, so a reader
spells it exactly as before. ``ServerArgs`` composes these classes, so the
record stays one flat object -- the split moves where declarations live, not
how config is shaped at runtime.
"""
"""Config fields of the ``spec`` namespace."""
from __future__ import annotations
@@ -1,11 +1,7 @@
"""What the per-model override declarations are written against.
"""Registry and read views shared by model override providers.
The declarations themselves live one directory down, in
``arg_groups/model_overrides/``: one module per model family, mirroring the
``models/`` naming. This module is what they all import -- the registry they
register into, the read-only views they are handed, and the few accessors that
answer questions about the model. It deliberately depends on nothing in
``overrides.py``, so a family module never has to import its way back up.
Model-family modules in ``model_overrides/`` import this module without
creating a dependency back to ``overrides.py``.
"""
import logging
@@ -18,10 +14,9 @@ from sglang.srt.utils.common import is_mps, is_no_spec_infer_or_topk_one
logger = logging.getLogger(__name__)
# Constant per-architecture overrides (populated by the migration sweeps).
# Constant per-architecture overrides.
MODEL_OVERRIDES: Dict[str, Dict[str, Any]] = {
# These models run in bfloat16 regardless of the requested dtype
# (faithful port of the legacy unconditional arch branch).
# These models require bfloat16 regardless of the requested dtype.
"MistralLarge3ForCausalLM": {"dtype": "bfloat16"},
"PixtralForConditionalGeneration": {"dtype": "bfloat16"},
}
@@ -31,9 +26,7 @@ MODEL_OVERRIDES: Dict[str, Dict[str, Any]] = {
_MODEL_OVERRIDE_FNS: Dict[str, List[Callable[..., dict]]] = {}
# Predicate-keyed providers, in registration order — for legacy branches
# matched by substring/predicate on the architecture string rather than an
# exact name (e.g. '"Step3p5ForCausalLM" in model_arch').
# Predicate-keyed providers, in registration order.
_PREDICATE_OVERRIDE_FNS: List[Tuple[Callable[[str], bool], Callable[..., dict]]] = []
@@ -78,12 +71,7 @@ def _invoke_provider(
class ResolvedView:
"""Read-only view of the resolving configuration handed to post-process
passes: the accumulated declarations overlaid on the pristine
``server_args`` (residual imperative writes of non-resolved fields show
through the fallthrough) — exactly the state the legacy handler at the
same slot observed. Writes are rejected: passes return declarations.
"""
"""Read-only snapshot of declarations over raw inputs, used by post-process passes."""
__slots__ = ("_server_args", "_overlay")
@@ -104,14 +92,9 @@ class ResolvedView:
class ResolvingConfig:
"""Live read view of the resolution result: the declaration stash over the
record's fields, looked up per read.
"""Live view of declarations over raw inputs.
``ResolvedView`` snapshots the overlay when it is built, which is what a
post-process pass wants -- it reads the state at its slot. A resolver that
reads *after* declaring, or after calling something that declares, needs the
current answer instead, so this one walks the stash on every read. It falls
through to the field, which is where the raw input lives.
Unlike ``ResolvedView``, each read observes declarations made since construction.
"""
__slots__ = ("_server_args",)
@@ -140,10 +123,7 @@ def resolving_view(server_args: Any) -> ResolvingConfig:
def _declaration_overlay(server_args: Any) -> Dict[str, Any]:
"""What the declarations say so far, last writer wins.
Nothing writes the fields, so a mid-resolution reader needs this to see a
decision at all; the fields keep what the caller supplied."""
"""Merge declarations in order, with the last writer winning."""
overlay: Dict[str, Any] = {}
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
overlay.update(declared)
@@ -151,13 +131,7 @@ def _declaration_overlay(server_args: Any) -> Dict[str, Any]:
def resolved_view(server_args: Any) -> ResolvedView:
"""Read-only view of the resolving configuration: the declarations
overlaid on the fields, snapshotted per call.
For mid-resolution code that is not a pass (``__post_init__`` handlers and
hooks) that must answer with what resolution decided -- a declaration-only resolver (a model-specific
override, a registry entry) never writes the field, so a field read there
answers with the raw input."""
"""Snapshot current declarations over the record fields in a read-only view."""
return ResolvedView(server_args, overlay=_declaration_overlay(server_args))
@@ -200,12 +174,7 @@ def record_of(view: Any) -> Any:
def is_attention_backend_not_set(cfg: Any):
"""None of the three attention backends has been decided yet.
Takes the view rather than the record: every read is a view read, and the
callers that hold a view (the override providers) would otherwise have to
reach back through it for a record.
"""
"""Return whether all three attention backends are unset in the resolving view."""
return (
cfg.attention_backend is None
and cfg.prefill_attention_backend is None
@@ -221,12 +190,7 @@ def use_mla_backend(server_args: Any):
def model_config_of(server_args: Any):
"""The model configuration this record describes, built once and memoised.
Takes a view as readily as the record: a view is a read overlay of one
record, the memo has to live on that record either way, and the callers
that hold a view would otherwise all have to unwrap it themselves.
"""
"""Build and memoize ModelConfig on the underlying record, accepting records or views."""
if isinstance(server_args, (ResolvedView, ResolvingConfig)):
server_args = record_of(server_args)
# Lazy init to avoid circular import
@@ -258,13 +222,7 @@ def model_config_of(server_args: Any):
def ep_joiner_of(cfg: Any) -> bool:
"""Whether this process was launched as an elastic-EP joiner.
The one definition. After publish the answer is a bag leaf --
`get_exec().moe.is_ep_joiner` -- computed from this function by the
declaration in `arg_groups/fields/exec_.py`; resolution needs it before
there is a bag to read, which is why it is still a function.
"""
"""Return whether this process was launched as an elastic-EP joiner."""
return cfg.ep_join_mode in ("scale", "recover")
@@ -279,13 +237,7 @@ def startup_weight_load_overlap_of(cfg: Any) -> bool:
def mamba_extra_buffer_of(cfg: Any) -> bool:
"""The predicate, read off a config-shaped object mid-resolution.
This is the one definition. After publish the answer is a bag leaf --
``get_exec().mamba.enable_mamba_extra_buffer`` -- computed from this same
function by the declaration in ``arg_groups/fields/exec_.py``. Resolution
needs it before there is a bag to read, which is why it is still a
function."""
"""Return whether the resolved Mamba cache strategy requires an extra buffer."""
return cfg.disable_radix_cache is False and cfg.mamba_radix_cache_strategy in (
"extra_buffer",
"extra_buffer_lazy",
@@ -1,8 +1,4 @@
"""Config-time override declarations for bailing_moe_v3.
Architectures: BailingMoeV3ForCausalLM,
BailingMoeV3VLForConditionalGeneration.
"""
"""Config-time override declarations for bailing_moe_v3."""
import logging
from typing import Any
@@ -1,7 +1,4 @@
"""Config-time override declarations for cohere2_moe.
Architectures: Cohere2MoeForCausalLM, Cohere2VisionForConditionalGeneration.
"""
"""Config-time override declarations for cohere2_moe."""
import logging
from typing import Any
@@ -1,7 +1,4 @@
"""Config-time override declarations for deepseek_v2.
Architectures: DeepseekV32ForCausalLM, DeepseekV3ForCausalLM, Dots3NoteForCausalLM, Glm5NextForConditionalGeneration, GlmMoeDsaForCausalLM, HYV4ForCausalLM, HYV4ForCausalLMNextN, KimiK25ForConditionalGeneration, LongcatFlashForCausalLM, LongcatFlashForCausalLMNextN, MistralLarge3ForCausalLM, PixtralForConditionalGeneration.
"""
"""Config-time override declarations for deepseek_v2."""
import logging
from typing import Any, Dict
@@ -32,11 +29,7 @@ logger = logging.getLogger(__name__)
"Dots3NoteForCausalLM",
)
def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
"""Order-safe declarations of the DeepSeek/DSA branch. The CP parallel
writes (enable_dp_attention/ep_size/moe_a2a_backend have post-monolith
writers), the kv-cache/split-backend defaults, the quant/moe block (read
before it by _set_default_dsa_kv_cache_dtype) and the env writes stay in
the branch."""
"""Declare DeepSeek/DSA defaults; ordered CP, KV-cache, and MoE passes run in model_hook."""
cfg = resolving_view(server_args)
from sglang.srt.configs.model_config import (
is_deepseek_dsa,
@@ -1,7 +1,4 @@
"""Config-time override declarations for deepseek_v4.
Architectures: DeepseekV4ForCausalLM.
"""
"""Config-time override declarations for deepseek_v4."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for exaone.
Architectures: Exaone4ForCausalLM, ExaoneMoEForCausalLM.
"""
"""Config-time override declarations for exaone."""
import logging
from typing import Any
@@ -1,7 +1,4 @@
"""Config-time override declarations for falcon_h1.
Architectures: FalconH1ForCausalLM, JetNemotronForCausalLM, JetVLMForConditionalGeneration.
"""
"""Config-time override declarations for falcon_h1."""
from typing import Any
@@ -1,7 +1,4 @@
"""Config-time override declarations for gemma2_gemma3.
Architectures: Gemma2ForCausalLM, Gemma3ForCausalLM, Gemma3ForConditionalGeneration, Gemma3nForCausalLM, Gemma3nForConditionalGeneration.
"""
"""Config-time override declarations for gemma2_gemma3."""
import logging
from typing import Any
@@ -1,7 +1,4 @@
"""Config-time override declarations for gemma4.
Architectures: Gemma4ForCausalLM, Gemma4ForConditionalGeneration, Gemma4UnifiedForConditionalGeneration.
"""
"""Config-time override declarations for gemma4."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for gigachat35.
Architectures: GigaChat35ForCausalLM, GigaChat35ForCausalLMNextN.
"""
"""Config-time override declarations for gigachat35."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for glm4_moe.
Architectures: Glm4MoeForCausalLM.
"""
"""Config-time override declarations for glm4_moe."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for gpt_oss.
Architectures: GptOssForCausalLM.
"""
"""Config-time override declarations for gpt_oss."""
import logging
from typing import Any, Dict
@@ -46,9 +43,7 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict:
# rather than landing on torch_native (no sliding window, no sinks).
overrides["attention_backend"] = "triton"
if get_platform().is_xpu:
# Check for bf16 dtype on Intel XPU. Reads the pristine dtype request,
# which equals the legacy mid-branch read: dtype had no earlier writer
# for this arch.
# Intel XPU requires bfloat16.
if cfg.dtype == "auto":
logger.warning(
"GptOssForCausalLM on Intel XPU currently supports bfloat16 dtype only"
@@ -1,7 +1,4 @@
"""Config-time override declarations for granitemoehybrid.
Architectures: GraniteMoeHybridForCausalLM.
"""
"""Config-time override declarations for granitemoehybrid."""
from typing import Any
@@ -1,7 +1,4 @@
"""Config-time override declarations for inkling.
Architectures: InklingForConditionalGeneration, InklingForConditionalGenerationMTP.
"""
"""Config-time override declarations for inkling."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for interns2_mobius.
Architectures: InternS2MobiusForConditionalGeneration.
"""
"""Config-time override declarations for interns2_mobius."""
from typing import Any
@@ -1,7 +1,4 @@
"""Config-time override declarations for kimi_k3.
Architectures: KimiK3ForConditionalGeneration.
"""
"""Config-time override declarations for kimi_k3."""
import inspect
import logging
@@ -1,7 +1,4 @@
"""Config-time override declarations for lfm2.
Architectures: Lfm2ForCausalLM, Lfm2MoeForCausalLM.
"""
"""Config-time override declarations for lfm2."""
from typing import Any
@@ -1,7 +1,4 @@
"""Config-time override declarations for llama4.
Architectures: Llama4ForCausalLM, Llama4ForConditionalGeneration.
"""
"""Config-time override declarations for llama4."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for mimo_v2.
Architectures: MiMoV2FlashForCausalLM, MiMoV2ForCausalLM.
"""
"""Config-time override declarations for mimo_v2."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for minicpm.
Architectures: MiniCPMForCausalLM, MiniCPMSALAForCausalLM.
"""
"""Config-time override declarations for minicpm."""
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for minicpmv.
Architectures: MiniCPMV4_6ForConditionalGeneration.
"""
"""Config-time override declarations for minicpmv."""
from typing import Any
@@ -1,7 +1,4 @@
"""Config-time override declarations for minimax_m2.
Architectures: MiniMaxM2ForCausalLM.
"""
"""Config-time override declarations for minimax_m2."""
import logging
from typing import Any
@@ -1,7 +1,4 @@
"""Config-time override declarations for minimax_m3.
Architectures: MiniMaxM3SparseForCausalLM, MiniMaxM3SparseForConditionalGeneration.
"""
"""Config-time override declarations for minimax_m3."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for moss_vl.
Architectures: MossVLForConditionalGeneration.
"""
"""Config-time override declarations for moss_vl."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for muse_glimmer.
Architectures: MuseGlimmerForCausalLM, MuseGlimmerForConditionalGeneration.
"""
"""Config-time override declarations for muse_glimmer."""
import logging
from typing import Any
@@ -1,8 +1,4 @@
"""Config-time override declarations for nemotron_h.
Architectures: NemotronHForCausalLM, NemotronHPuzzleForCausalLM,
NemotronH_Omni_Reasoning_V3.
"""
"""Config-time override declarations for nemotron_h."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for olmo2.
Architectures: Olmo2ForCausalLM.
"""
"""Config-time override declarations for olmo2."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for qwen3_5.
Architectures: InternS2MobiusForConditionalGeneration, InternS2PreviewForConditionalGeneration, Qwen3NextForCausalLM, Qwen3_5ForConditionalGeneration, Qwen3_5MoeForConditionalGeneration.
"""
"""Config-time override declarations for qwen3_5."""
from typing import Any
@@ -39,9 +36,7 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict:
use_mla_backend=use_mla_backend(server_args),
model_config=model_config_of(server_args),
)
# The mamba radix-cache pass runs before this dispatch: read the
# declared strategy through the view (the legacy branch observed the
# already-written field here).
# The Mamba radix-cache pass has already resolved the strategy.
if default_attn_backend == "trtllm_mha" and not (
not mamba_extra_buffer_of(resolved_view(server_args))
and not cfg.disable_radix_cache
@@ -1,7 +1,4 @@
"""Config-time override declarations for qwen3_moe.
Architectures: InternS2PreviewForConditionalGeneration, Qwen3MoeForCausalLM, Qwen3NextForCausalLM, Qwen3VLMoeForConditionalGeneration, Qwen3_5ForConditionalGeneration, Qwen3_5MoeForConditionalGeneration, Qwen4ExpForConditionalGeneration.
"""
"""Config-time override declarations for qwen3_moe."""
import logging
from typing import Any, Dict
@@ -1,7 +1,4 @@
"""Config-time override declarations for qwen3_vl.
Architectures: Qwen3VLForConditionalGeneration, Qwen3VLMoeForConditionalGeneration.
"""
"""Config-time override declarations for qwen3_vl."""
import logging
from typing import Any, Optional
@@ -1,7 +1,4 @@
"""Config-time override declarations for qwen4_exp.
Architectures: Qwen4ExpForConditionalGeneration.
"""
"""Config-time override declarations for qwen4_exp."""
import logging
from typing import Any, Dict
+45 -175
View File
@@ -11,23 +11,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Declarative model-override registry.
"""Configuration declarations and model override dispatch.
Model-identity adjustments to the server configuration are DECLARED here and
appended to the record's declaration stash (gate order, last writer wins).
Nothing here writes back onto ``ServerArgs``: the record holds the user's raw
input, and a decision is read through ``resolution_result`` or the published
config bags — model code never mutates ``ServerArgs`` fields imperatively. That
holds without exception: a resolver this tree does not own assigns onto a
stand-in (``record_foreign_defaults``), and what it set is declared.
Two declaration forms, keyed on ``hf_config.architectures[0]``:
- ``MODEL_OVERRIDES``: pure-constant cases — ``arch -> {field: value}``.
- ``@register_model_override(arch)``: derived cases — a callable
``fn(server_args, hf_config) -> dict`` that faithfully carries today's
conditional logic. ``server_args`` is pristine and must be treated
read-only: the callable returns declarations, it never writes.
Declarations preserve raw ServerArgs inputs and are applied in order, last
writer wins. Read them through ``resolution_result`` or published config bags.
Model providers use ``MODEL_OVERRIDES`` for constants or
``register_model_override`` for functions returning field-value dictionaries.
"""
from __future__ import annotations
@@ -45,8 +34,7 @@ from sglang.srt.arg_groups.arg_utils import (
with_fallback,
)
# Re-exported for the callers that already import these names from here; the
# declarations under ``model_overrides/`` import them from the base directly.
# Compatibility re-exports; model providers import model_override_base directly.
from sglang.srt.arg_groups.model_override_base import ( # noqa: F401
_MODEL_OVERRIDE_FNS,
_PREDICATE_OVERRIDE_FNS,
@@ -100,25 +88,10 @@ def register_post_process(fn: Callable[..., dict]) -> Callable[..., dict]:
def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
"""Invoke one pass at its legacy handler slot.
"""Run a pass against the current resolution snapshot and record its decisions.
Evaluates the pass on the resolving state (a read-only view with the
accumulated declarations overlaid from the stash) and appends its
declaration to the stash, which is what the config bags are projected from.
The fields stay untouched.
A slot that runs after resolution -- ``check_server_args`` hosts one -- lands
in the same stash, which publish projects from later, so it needs no field
write either. After *publish* there is no such later projection: the stash
would grow an entry nothing reads.
So what is refused is the *declaration*, not the record. A pass that returns
an empty dict is a validation, and it may run on the published instance --
it has to, because ``Engine(server_args=sa)`` after ``Engine.shutdown()``
re-runs ``check_server_args`` on the very instance the context still holds.
A pass that returns a non-empty dict there is refused by the guard in
``declare_resolution``, as a late declaration is -- post-publish changes go
to the bags through ``get_context().override(...)``.
After publication, only validation passes returning an empty dict are allowed;
Engine restart may re-run those validations on the same record.
"""
declared = fn(ResolvedView(server_args, overlay=_declaration_overlay(server_args)))
@@ -133,23 +106,10 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
def declare_resolution(server_args: Any, source: str, **fields: Any) -> None:
"""Record a resolution write in the declaration stash.
"""Append field decisions without changing raw inputs.
The stash *is* the resolution result: the bags are projected from it,
`resolution_result` answers from it, and no field is written. A resolver
reading a field another resolver may have decided must read `resolving_view`
(or `resolved_view(server_args)`).
Every declaration goes through here, whenever it is made: inside
``__post_init__``, at launcher stage (LoRA normalization, the auto-detected
parsers -- they decide what the process will run with, so they belong to the
pipeline even though they run after it), and on a copy about to cross a
process boundary. A name that is not a field is rejected here rather than
becoming an attribute nothing reads.
Refuses the published config. The stash is projected at publish and never
again, so a declaration afterwards is a silent no-op; post-publish changes
go to the bags through ``get_context().override(...)``.
Reject unknown fields and declarations on the published record. After
publication, use ``RuntimeContext.override`` to update config bags.
"""
if is_record(server_args):
unknown = sorted(set(fields) - field_names(type(server_args)))
@@ -174,14 +134,7 @@ def declare_resolution(server_args: Any, source: str, **fields: Any) -> None:
class _ForeignDefaults:
"""The stand-in handed to a resolver this tree does not own.
Reads fall through to the resolving view, so a plugin sees what resolution
has decided so far rather than the raw input -- better than what it used to
get, which was the record's own fields. Writes are captured here and
declared by the caller, so the record is never written and the write seal
has no exception.
"""
"""Capture plugin writes while reading through the current resolving view."""
__slots__ = ("_cfg", "_written")
@@ -202,24 +155,9 @@ class _ForeignDefaults:
def record_foreign_defaults(
server_args: Any, source: str, resolve: Callable[[Any], Any]
) -> Any:
"""Run a resolver this tree does not own, and declare what it set.
"""Run a plugin resolver, declare its field writes, and return its result.
Out-of-tree platform plugins and registered speculative algorithms are
handed a configuration and assign fields on it. That interface is not ours
to change, so the assignment stays the contract -- it just lands on a
stand-in instead of the record, and what it set is declared like any other
decision. Nothing writes the record, which is why there is no longer a
named hole in the seal.
Returns whatever the resolver returned, so a provider with a return value
goes through the same capture.
Non-field names are dropped: a plugin scribbling on an attribute that is
not configuration is not a decision, and it was invisible to the previous
diff for the same reason.
A stand-in record (tests drive the hooks with a plain namespace) has no
view to read, so the resolver runs against it directly and uncaptured.
Ignore non-field writes. Non-record test objects are passed through directly.
"""
if not is_record(server_args):
return resolve(server_args)
@@ -236,20 +174,9 @@ def record_foreign_defaults(
def resolution_result(server_args: Any, field: str, default: Any = None) -> Any:
"""What resolution decided for ``field``: the declaration if there is one,
otherwise what the caller supplied, otherwise the field's declared
fallback.
"""Read the last declaration, then raw input, then the declared fallback.
The fallback is last because it is what the field means when nobody said
anything -- an operator who types a value and a pass that decides one both
sit above it. It is read from the declaration rather than filled in by a
pass, so there is no slot to place and no second call to make idempotent.
This is what the config projection reads. Reading the field instead would
work whatever the caller passed onto the record -- and
the point of declaring is that they will not, so the projection must not
depend on it. A config that never ran the pipeline (a mock, a partial
fixture) carries no raw snapshot; its fields are all it has.
Records without a raw-input snapshot fall back to their current fields.
"""
for _source, declared in reversed(
getattr(server_args, "_resolved_overrides", None) or ()
@@ -263,12 +190,7 @@ def resolution_result(server_args: Any, field: str, default: Any = None) -> Any:
def pre_capture_activation_reserve_mb_of(cfg: Any, gpu_mem: Optional[float]) -> float:
"""The activation working-set reserve held back before cuda-graph capture.
The config-shaped half of the pair; `runtime_context` carries the
published-bag half, and `TestDerivedPredicatesAgreeAcrossTiers` pins the
two equal.
"""
"""Return the activation reserve in MB before CUDA graph capture."""
if cfg.disaggregation_mode == "decode":
running_requests = (
cfg.max_running_requests or cfg.cuda_graph_config.decode.max_bs or 1
@@ -378,13 +300,6 @@ def collect_model_override_declarations(
return declarations
# ---------------------------------------------------------------------------
# Derived per-family declarations (faithful ports of legacy arch branches).
# Callables read the PRISTINE server_args, never write; logging is kept
# verbatim from the legacy branch for operator-visible fidelity.
# ---------------------------------------------------------------------------
# Importing the package is what registers the per-model declarations.
import sglang.srt.arg_groups.model_overrides # noqa: F401
@@ -426,9 +341,7 @@ def _step3p_overrides(server_args: Any, hf_config: Any) -> dict:
# ---------------------------------------------------------------------------
# Architectures whose monolith branch routes through the mamba radix cache
# handling (hybrid linear-attention models). Keep in sync with the branch
# guards in _handle_model_specific_adjustments.
# Keep this hybrid-model set in sync with model_hook.handle_model_specific_adjustments.
_MAMBA_RADIX_CACHE_ARCHS = frozenset(
{
"KimiLinearForCausalLM",
@@ -508,15 +421,7 @@ def supports_mamba_cache_extra_buffer(view: Any, model_arch: str) -> bool:
@register_post_process
def _mamba_radix_cache_resolution(view: Any) -> dict:
"""Resolve the hybrid-mamba radix cache fields (pure).
Slot pass: invoked at each legacy ``_handle_mamba_radix_cache`` slot —
the hybrid-spec call at the head of the monolith and the per-arch branch
calls — where it reads the mid-resolution ``page_size`` /
``disable_overlap_schedule`` exactly as the legacy helper did. The arch
guard replicates the union of the legacy call-site guards so the pass is
self-sufficient in the end-state pass list.
"""
"""Resolve hybrid-Mamba cache settings using the current page size and overlap policy."""
from sglang.srt.configs.linear_attn_model_registry import (
get_linear_attn_spec_by_arch,
)
@@ -553,10 +458,7 @@ def _mamba_radix_cache_resolution(view: Any) -> dict:
@register_post_process
def _dsa_kv_cache_dtype_default(view: Any) -> dict:
"""Slot pass in the DSA arm, ordered before the split-backend
resolution: default the kv-cache dtype from the device capability
(Blackwell FP8, Hopper bf16) and normalize the bf16 alias. Reads the
PRISTINE dsa split backends (their resolution runs after this pass)."""
"""Default DSA KV-cache dtype before resolving split attention backends."""
from sglang.srt.configs.model_config import is_deepseek_dsa
hf_config = model_config_of(view).hf_config
@@ -658,9 +560,10 @@ def _check_tilelang_dsa_fp8_kv(
@register_post_process
def _dsa_split_backend_resolution(view: Any) -> dict:
"""Slot pass in the DSA arm: default the DSA prefill/decode split
backends from the mid-resolution kv-cache dtype and the device
capability. The hisparse arm takes precedence under --enable-hisparse."""
"""Resolve DSA split backends from KV-cache dtype and device capability.
HiSparse settings take precedence when enabled.
"""
from sglang.srt.configs.model_config import is_deepseek_dsa
hf_config = model_config_of(view).hf_config
@@ -811,10 +714,7 @@ _DEEPSEEK_FAMILY_ARCHS = frozenset(
@register_post_process
def _deepseek_moe_quant_resolution(view: Any) -> dict:
"""Slot pass invoked from inside the DeepSeek arch branch ("Set moe
backend for DeepSeek"), NOT a dispatch-time declaration: the DSA
kv-cache-dtype default earlier in the branch must read the PRISTINE
quantization, so this resolution has to stay at its legacy slot."""
"""Resolve DeepSeek MoE quantization after DSA has read the input quantization."""
hf_config = model_config_of(view).hf_config
model_arch = hf_config.architectures[0]
if model_arch not in _DEEPSEEK_FAMILY_ARCHS:
@@ -897,10 +797,7 @@ def _deepseek_moe_quant_resolution(view: Any) -> dict:
@register_post_process
def _deepseek_spec_moe_resolution(view: Any) -> dict:
"""Slot pass at the DeepSeek branch's HIP arm: draft (nextn) spec-MoE
backends for the DeepSeek fp4 checkpoint. Reads the mid-resolution
quantization (after _deepseek_moe_quant_resolution) and the pre-a2a
ep_size, exactly like the legacy in-branch writes."""
"""Resolve HIP draft MoE backends after quantization and before A2A adjusts EP size."""
hf_config = model_config_of(view).hf_config
model_arch = hf_config.architectures[0]
@@ -1008,12 +905,10 @@ _FLASHINFER_ALLREDUCE_FUSION_ARCHS = frozenset(
@register_post_process
def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
"""Slot pass at the monolith tail: auto-enable FlashInfer AllReduce
Fusion on SM90/SM100 for models with explicit support. auto resolves to
mnnvl on Blackwell (single- and multi-node) and trtllm on SM90
single-node systems. Reads the mid-resolution enable_dp_attention /
moe_a2a_backend (after the DeepSeek CP and a2a declarations), exactly
like the legacy tail block."""
"""Enable supported FlashInfer AllReduce fusion after CP and A2A resolution.
Auto selects MNNVL on Blackwell and TRTLLM on single-node SM90.
"""
hf_config = model_config_of(view).hf_config
model_arch = hf_config.architectures[0]
# V4.1 TP4 uses the custom push plane for decode and fused MoE finalize.
@@ -1091,8 +986,7 @@ def _deterministic_sampling_backend(view: Any) -> dict:
def _deterministic_is_deepseek_model(view: Any) -> bool:
"""Faithful copy of the deterministic handler's arch probe (pure read;
the handler keeps its own copy for the later deepseek validation)."""
"""Check the model architecture for deterministic DeepSeek backend selection."""
from sglang.srt.connector import ConnectorType
from sglang.srt.utils.common import parse_connector_type
@@ -1179,10 +1073,7 @@ def _attention_backend_default(view: Any) -> dict:
@register_post_process
def _mla_backend_page_constraints(view: Any) -> dict:
"""Page-size constraints of the MLA/TRTLLM backend family (the raises and
the cutedsl prefill fallback stay in the handler; only the page snaps are
declared). The snaps chain on a local value exactly as the legacy blocks
chained on self.page_size."""
"""Resolve MLA/TRTLLM page-size constraints, chaining adjustments on the local value."""
page_size = view.page_size
if (
view.attention_backend == "flashmla"
@@ -1251,10 +1142,7 @@ def _mla_backend_page_constraints(view: Any) -> dict:
@register_post_process
def _mla_kv_cache_dtype_checks(view: Any) -> dict:
"""Read-only validation pass in the attention-backend compatibility
handler: the TRT-LLM and tokenspeed MLA backends constrain the resolved
kv-cache dtype (declarations never reach the field, so the checks read
the view)."""
"""Validate resolved KV-cache dtype for TRTLLM and tokenspeed MLA backends."""
if (
view.attention_backend == "trtllm_mla"
or view.decode_attention_backend == "trtllm_mla"
@@ -1296,10 +1184,7 @@ def _hisparse_validation(view: Any) -> dict:
@register_post_process
def _cutedsl_prefill_backend_fill(view: Any) -> dict:
"""Slot pass in the attention-backend compatibility handler: CuteDSL MLA
is decode-only, so validate the combination and default the prefill side
to trtllm_mla. The trtllm_mha check that follows at the legacy slot reads
the resolved value through the view."""
"""Validate decode-only CuteDSL MLA and default its prefill backend to trtllm_mla."""
if not (
view.attention_backend == "cutedsl_mla"
or view.decode_attention_backend == "cutedsl_mla"
@@ -1484,9 +1369,7 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict:
@register_post_process
def _dp_lm_head_validation(view: Any) -> dict:
"""Read-only validation pass: dp-attention is a prerequisite for the
dp LM head and the TP LM-head all-to-all path. Reads the mid-resolution
values through the view."""
"""Require DP attention for DP LM head and TP LM-head all-to-all."""
if view.enable_dp_lm_head:
assert view.enable_dp_attention, (
"Please enable dp attention when setting enable_dp_lm_head. "
@@ -1512,10 +1395,7 @@ def _dp_lm_head_validation(view: Any) -> dict:
@register_post_process
def _moe_runner_backend_quant_constraints(view: Any) -> dict:
"""The quantization-driven moe_runner_backend resolutions at the head of
_handle_moe_kernel_config. The backend-compatibility asserts and the
disable_shared_experts_fusion writes (post-publish writers exist for that
field) stay in the handler."""
"""Resolve MoE runner backends from quantization."""
moe_runner_backend = view.moe_runner_backend
if view.quantization == "nvfp4_online":
if not get_platform().is_sm100:
@@ -1580,10 +1460,10 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
@register_post_process
def _moe_runner_fusion_disable(view: Any) -> dict:
"""FlashInfer CuteDSL / TRT-LLM / TRT-LLM-routed MoE runners require the
shared-experts fusion disabled; declared at the legacy write slots in
_handle_moe_kernel_config (before the deprecated cutlass env override, so
the runner value observed is the pre-override one)."""
"""Disable shared-expert fusion for incompatible MoE runners.
Runs before the deprecated cutlass environment override.
"""
runner = view.moe_runner_backend
if runner == "flashinfer_cutedsl":
logger.warning(
@@ -1605,10 +1485,7 @@ def _moe_runner_fusion_disable(view: Any) -> dict:
@register_post_process
def _a2a_fusion_adjustments(view: Any) -> dict:
"""A2A-backend-driven shared-experts fusion adjustments, declared at the
legacy write slots in _handle_a2a_moe: Waterfill requires the
fusion enabled; FlashInfer, FlashInfer MegaMOE, and DeepEP v2 A2A require it disabled.
"""
"""Set shared-expert fusion requirements for the selected A2A backend."""
if view.moe_a2a_backend in ("deepep", "megamoe") and view.enable_waterfill:
if view.disable_shared_experts_fusion:
logger.warning(
@@ -1769,8 +1646,7 @@ def _dllm_page_size(view: Any) -> dict:
)
return {"page_size": config.block_size}
if view.page_size > config.block_size:
# Legacy scheduler-init fallback, folded into the pass: the page
# size must not exceed the dllm block size.
# The page size must not exceed the DLLM block size.
logger.warning(
"WARNING: "
f"The page size {view.page_size} should not be larger than dllm block size {config.block_size}."
@@ -1784,10 +1660,7 @@ def validate_declarations(
server_args: Any,
declarations: Sequence[Tuple[str, Dict[str, Any]]],
) -> None:
"""Fail-fast whitelist check at declaration time: a registry typo or a
not-yet-resolvable field must be rejected at its slot, not only at
publish time. Declarations never mutate ``server_args``.
"""
"""Reject unknown or non-resolvable fields before publication."""
# Non-dataclass fixtures carry no Arg metadata (mirrors the
# resolvable_fields escape); only real ServerArgs is validated.
if not is_record(server_args):
@@ -1805,10 +1678,7 @@ def validate_declarations(
@register_post_process
def _hrm_text_attention_force(view: Any) -> dict:
"""HRM-Text's bidirectional prefix attention only works on the Triton
backend. Invoked as the last attention declaration of the resolution
(mirroring the legacy runner-side force, which ran after the whole
pipeline)."""
"""Force Triton for HRM-Text bidirectional prefix attention after backend resolution."""
if view.attention_backend not in (None, "triton"):
logger.warning(
f"Overriding --attention-backend "
+5 -59
View File
@@ -1,10 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
"""The resolution pipeline: the ordered dispatcher every publishing entry runs.
``ServerArgs.resolve_once`` is the only caller. It lives here rather than on the
record because a step decides *about* the record; none of them is a member of
it.
"""
"""Ordered configuration resolution, called by ``ServerArgs.resolve_once``."""
from __future__ import annotations
@@ -22,32 +17,10 @@ from sglang.srt.arg_groups.resolution_hooks import run_hook
def run_resolution_pipeline(server_args: Any) -> None:
"""
Orchestrates the handling of various server arguments, ensuring proper configuration and validation.
"""Resolve and validate server arguments in dependency order.
Dispatcher style principles:
1. Keep this function as an ordered dispatcher. Each step should be a
named call into an ``arg_groups`` family; put imports, conditionals,
mutations, and raises inside the family instead of inline here.
2. Keep the dummy-model boundary as early as correctness allows. Only
model-independent bootstrap, API/network/protocol validation, and
errors that should fire for dummy models should run before it.
3. Order handlers by dependency domains, not by historical insertion:
internal/bootstrap, API/network/protocol, model source/path
resolution, hardware/platform, model-specific adjustment,
parallelism, kernel/attention backend, cuda graph, memory/cache,
and advanced/debug features.
4. Hide narrow integrations behind general handler names. The
dispatcher should say what phase is being handled, not expose a
vendor-, hook-, or feature-specific implementation detail.
5. Give each handler one clear contract: what state it expects, what it
may mutate, and whether it validates only. Long ordering comments
belong in the helper or signal that the helper should be split.
6. Call each step through ``run_hook(handle_x, server_args, ...)``, not
``handle_x(server_args, ...)`` directly -- see
``arg_groups/resolution_hooks.py``. This is every step's fixed
position in the pipeline either way; registering an override changes
what runs here, never when.
Keep model-independent validation before the dummy-model return. Call steps
through ``run_hook`` so plugins can replace them without changing their order.
"""
# What the caller asked for, before any handler runs; this plus the
@@ -128,20 +101,14 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_model_source_paths, server_args)
# Validate mm_process_config.
run_hook(handle_multimodal, server_args)
# Validate SSL arguments early.
run_hook(handle_ssl_validation, server_args)
# Validate transcription/ASR-specific server args.
run_hook(handle_asr_validation, server_args)
# Handle deprecated arguments.
run_hook(handle_deprecated_args, server_args)
# Handle deprecated environment variables for prefill delayer.
run_hook(handle_prefill_delayer_env_compat, server_args)
# Set missing default values.
run_hook(handle_missing_default_values, server_args)
# expert_pack may replace a raw GGUF input with its generated local
@@ -207,7 +174,6 @@ def run_resolution_pipeline(server_args: Any) -> None:
# handle_gpu_memory_settings so the chunk size feeds memory budgeting.
run_hook(apply_glm5_chunked_prefill_default, server_args)
# Handle device-specific backends.
from sglang.srt.arg_groups.platform_hook import (
handle_amd_specifics,
handle_cpu_backends,
@@ -231,12 +197,10 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_platform_defaults, server_args)
# Handle memory-related, chunked prefill, and CUDA graph batch size configurations.
from sglang.srt.arg_groups.memory_hook import handle_gpu_memory_settings
run_hook(handle_gpu_memory_settings, server_args)
# Apply model-specific adjustments.
from sglang.srt.arg_groups.model_hook import (
handle_model_capability_adjustments,
handle_model_specific_adjustments,
@@ -247,7 +211,6 @@ def run_resolution_pipeline(server_args: Any) -> None:
# After the model overrides: Qwen4-Exp declares the PLE offload default there.
run_hook(handle_offload_compatibility, server_args)
# Set kernel backends.
run_post_process_pass(server_args, _sampling_backend_default)
# Must run before _handle_attention_backend_compatibility so the
# deterministic backend is set before auto-detection fills it in.
@@ -291,19 +254,14 @@ def run_resolution_pipeline(server_args: Any) -> None:
# _validate_prefill_only_disable_kv_cache_args().
run_hook(handle_prefill_only_disable_kv_cache, server_args)
# Handle Hicache settings.
run_hook(handle_hicache, server_args)
# Handle data parallelism.
run_hook(handle_data_parallelism, server_args)
# Normalize load balancing defaults.
run_hook(handle_load_balance_method, server_args)
# Handle context parallelism.
run_hook(handle_context_parallelism, server_args)
# Handle MoE configurations.
from sglang.srt.arg_groups.moe_hook import (
handle_a2a_moe,
handle_moe_kernel_config,
@@ -320,11 +278,8 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(handle_elastic_ep, server_args)
run_hook(validate_experimental_sgl_marlin, server_args)
# Handle pipeline parallelism.
run_post_process_pass(server_args, _pipeline_parallel_overlap_disable)
# Handle speculative decoding logic.
from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding
run_hook(handle_speculative_decoding, server_args)
@@ -343,26 +298,20 @@ def run_resolution_pipeline(server_args: Any) -> None:
run_hook(validate_mega_moe_token_budget_for_model, server_args)
# Handle model loading format.
run_hook(handle_load_format, server_args)
# Handle Encoder disaggregation.
run_hook(handle_encoder_disaggregation, server_args)
# Validate tokenizer settings.
run_hook(handle_tokenizer_batching, server_args)
# Propagate environment variables.
run_hook(handle_environment_variables, server_args)
# Validate cache settings.
run_hook(handle_cache_compatibility, server_args)
run_hook(handle_page_major_kv_layout, server_args)
run_hook(handle_unified_memory_pool, server_args)
# Handle diffusion LLM inference.
from sglang.srt.arg_groups.dllm_hook import handle_dllm_inference
run_hook(handle_dllm_inference, server_args)
@@ -370,14 +319,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
# Handle crash dump environment variables (must run before CUDA init).
run_hook(handle_crash_dump_env, server_args)
# Handle debug utilities.
run_hook(handle_debug_utils, server_args)
# Handle any other necessary validations.
run_hook(handle_other_validations, server_args)
# Model-capability adjustments that legacy code applied at model-load
# time; last declarations of the resolution, mirroring that order.
# Apply model-capability constraints after backend selection.
run_hook(handle_model_capability_adjustments, server_args)
finalize_cuda_graph_prefill_max_context(server_args)
@@ -1,45 +1,16 @@
"""Out-of-tree replacement for a named step of the resolution pipeline.
"""Plugin overrides for named resolution steps.
`run_resolution_pipeline` calls its steps by name, hardcoded, with no
per-step dispatch through `self` -- there is nothing on `ServerArgs` left to
subclass in order to change how one step decides. This is the replacement
for that: a decorator that wraps whatever currently runs under a given name,
and a dispatcher the pipeline calls instead of the bare function. The name
itself is never spelled out at the call site -- `run_hook` reads it off the
function it was handed -- so there is exactly one place a step's name is
written by hand: the whitelist below, and whatever a downstream registrant
passes to the decorator.
Whitelisted names only, the same discipline `Arg(resolvable=True)` uses for
declarable fields: this project has already been bitten once by an
unqualified name collision in this exact pipeline (`_parse_cuda_graph_config`
and `_handle_cuda_graph_config` merged under one rename and the dispatcher
called itself). A name that is not on the list fails loudly at import time,
not silently at the call site three modules away.
Every step takes exactly `(server_args)`, `handle_hardware_runtime_validation`
included -- it does not read `server_args` (see the comment at its
definition), but it takes the parameter anyway so `run_hook` never has to
special-case an arity. An override's own signature always matches: `def
mine(server_args, previous)`.
Overrides receive ``(server_args, previous)`` and may call ``previous`` to
wrap the existing step. Only whitelisted names are accepted. Registration
changes the implementation at that step's existing pipeline position.
"""
from __future__ import annotations
from typing import Any, Callable, Dict, FrozenSet, List
# Every step this pipeline runs by name that a downstream package may replace.
# Add a name here only alongside the call site's own switch to `run_hook` --
# an entry with no matching `run_hook(...)` call is a name nothing will ever
# look up. `test_every_whitelisted_hook_has_a_call_site` in
# test_resolution_hook_registry.py holds the two directions of this equal by
# construction, so this list cannot drift from `pipeline.py` silently.
#
# `handle_offload_compatibility` runs at two different points in the pipeline
# (once before model-specific adjustments, once after -- see the comments at
# its call sites). An override registered for it applies at both, identically;
# there is no way to target "just the second call" through this mechanism,
# because the name is all `run_hook` has to key on.
# Names must match run_hook call sites in pipeline.py. An override applies
# to every invocation of that name, including repeated steps.
_OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
{
"handle_mega_moe",
@@ -125,42 +96,16 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
}
)
# name -> registered overrides, oldest first. Each takes `(server_args,
# previous)`, where `previous` is the callable it wraps -- the built-in on
# the first registration, the previous registrant's own wrapper on every one
# after. Process-global as a `dict[str, list]` so a test isolates it the way
# test_model_overrides.py isolates `_MODEL_OVERRIDE_FNS`:
# `patch.dict(..., clear=True)`.
# Step name -> wrappers in registration order.
_HOOKS: Dict[str, List[Callable[[Any, Callable[[Any], None]], None]]] = {}
def register_resolution_hook(name: str):
"""Replace (or wrap) the pipeline step named ``name``.
"""Register ``fn(server_args, previous)`` for a pipeline step.
The decorated function is called as ``fn(server_args, previous)``.
``previous`` is a plain ``server_args -> None`` callable: the built-in
step on the first registration for this name, or the previous
registrant's own wrapper on every registration after that. Call it to
run what would have run without this override -- the `super().handle_x()`
shape, expressed as an explicit argument instead of a method-resolution
lookup, because there is no class hierarchy here for `super()` to walk.
Not calling it is a full replacement.
Registering twice for the same name does not replace the first
registration; it wraps it. The **last** registration is outermost --
runs first, and decides whether/when its `previous` (everything
registered before it, down to the built-in) runs at all. Two downstream
packages that both target the same name compose in whichever order they
happened to import in; if that order matters to you, make one of them
import the other first.
This changes *what* runs at the step's existing position in the
pipeline, never *when*: the call site in `pipeline.py` is unmoved, so
every other step keeps the order it already had. A wrapped step's own
declarations reach `resolution_result` the same way any declaration
does -- only readers from this position onward see them; a step that
already ran and read the old value before this one's `previous` (or the
built-in) declared its replacement has already made its decision on it.
``previous(server_args)`` calls the previously registered wrapper, or the
built-in step for the first registration. Omitting it replaces the step.
The last registration runs outermost, so plugin import order matters.
"""
if name not in _OVERRIDABLE_HOOKS:
raise ValueError(
@@ -178,22 +123,10 @@ def register_resolution_hook(name: str):
def run_hook(builtin: Callable[[Any], None], server_args: Any) -> None:
"""Run ``builtin`` -- the registered chain if anything overrode it under
its name, ``builtin`` directly otherwise.
"""Run registered wrappers for ``builtin.__name__``, or the built-in step.
The name is ``builtin.__name__``, not a second argument: the call site
already has the function in scope (a function-local import, same as
every other step), and spelling the name out again next to it is the
exact "two copies that can silently disagree" shape this project keeps
removing elsewhere. ``builtin`` must therefore be a plain, named
function -- every real call site is -- not a lambda or a bound method.
Called from the step's fixed position in `run_resolution_pipeline`. The
chain is rebuilt from the registry on every call rather than cached at
registration time, because at registration time (import time, before any
`ServerArgs` exists) there is no `server_args` yet and the first
registrant's `previous` cannot be bound to anything real until a call
actually happens.
``builtin`` must be a named function. Wrappers receive the same server
arguments as the built-in step.
"""
step = builtin
for fn in _HOOKS.get(builtin.__name__, ()):
+56 -258
View File
@@ -11,35 +11,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""A single structured accessor for process-static runtime state.
"""Process-wide configuration, parallel state, flags, and resources.
``get_parallel()`` returns a ``ParallelContext`` for configuration, ranks, and
process-group handles. Reads use scoped overrides, then permanent overrides,
then the published configuration. ``publish`` records ranks from ``SpawnRanks``;
distributed initialization records group handles.
``get_server_args()`` retains the raw input for diagnostics. Namespace
accessors such as ``get_exec()`` return resolved configuration published
from declarations; ``RuntimeContext.override`` changes those bags.
``get_parallel()`` also exposes ranks and process groups, with scoped and
permanent overrides taking precedence over published values.
``get_server_args()`` returns the process-wide ``ServerArgs``. This is the
user's raw input, kept **read-only** for debug and reproduction; what
resolution decided lives in the declarations (``resolution_result``) and, for
business code, in the namespace bags below -- never on this object's fields. The context owns the storage:
publishing goes through ``RuntimeContext.set_server_args`` (the legacy
``set_global_server_args_for_scheduler`` is a thin shim over this slot;
``get_global_server_args`` is retired and raises).
``get_exec()`` / ``get_memory()`` / ``get_schedule()`` / ``get_device()`` /
``get_model()`` / ``get_spec()`` / ``get_lora()`` / ``get_mm()`` /
``get_disagg()`` / ``get_serving()`` / ``get_observability()`` return the
resolved **config namespace bags** — the single source of truth for config,
snapshotted from ``server_args`` at publish, one bag per namespace class in
``arg_groups/fields/`` (multi-level under ``exec.*``). Reads are attribute
chains (``get_exec().moe.moe_runner_backend``); bags are read-only by bare
assignment (written via ``override``).
``get_flags()`` returns the runtime-flags tier: state that is **not** a pure
function of config (the capture lifecycle, ACTIVE MoE backend, DP runtime) —
never a mirror of config. Flags live in typed dataclass groups; reads and
writes are plain attribute access, and each group offers a transactional,
test-only ``override(**kw)``.
``get_flags()`` holds mutable runtime state; ``get_resources()`` owns process
handles, and ``get_forward()`` holds per-forward state.
"""
from __future__ import annotations
@@ -612,13 +593,7 @@ class SpFlags(_FlagGroupBase):
class Flags(_FlagGroupBase):
"""Root of the runtime-flags tier.
Resolved configuration lives in the config bags below (projected from the
declarations at publish) — this tier only carries genuine runtime
state whose value is not a function of the configuration alone, grouped
by lifecycle (``capture``) or subsystem (``moe`` / ``dp`` / ``sp``).
"""
"""Mutable runtime state; resolved configuration lives in the config bags."""
capture: CaptureFlags = msgspec.field(default_factory=CaptureFlags)
moe: MoeFlags = msgspec.field(default_factory=MoeFlags)
@@ -662,20 +637,12 @@ class Resources(_FlagGroupBase):
class ForwardFlags:
"""Per-forward runtime flags with one API and two backings.
"""Scoped per-forward flags.
Flags read only from eager Python are backed by context variables, so
nested scopes and threads stay isolated (a new thread sees the defaults).
Flags that are read or written *inside torch.compile-traced model code*
(``_GRAPH_VISIBLE``) are backed by plain dict slots instead: dynamo
cannot trace ``ContextVar.get``/``set``, while plain reads it guards on
— the storage form these flags had before joining the tier. Their
writers and readers are single-threaded per process (TBO interleaves
ubatches on one thread; attention-TP input scattering excludes TBO), so
context isolation is not needed for correctness.
``scoped(**kw)`` — the one regular write path — restores on exit for
both backings. ``set()`` exists for the legacy unscoped setters' shims.
Eager-only flags use ContextVars for thread and nested-scope isolation.
``_GRAPH_VISIBLE`` flags use plain dict slots because Dynamo cannot trace
ContextVar access. Their readers and writers run on one thread per process.
``scoped`` restores both backings on exit; ``set`` supports unscoped setters.
"""
_DEFAULTS = {
@@ -792,24 +759,12 @@ class ForwardFlags:
class _ConfigBag:
"""A resolved-config namespace bag.
"""Resolved namespace with nested bags and guarded writes.
Values are snapshotted from ``server_args`` at ``publish`` and this bag is
the **single source of truth** for its fields thereafter. Read is plain
attribute access; the bag is read-only by bare assignment. The sanctioned
writers are ``get_context().override(source, ...)`` (permanent) and
the scoped ``.override(**kw)`` context manager (tests). Sub-namespaces
(e.g. ``exec.moe``) are nested ``_ConfigBag`` instances reached by attribute.
Leaves and sub-bags are stored as **real instance attributes** (in
``__dict__``), so ``bag.leaf`` / ``bag.sub`` is a plain attribute load that
``torch.compile`` / dynamo can trace — config reads inside a compiled model
forward (e.g. ``get_exec().comm.enable_symm_mem`` in the embedding layer)
must not graph-break. ``_fields`` / ``_subs`` keep the authoritative
name→value maps used for override routing, membership, and scoped restore;
``__getattr__`` is only a fallback for genuinely absent names. (Deliberately
no ``__slots__``: leaves are dynamic, and the ``__dict__`` is what makes the
reads traceable.)
Publication snapshots resolved values; ``RuntimeContext.override`` updates
them permanently and ``override`` restores scoped changes on exit.
Leaves and sub-bags are real instance attributes so Dynamo can trace reads.
``_fields`` and ``_subs`` track names for routing and restoration.
"""
def __init__(self, path: str):
@@ -837,14 +792,12 @@ class _ConfigBag:
)
def _set(self, name: str, value: Any) -> None:
"""Internal write (publish + override) that bypasses the read-only guard.
Updates both the bookkeeping map and the real attribute (traceable read)."""
"""Update the leaf map and traceable attribute, bypassing the write guard."""
object.__getattribute__(self, "_fields")[name] = value
object.__setattr__(self, name, value)
def _set_sub(self, name: str, sub: _ConfigBag) -> None:
"""Register a nested bag as both a bookkeeping entry and a real
attribute (so ``bag.sub`` is a plain, traceable attribute load)."""
"""Register a nested bag in the lookup map and as a traceable attribute."""
object.__getattribute__(self, "_subs")[name] = sub
object.__setattr__(self, name, sub)
@@ -853,13 +806,10 @@ class _ConfigBag:
@contextmanager
def override(self, **kwargs):
"""Scoped, transactional override of this bag's own leaves (keys
validated before any write; restored on exit).
"""Temporarily override this bag's leaves, validating all keys before writing.
For a window where one runner's value differs from the process's — a
draft model loading under ``--speculative-draft-load-format`` while the
target keeps ``--load-format`` — and for tests forcing a code path.
A permanent change goes through ``get_context().override``."""
Restores on exit; use ``RuntimeContext.override`` for permanent changes.
"""
fields = object.__getattribute__(self, "_fields")
unknown = set(kwargs) - set(fields)
if unknown:
@@ -876,16 +826,7 @@ class _ConfigBag:
def _build_config_bags(server_args: Any) -> dict:
"""Snapshot the resolution result into the namespace bag tree.
The tree is ``namespace_of``: each field is placed by the namespace class
that declares it (``arg_groups/fields/``). Each leaf comes from
``resolution_result`` -- the declaration if resolution made one, else what
the caller supplied. Returns ``{top_level_name: _ConfigBag}``, arbitrarily
nested (``exec.moe.eplb.…``). Only dataclass fields are placed, so derived
properties and methods are naturally excluded (they stay on the bag). A
name used as both a leaf and a subgroup at the same level is a hard error
— no silent shadowing."""
"""Project resolved fields by namespace, rejecting leaf/subgroup name collisions."""
from sglang.srt.arg_groups.arg_utils import namespace_of
from sglang.srt.arg_groups.overrides import resolution_result
@@ -894,10 +835,6 @@ def _build_config_bags(server_args: Any) -> dict:
for field, path in namespace_of(type(server_args)).items():
value = resolution_result(server_args, field, _MISSING)
if value is _MISSING:
# Every placed field is a dataclass field, so a resolved config
# always carries it; a miss means a malformed/partial config object
# was published. Fail loud here rather than silently omitting the
# leaf (which surfaces later as a confusing "not a published leaf").
raise AttributeError(
f"config field {field!r} belongs to namespace {path!r} but is absent from "
f"the published {type(server_args).__name__}; cannot project its bag leaf"
@@ -930,18 +867,9 @@ def _build_config_bags(server_args: Any) -> dict:
def _install_derived_leaves(tops: dict, server_args: Any) -> None:
"""Compute the declared config-derived fields into their bags.
"""Compute declared derived leaves once at publication.
A `Derived(fn=...)` is a pure function of the published configuration, so it
is computed once, here, and stored as an ordinary leaf: readers get a plain
attribute load, and there is one answer rather than a pre-publish spelling
and a post-publish one that have to be kept saying the same thing.
The function is handed the whole resolved config, not the bag it lands in.
A derivation is free to span namespaces and they do -- the mamba
extra-buffer predicate reads `memory.disable_radix_cache` alongside its own
`exec.mamba` strategy -- which is exactly why it cannot be written as a
method on either bag.
Derivations receive the whole resolved config because they may span namespaces.
"""
import importlib
@@ -990,9 +918,6 @@ def _resolved_or_field(server_args: Any, name: str, default: Any) -> Any:
decided = resolution_result(server_args, name)
if decided is not None:
return decided
# The default is for the callers that hand over something record-shaped but
# not a record -- the fake configs the context tests publish, and `object()`
# for the sentinel publish. A real ServerArgs always has the field.
return getattr(server_args, name, default)
@@ -1023,10 +948,7 @@ class RuntimeContext:
self.forward = ForwardFlags()
def get_stream(self, name: str) -> Any:
"""Named process-level side stream: get-or-create, shared by
name (the keyed-lazy pattern of the persistent buffers). Creation is
a driver call that must stay outside cuda-graph capture — call sites
lease their stream at init/warmup time."""
"""Get or create a named process stream. Call before CUDA graph capture."""
from sglang.srt.arg_groups.overrides import resolution_result
stream = self.resources.streams.get(name)
@@ -1049,9 +971,7 @@ class RuntimeContext:
return stream
def get_buffer(self, name: str, factory: Any) -> Any:
"""Named process-level persistent buffer: get-or-create via
``factory()``, shared by name (the keyed-lazy pattern of the
persistent buffers / named streams)."""
"""Get or create a named persistent buffer using ``factory()``."""
buf = self.resources.buffers.get(name)
if buf is None:
buf = factory()
@@ -1068,24 +988,11 @@ class RuntimeContext:
return server_args
def set_server_args(self, server_args: ServerArgs) -> None:
"""Publish the process-wide ``ServerArgs`` into the context-owned slot.
Overwrite-allowed: a re-publish replaces the slot (test kits re-publish
per test; production ordering discipline lives at the call-sites, e.g.
the draft-worker guard in ``ModelRunner.__init__``). The published
object is the raw input; the resolution it carries is its declaration
stash, which is what the bags are projected from.
"""
# Seed the capture tier for the new lifecycle (defaults for sentinel
# and mock publishes, which carry no config). Through the resolution,
# not the field: the field is the operator's input.
"""Replace the raw input and rebuild resolved config bags, clearing override provenance."""
self.flags.capture.enable_torch_compile = bool(
_resolved_or_field(server_args, "enable_torch_compile", False)
)
self._server_args = server_args
# Snapshot resolved config into the namespace bags (the single source of
# truth for config reads). Placed by `namespace_of`; a mock/partial
# config that declares no namespace yields an empty tree (no bags).
# Preserve the launcher-assigned device when rebuilding config bags.
stated = {}
if self._config_bags is not None:
@@ -1109,19 +1016,13 @@ class RuntimeContext:
"max_speculative_num_draft_tokens",
max_draft_tokens_of(server_args),
)
# Wire the published `parallel` bag onto the live wrapper: it is the slot
# the `config` property reads, which is how config-only leaves like
# pp_max_micro_batch_size are spelled.
self.parallel._config = self._config_bags.get("parallel")
# A direct install is roleless; ``publish`` assigns the role afterwards.
self._overrides_log = []
self._publish_role = None
def config_bag(self, name: str) -> _ConfigBag:
"""Return the top-level config namespace bag (``device`` / ``model`` /
``exec`` / ``schedule`` / ``memory`` / ``spec`` / ``lora`` / ``mm`` /
``disagg`` / ``serving`` / ``observability``). Fails closed until
``publish`` / ``set_server_args`` has projected it."""
"""Return a top-level namespace bag; raise if it has not been published."""
bags = self._config_bags
if not bags or name not in bags:
raise ValueError(f"config namespace {name!r} not published")
@@ -1159,16 +1060,10 @@ class RuntimeContext:
)
def override(self, source: str, **fields) -> None:
"""The business mutation entry: write resolved config
leaves onto the namespace bags — the single source of truth. It does
**not** touch ``server_args`` (the pristine startup record) and there is
no write-through, so the old "wrote one store, read another" desync class
cannot occur.
"""Update resolved config leaves without changing the raw input.
Each flat field name is routed to its bag by ``namespace_of`` (flat
names are unique across namespaces). Validation is all-or-nothing: an
unknown / unprojected field aborts before any write. ``source`` is
recorded for provenance / reproduction.
Validate all field names and target bags before writing. Record ``source``
and the changed fields for provenance.
"""
if not fields:
return
@@ -1204,12 +1099,7 @@ class RuntimeContext:
self._overrides_log.append((source, dict(fields)))
def config_leaf(self, name: str):
"""One resolved config leaf by field name — the read side of ``override``.
Callers that hold a field name rather than a namespace (a readback
endpoint, a control-plane handler) would otherwise have to know which
bag it lives in.
"""
"""Read a resolved config leaf by its flat field name."""
bags = self._config_bags
if bags is None:
raise ValueError("config not published; cannot read a config leaf")
@@ -1227,36 +1117,16 @@ class RuntimeContext:
return getattr(bag, name)
def overrides_log(self) -> list:
"""Provenance of post-publish ``override`` calls: ``[(source, {field: value})]``.
Returns deep-ish copies (source, dict(fields)) so callers inspecting the
log cannot mutate the recorded provenance in place."""
"""Return ``(source, fields)`` entries with shallow copies of the field dicts."""
return [(source, dict(fields)) for source, fields in self._overrides_log]
def resolved_server_args_dict(self, base: dict | None = None) -> dict:
"""Serialize the *resolved* config: the pristine ``server_args`` fields
with every post-publish ``override`` overlaid.
"""Overlay this process's runtime overrides on the resolved startup config.
``get_internal_state`` reports this, and ``/server_info`` carries it in
the ``internal_states`` block, so scheduler-side runtime changes show up
in a readback: HiCache attach/detach, the generated forward-pass-metrics
endpoint, tunables set via ``/set_internal_state``.
``base`` defaults to ``server_args.resolved_dict()`` -- the record's
fields as resolution decided them, nested dataclasses expanded. (It used
to be ``dict(vars(server_args))``, which carried the private resolution
bookkeeping and the ``model_config`` memo into the readback.) Override
leaves are flat ``ServerArgs`` field names, so overlaying them onto the
top level of the base is exact.
The log is per process: it carries what *this* process overrode. A
weight reload records ``model_path`` and ``load_format`` from the
scheduler process (``ModelRunner.update_model_fields``); the tokenizer
process records only ``load_format`` and keeps ``model_path`` /
``served_model_name`` as ``TokenizerManager`` attributes, which
``TokenizerManager.resolved_config_dict`` overlays on top of this dump.
The top-level ``/server_info`` fields are the startup record, not this
dump.
``base`` defaults to ``server_args.resolved_dict()``. Scheduler readbacks
include these overrides in ``/server_info``'s ``internal_states`` block;
the top-level fields describe startup config. TokenizerManager separately
overlays its per-instance model identity.
"""
d = self.server_args.resolved_dict() if base is None else dict(base)
for _source, fields in self._overrides_log:
@@ -1264,24 +1134,10 @@ class RuntimeContext:
return d
def override_server_args(self, **fields) -> _ServerArgsOverride:
"""Test-only scoped override for the config tier — the sibling of
``get_parallel().override()`` and the flag groups' ``override()``:
tests force execution paths by overriding the context instead of
hand-building config objects.
"""Publish a temporary dummy config for tests and restore the previous context.
``install()`` (or entering it as a context manager) publishes a fresh
dummy-boundary ``ServerArgs`` carrying ``fields`` and returns it;
``restore()`` (or exiting) reinstates whatever the slot held before.
This is the sanctioned way for a test to get a published context, and
it stays. The transitional reason it was introduced for — production
code branching on raw ``server_args`` fields at runtime — is gone (the
read ratchet pins business reads at zero), but a test that exercises
bag readers still needs bags, and the bag tree is projected *from an
instance*: something has to publish one. Prefer the finer-grained
scoped overrides (``get_exec().override(...)``, the flag groups'
``override``) on top of a published context when a test only needs to
force one leaf.
Use as a context manager or call ``install`` / ``restore`` explicitly.
Prefer a bag's scoped ``override`` when only existing leaves need changing.
"""
return _ServerArgsOverride(self, fields)
@@ -1341,19 +1197,11 @@ class _ServerArgsOverride:
raise ValueError(
f"override_server_args: unknown ServerArgs field(s): {sorted(unknown)}"
)
# Declared so the projection sees it; late, because the record is
# resolved already and not yet published.
# Split on whether the name is a field, not on whether it starts with
# an underscore: `_speculative_draft_quantization_explicitly_set` is a
# real field, and seeding it as a raw attribute would leave the earlier
# declaration authoritative, so `resolution_result` and the bag would
# both keep answering the pre-override value.
# Declare config fields even when underscore-prefixed; only non-fields seed caches.
fields = set(type(server_args).__struct_fields__)
declared = {n: v for n, v in self._fields.items() if n in fields}
if declared:
declare_resolution(server_args, "override_server_args", **declared)
# What is left seeds the record's own private caches (`_model_config`
# and friends), which are not configuration and never were.
seeds = {n: v for n, v in self._fields.items() if n not in fields}
for name, value in seeds.items():
msgspec.Struct.__setattr__(server_args, name, value)
@@ -1462,7 +1310,7 @@ def get_observability() -> _ConfigBag:
return _CONTEXT.config_bag("observability")
# --- Per-role namespace sets (2c) -------------------------------------------
# --- Per-role namespace sets -------------------------------------------
#
# ``publish(role=...)`` records which process type installed the config; this
# table declares which top-level config namespaces each role reads. ``None``
@@ -1483,22 +1331,10 @@ ROLE_NAMESPACE_SETS: dict[str, frozenset[str] | None] = {
# Reads (almost) everything by design — the model-executing process.
"scheduler": None,
"test": None,
# The DP controller's static read set, checked against the module: the
# elastic-EP gate, the load-balance method, the watchdog timeout, and the
# disaggregation mode.
# `observability` and `serving` were added when the controller's metrics
# gate, tracing setup and worker-port broadcast stopped reading the record:
# under `enforce` the set is what the process may read, so a conversion
# that reaches a new namespace has to widen it in the same commit.
"dp_controller": frozenset(
{"exec", "parallel", "device", "disagg", "observability", "serving"}
),
# Record-mode audit (2026-08-06, text model, /generate + /get_server_info +
# /v1/models): reads exactly {"serving"} — the per-instance managers read
# self.server_args by design. Still declared full, because that run did not
# exercise the multimodal processors, LoRA/score endpoints, the disagg
# roles, or the gRPC bridge; narrowing needs those shapes audited too, and
# a wrong set fails a request rather than a test.
# Keep unrestricted until multimodal, LoRA, disaggregation, and gRPC reads are audited.
"tokenizer": None,
# Deployment shapes not exercised locally; audit before restricting.
"detokenizer": None,
@@ -1712,16 +1548,9 @@ def _attention_ranks(parallel, tp_rank: int) -> dict:
def assert_published(server_args, *, role: str) -> RuntimeContext:
"""This record, under this role, is already published -- or fail loud.
"""Require this record and role to have been published by the process entry.
Publishing is the process entry's job: `run_scheduler_process`,
`init_multi_tokenizer`, a spawned encoder worker, the benchmark work
functions. A constructor arriving here unpublished means one of those
entries is missing.
A `publish` at this point re-projects the bags over a live process,
discarding every `override()` taken since and the provenance log with it,
so this raises.
Re-publishing here would discard runtime overrides and their provenance.
"""
if _CONTEXT._server_args is server_args and _CONTEXT._publish_role == role:
return _CONTEXT
@@ -1866,13 +1695,9 @@ def reset_context() -> None:
def remote_instance_transfer_engine_enabled(load_format: str | None = None) -> bool:
"""Whether remote-instance weight loading runs over the transfer engine.
"""Check transfer-engine loading against current model config.
Every input is a ``model`` leaf, so this derives from the bags and follows a
post-publish override; ``ServerArgs.remote_instance_weight_loader_use_transfer_engine``
is the pre-publish equivalent, and both go through the same helper.
``load_format`` is the caller's own (a draft runner loading under
``--speculative-draft-load-format`` has one the process record does not).
``load_format`` can override the process value for a draft runner.
"""
from sglang.srt.arg_groups.overrides import remote_instance_transfer_engine_of
@@ -1880,15 +1705,9 @@ def remote_instance_transfer_engine_enabled(load_format: str | None = None) -> b
def max_prefill_buffer_tokens() -> int:
"""The prefill-buffer ceiling: ``chunked_prefill_size``, except PP dynamic
chunking can grow chunks toward ``max_prefill_tokens`` and probe at 1.25x.
"""Return the registered prefill ceiling, or derive it from current config.
The default derives from published leaves (``schedule`` plus the configured
PP size), so it follows post-publish overrides;
``overrides.max_prefill_buffer_tokens`` is the pre-publish equivalent and
``TestDerivedPredicatesAgreeAcrossTiers`` pins the two equal. Records with
a registered ceiling provider (see ``register_prefill_buffer_ceiling``)
answer through it.
PP dynamic chunking can grow toward ``max_prefill_tokens`` and probe at 1.25x.
"""
schedule = get_schedule()
chunked = (
@@ -1905,14 +1724,7 @@ def max_prefill_buffer_tokens() -> int:
def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float:
"""The activation working-set reserve held back before cuda-graph capture.
Derived from published leaves across four bags (``disagg`` / ``schedule`` /
``exec.graph`` / ``spec``) plus the configured parallel sizes, so it follows
a post-publish override; ``pre_capture_activation_reserve_mb_of`` in
``arg_groups.overrides`` is the config-shaped equivalent and
``TestDerivedPredicatesAgreeAcrossTiers`` pins the two equal.
"""
"""Return the activation reserve in MB before CUDA graph capture, using current config."""
schedule = get_schedule()
if get_disagg().disaggregation_mode == "decode":
running_requests = (
@@ -1936,12 +1748,7 @@ def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float:
return reserved_mem
# --- Platform facts -----------------------------------------------------------
#
# One address for what kind of machine this is, so a reader asks
# `get_platform().is_sm100` and an override is stated once instead of patched
# into every module that imported a probe. True before publish, so the context
# probes when no override is installed; `utils.common` holds the implementation.
# Platform probes work before publication and support scoped overrides.
_PLATFORM_PROBES: Dict[str, str] = {
"is_cuda": "is_cuda",
@@ -2072,14 +1879,7 @@ def override_platform(**facts: Any) -> _PlatformOverride:
return _PlatformOverride(**facts)
# --- Derived config accessors ------------------------------------------------
#
# A few values are computed from several config fields plus the HF config, so
# they are derived accessors rather than namespace leaves. Business code must
# not reach for the startup record to get them: these accessors are the named
# home, and this module — which owns the slot — is the only place that reads
# it. Each one keeps the pre-publish function's exact semantics, including which model
# config it derives from (always the process's, i.e. the target's).
# Derived config accessors use the process model (the target during speculation).
def mamba_cache_chunk_size() -> int:
@@ -2146,8 +1946,6 @@ def attention_backends() -> tuple:
"""
from sglang.srt.arg_groups.overrides import attention_backends_of
# All three leaves live in the same bag, so the resolution pipeline's own
# helper applies directly -- one definition of the fallback rule.
return attention_backends_of(get_exec().kernel)
+25 -184
View File
@@ -11,25 +11,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Server argument declarations, resolution, and CLI registration.
Keep this file in the following top-level order:
1. Imports and the module logger.
2. Public extension-point choice lists, with each legacy ``add_*`` alias
immediately below the choice list it extends.
3. Shared (non-extensible) choice lists, scalar defaults, and deprecated
aliases. A choice list used by only one field belongs inline in that field.
4. ``ServerArgs``: fields first, then resolution/validation helpers, then CLI
registration and small query helpers. New resolution steps are appended at
the end of ``arg_groups.pipeline.run_resolution_pipeline``, immediately
marked complete, unless an earlier dependency is documented explicitly.
5. Module-level ``ServerArgs`` construction/runtime shims.
6. Networking constants and ``PortArgs``.
Model- or vendor-specific utilities belong in ``sglang.srt.arg_groups`` (or
their owning subsystem), not before ``ServerArgs`` in this module.
"""
"""Server argument construction, resolution, CLI registration, and network ports."""
from __future__ import annotations
@@ -96,16 +78,11 @@ def _real_kv_hash_modes():
return list(RealKvHashMode)
# Re-exported. These were importable from this module while the field
# declarations that used them lived here; the declarations moved to
# `arg_groups/fields/` but out-of-tree code -- and `tokenizer_control_mixin`
# for `LoRARef` -- still reaches them through `sglang.srt.server_args`.
# Compatibility re-exports for callers importing through server_args.
from sglang.srt.arg_groups.arg_utils import NS, A, Arg # noqa: F401
from sglang.srt.arg_groups.argparse_actions import LoRAPathAction # noqa: F401
# Re-exported for out-of-tree plugins, which have always reached these
# through `sglang.srt.server_args`. The lists and their adders moved to
# `arg_groups/choices.py` with the field declarations that name them.
# Public choice lists and plugin registration helpers.
from sglang.srt.arg_groups.choices import ( # noqa: F401
ATTENTION_BACKEND_CHOICES,
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS,
@@ -219,10 +196,7 @@ def __getattr__(name: str) -> Any:
def _plain(value: Any) -> Any:
"""``asdict``'s conversion, applied to one value: a record -- Struct or
dataclass, since nested config values are both -- becomes a dict, containers
recurse, everything else is deep-copied (a caller mutating the dump must not
reach the live configuration)."""
"""Convert nested Structs and dataclasses to dicts, copying all values."""
if not isinstance(value, type) and is_record(value):
return {
field.name: _plain(getattr(value, field.name))
@@ -238,74 +212,23 @@ def _plain(value: Any) -> Any:
class ServerArgs:
"""Server-wide configuration for SGLang.
"""Raw server configuration, sealed when resolution starts.
Adding new arguments
--------------------
1. **Place the field in the right section.** Arguments are grouped by
comment blocks (``# Model and tokenizer``, ``# LoRA``, etc.).
Add new fields to the matching section, or create a new section
with a ``# ---`` banner when none fits.
2. **Use the ``A[T, ...]`` annotation.** ``A`` is an alias for
``typing.Annotated``. The primary CLI flag is auto-derived from the
field name (``tp_size`` → ``--tp-size``). Use ``aliases`` for
longer alternate names
(``aliases=["--tensor-parallel-size"]``)::
# Bare string — simplest form (just help text):
host: A[str, "The host of the HTTP server."] = "127.0.0.1"
trust_remote_code: A[bool, "Whether to allow custom models."] = False
# Arg(...) — when you need choices, aliases, type_parser, etc.:
load_format: A[str, Arg(help="...", choices=CHOICES)] = "auto"
model_path: A[str, Arg(help="...", aliases=["--model"])]
See ``Arg`` in ``arg_groups/arg_utils.py`` for the full list of
supported metadata (``choices``, ``aliases``, ``type_parser``,
``nargs``, ``const``, ``action``, ``no_cli``, …).
3. **Manual entries in ``add_cli_args`` — only for special cases.**
A few arguments cannot use the annotation style and must be
registered manually in ``add_cli_args``:
- **Deprecated flags** that redirect to another field via one of the
``Deprecated*Action`` classes in ``arg_groups/argparse_actions.py``
(that module's header says which shape fits which migration).
- **Dynamic choices** computed at runtime (e.g. ``reasoning_parser``
whose choices come from a plugin registry).
- The ``--config`` meta-argument (not a dataclass field).
Everything else should use the ``A[T, ...]`` annotation.
The fields live in ``arg_groups/fields/`` -- one class per config
namespace, assembled here rather than inherited, so what the record holds
is one readable call and not a property of a base-class list. The order
they appear in is ``POSITIONAL_FIELD_ORDER``, not the order the namespaces
are listed in: field order *is* the positional constructor signature.
Add fields to the matching namespace in ``arg_groups/fields/`` using
``A[T, help]`` or ``A[T, Arg(...)]``; see ``arg_utils.Arg`` for CLI metadata.
Only dynamic choices, deprecated flags, and ``--config`` need manual
registration in ``add_cli_args``. ``POSITIONAL_FIELD_ORDER`` preserves
the positional constructor signature when these namespaces are assembled.
"""
def __post_init__(self):
"""Construction leaves the record at what the caller asked for.
Resolution is a separate act, entered through ``resolve_once``: the
launcher runs it once per engine, and every publishing process asks the
gate on the way in. A record that is only constructed -- a fixture, a
config being inspected, one being handed to a subprocess that will
resolve it itself -- stays raw.
"""
"""Leave construction unresolved; launchers and publishers call ``resolve_once``."""
def resolve_once(self) -> None:
"""Run the resolution pipeline, unless this record has been through it.
"""Resolve once, preserving declarations across pickling to child processes.
Resolution is a deterministic function of the raw inputs -- two records
built from the same arguments declare the same things -- but the
handlers do not survive a second pass over their own output: DP
attention halves ``chunked_prefill_size`` again on every re-entry.
The publishing entry of every process calls this. In a child the record
arrived by pickle and brought its declarations along, so the child has
nothing left to derive and projects what the parent decided.
Handlers are not idempotent over their own output. A failed resolution
cannot be retried on the same record.
"""
if getattr(self, "_resolution_finished", False):
return
@@ -318,63 +241,30 @@ class ServerArgs:
)
from sglang.srt.arg_groups.pipeline import run_resolution_pipeline
# Sealed for the duration, not just afterwards: everything below this
# line reads the input and declares against it. No exceptions -- even a
# resolver from outside this tree assigns onto a stand-in, not here.
self._input_frozen = True
try:
run_resolution_pipeline(self)
except BaseException:
# The handlers that ran already declared, and they are not
# idempotent over their own output.
self._resolution_failed = True
raise
finally:
self._input_frozen = False
# Set here too, because the dummy/absent-model path returns before the
# end of the pipeline that normally sets it: the gate is about whether
# the handlers ran, not how far they got.
# Also mark the dummy/absent-model path, which returns early from the pipeline.
self._resolution_finished = True
@property
def launch_command(self) -> str | None:
"""How this record was created, verbatim.
`resolved_dict` answers with what resolution decided; this answers with
what the operator asked for, which is a different question and the one
"why is this server configured like this?" usually means. The two are
not derivable from each other: a field the operator never set reads the
same as one they set to the value resolution would have picked anyway.
The launcher stores the arguments it parsed; the in-process `Engine`
stores the call that built the record, since there was no command line.
`None` on a record built directly (a fixture, a subprocess copy that
predates this, a config being inspected).
"""
"""Original CLI arguments or Engine constructor call; ``None`` for direct construction."""
return getattr(self, "_launch_command", None)
def resolved_dict(self) -> dict[str, Any]:
"""This configuration as a plain dict of resolved field values.
What the whole-object readbacks report (`/server_info` and its gRPC and
in-process twins). A plain `asdict` reads the fields, which
carry the raw input; this reads the declarations, so it answers with what
resolution decided. Nested dataclass fields are expanded
the way `asdict` expands them; the private resolution bookkeeping and the
`model_config` memo are not fields and do not appear.
"""
"""Serialize resolved field values, expanding nested records and excluding bookkeeping."""
return {
field.name: _plain(resolution_result(self, field.name))
for field in record_fields(type(self))
}
# ------------------------------------------------------------------
# CUDA graph configuration resolution
# ------------------------------------------------------------------
# ===== END TO BE REFACTORED ====
LANGUAGE_MODEL_ONLY_ARCHITECTURES = (
"MuseGlimmerForConditionalGeneration",
"Cosmos3ForConditionalGeneration",
@@ -392,7 +282,6 @@ class ServerArgs:
@staticmethod
def add_cli_args(parser: argparse.ArgumentParser):
# Auto-derived from Annotated[..., Arg(...)] field metadata.
add_cli_args_from_dataclass(parser, ServerArgs)
# --- Fields with dynamic choices (computed at add_cli_args time) ---
@@ -493,17 +382,7 @@ class ServerArgs:
return self.url(port=self.engine_info_bootstrap_port)
def __setattr__(self, name, value):
# The record holds the operator's input. It is writable while the
# caller is still assembling it and sealed from the moment resolution
# starts: a resolver that writes a field would overwrite the very thing
# the record exists to remember, and the decision it meant to record
# belongs in the stash, where it carries a source and does not destroy
# the input it was derived from.
# Underscore names are mostly the record's own bookkeeping --
# `_input_frozen`, `_raw_input`, `_resolved_overrides`, the memo slots
# -- which resolution writes on purpose. A *field* spelled that way is
# still configuration, so the test cannot be on spelling alone or that
# one leaf stays writable on a read-only record.
# Seal configuration fields, including underscore-prefixed ones, but allow bookkeeping.
if not name.startswith("_") or name in _underscore_field_names():
if getattr(self, "_input_frozen", False):
raise AttributeError(
@@ -527,14 +406,9 @@ class ServerArgs:
msgspec.Struct.__setattr__(self, name, value)
def __reduce__(self):
"""Pickle the record *and* what resolution left on it.
"""Preserve resolution bookkeeping as well as Struct fields when pickling.
A Struct pickles its fields; everything else lives in the `dict=True`
namespace and would be dropped, which for this record means the input
snapshot, the declaration stash and the resolution flags -- the whole
reason a child can publish what its parent decided without resolving
again. Reconstruction restores the fields first and the bookkeeping
after, so the seal is re-armed only once the fields are in place.
Restore fields before bookkeeping so the write seal is re-armed last.
"""
return (
_rebuild_server_args,
@@ -552,12 +426,7 @@ class ServerArgs:
return remote_instance_transfer_engine_of(resolving_view(self), load_format)
# The namespaces whose *input* fields make up the record. A namespace declares
# its input and derived fields side by side; only the input half is collected,
# which is why this is a call and not a base-class list.
#
# A set, not an order: `collect_input_fields` orders by `field_order.py`, which
# is what keeps the positional constructor stable.
# Collect input fields only; field_order.py preserves positional argument order.
_INPUT_NAMESPACES = [
@@ -586,16 +455,9 @@ _INPUT_NAMESPACES = [
_annotations, _defaults, _namespaces = collect_input_fields(_INPUT_NAMESPACES)
ServerArgs.__annotations__ = {**_annotations, **ServerArgs.__annotations__}
# The assembled record has no base classes, so it carries the map the classes
# used to answer through their `_NS_PATH`.
ServerArgs._NS_BY_FIELD = _namespaces
# The classes themselves, so the bag projection can find the declarations
# that are not fields -- the derived half of each namespace.
ServerArgs._NAMESPACES = _INPUT_NAMESPACES
# `dict=True` so the record can carry what is not configuration -- the input
# snapshot, the declaration stash, the resolution flags, the memo slots. A
# Struct has no `__dict__` without it, and those are exactly the underscore
# names `_underscore_field_names()` is careful *not* to include.
# dict=True retains resolution bookkeeping and memoized values outside the fields.
ServerArgs = msgspec.defstruct(
"ServerArgs",
[
@@ -661,14 +523,7 @@ def m3_fp8_attn_gemm_enabled(args) -> bool:
@functools.lru_cache(maxsize=1)
def _underscore_field_names() -> frozenset:
"""Real dataclass fields whose names start with an underscore.
The read-only guard exempts underscore names because they are the record's
own bookkeeping (the stash, the flags, the cache keys). A *field* that
happens to start with an underscore is still resolved configuration --
`_speculative_draft_quantization_explicitly_set` is one -- and exempting it
by spelling would leave exactly one leaf writable on a read-only record.
"""
"""Configuration fields that must stay sealed despite their underscore prefix."""
return frozenset(
field.name for field in record_fields(ServerArgs) if field.name.startswith("_")
)
@@ -695,15 +550,7 @@ def set_global_server_args_for_tokenizer(server_args: ServerArgs):
def get_global_server_args() -> NoReturn:
"""Retired. It raises, because what it used to return is the problem: the
record answers with the operator's *input*, so a caller reading a field
resolution decided got a stale value and no error.
The name survives so that a caller importing it from this module lands on
a message instead of an ImportError. Annotated ``NoReturn`` so a type
checker rejects the call rather than accepting the attribute access after
it. The message lives once, in the exception.
"""
"""Retired accessor retained to raise a migration error for existing imports."""
raise RuntimeError(
"get_global_server_args() is retired. Read the value that is in effect "
"from its namespace bag -- `get_exec().kernel.attention_backend`, "
@@ -721,13 +568,7 @@ def _rebuild_server_args(cls, fields, bookkeeping):
def _declared_default(name: str):
"""The declared default of a field, for a manual `add_argument`.
`ServerArgs.<field>` used to answer with it. The record is a Struct now, so
that expression returns the slot descriptor instead -- which argparse
happily stores as the default, and the first reader gets a
`member_descriptor` where it expected a string.
"""
"""Return a field default; Struct class attributes are slot descriptors."""
return next(
field.default
for field in msgspec.structs.fields(ServerArgs)