From f766397f2eda00dca68a7f94c3634dfc7709392f Mon Sep 17 00:00:00 2001 From: Cheng Wan <54331508+ch-wan@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:47:13 -0700 Subject: [PATCH] [Refactor] Trim server configuration and runtime context comments (#40971) --- python/sglang/srt/arg_groups/arg_utils.py | 91 +---- python/sglang/srt/arg_groups/choices.py | 8 +- .../sglang/srt/arg_groups/deepseek_v4_hook.py | 11 +- python/sglang/srt/arg_groups/field_order.py | 12 +- .../sglang/srt/arg_groups/fields/__init__.py | 35 +- python/sglang/srt/arg_groups/fields/device.py | 9 +- python/sglang/srt/arg_groups/fields/disagg.py | 9 +- python/sglang/srt/arg_groups/fields/exec_.py | 9 +- python/sglang/srt/arg_groups/fields/lora.py | 9 +- python/sglang/srt/arg_groups/fields/memory.py | 9 +- python/sglang/srt/arg_groups/fields/mm.py | 9 +- python/sglang/srt/arg_groups/fields/model.py | 9 +- .../srt/arg_groups/fields/observability.py | 9 +- .../sglang/srt/arg_groups/fields/parallel.py | 9 +- .../sglang/srt/arg_groups/fields/schedule.py | 9 +- .../sglang/srt/arg_groups/fields/serving.py | 9 +- python/sglang/srt/arg_groups/fields/spec.py | 9 +- .../srt/arg_groups/model_override_base.py | 78 +---- .../model_overrides/bailing_moe_v3.py | 6 +- .../arg_groups/model_overrides/cohere2_moe.py | 5 +- .../arg_groups/model_overrides/deepseek_v2.py | 11 +- .../arg_groups/model_overrides/deepseek_v4.py | 5 +- .../srt/arg_groups/model_overrides/exaone.py | 5 +- .../arg_groups/model_overrides/falcon_h1.py | 5 +- .../model_overrides/gemma2_gemma3.py | 5 +- .../srt/arg_groups/model_overrides/gemma4.py | 5 +- .../arg_groups/model_overrides/gigachat35.py | 5 +- .../arg_groups/model_overrides/glm4_moe.py | 5 +- .../srt/arg_groups/model_overrides/gpt_oss.py | 9 +- .../model_overrides/granitemoehybrid.py | 5 +- .../srt/arg_groups/model_overrides/inkling.py | 5 +- .../model_overrides/interns2_mobius.py | 5 +- .../srt/arg_groups/model_overrides/kimi_k3.py | 5 +- .../srt/arg_groups/model_overrides/lfm2.py | 5 +- .../srt/arg_groups/model_overrides/llama4.py | 5 +- .../srt/arg_groups/model_overrides/mimo_v2.py | 5 +- .../srt/arg_groups/model_overrides/minicpm.py | 5 +- .../arg_groups/model_overrides/minicpmv.py | 5 +- .../arg_groups/model_overrides/minimax_m2.py | 5 +- .../arg_groups/model_overrides/minimax_m3.py | 5 +- .../srt/arg_groups/model_overrides/moss_vl.py | 5 +- .../model_overrides/muse_glimmer.py | 5 +- .../arg_groups/model_overrides/nemotron_h.py | 6 +- .../srt/arg_groups/model_overrides/olmo2.py | 5 +- .../srt/arg_groups/model_overrides/qwen3_5.py | 9 +- .../arg_groups/model_overrides/qwen3_moe.py | 5 +- .../arg_groups/model_overrides/qwen3_vl.py | 5 +- .../arg_groups/model_overrides/qwen4_exp.py | 5 +- python/sglang/srt/arg_groups/overrides.py | 220 +++--------- python/sglang/srt/arg_groups/pipeline.py | 64 +--- .../sglang/srt/arg_groups/resolution_hooks.py | 95 +----- python/sglang/srt/runtime_context.py | 314 ++++-------------- python/sglang/srt/server_args.py | 209 ++---------- 53 files changed, 234 insertions(+), 1177 deletions(-) diff --git a/python/sglang/srt/arg_groups/arg_utils.py b/python/sglang/srt/arg_groups/arg_utils.py index 5e00fa5660..4f9089aba0 100644 --- a/python/sglang/srt/arg_groups/arg_utils.py +++ b/python/sglang/srt/arg_groups/arg_utils.py @@ -72,23 +72,12 @@ class Arg(msgspec.Struct, frozen=True): action: Any | None = None action_kwargs: dict | None = None const: Any | None = None - # When True, this field is skipped by add_cli_args_from_dataclass. - # Use for fields that have no CLI surface (e.g. injected via Python only). + # Skip automatic CLI registration. no_cli: bool = False - # When True, config resolution (model overrides and post-process passes) - # may decide this field: the declaration stash accepts the name, and - # `resolution_result` and the config bags answer with the decision. The - # field keeps what the operator passed. + # Allow resolution passes to declare a value for this field. resolvable: bool = False - # What the field means when nobody said anything -- the bottom of the read - # chain: override, decision, input, then this. Not the dataclass default, - # which stays `None` because that is how the record spells "not typed". - # `None` here means the field declares no fallback, which is the same - # answer the read chain gives without one. - # - # Only a value fixed for the life of the configuration belongs here. One - # that depends on the machine, on another field, or on anything impure is a - # decision, and decisions stay in a hook where their order is visible. + # Scalar fallback when neither resolution nor input supplies a value. + # Machine- or config-dependent defaults belong in resolution hooks. fallback: Any = None @@ -112,44 +101,25 @@ class Derived(msgspec.Struct, frozen=True): class NS(msgspec.Struct, frozen=True): - """Namespace-path marker for a ServerArgs field, attached alongside the - field's metadata in ``Annotated``: + """Per-field namespace marker for records spanning multiple namespaces. - field: A[int, "help", NS("parallel")] = 1 - field: A[str, Arg(help="…"), NS("exec.moe")] = "auto" - - ``ServerArgs`` no longer uses it: its fields are declared in the - ``arg_groups/fields/`` classes, each of which carries the ``_NS_PATH`` it - stands for, so the module a declaration lives in *is* its namespace. What - is left for this marker is the case a class cannot express -- one ad-hoc - dataclass whose fields span several namespaces, which is what the - config-bag tests build.""" + Example: ``field: A[int, "help", NS("parallel")] = 1``. + ServerArgs instead collects each namespace class's ``_NS_PATH``. + """ path: str @functools.cache def namespace_of(cls) -> dict: - """``{field_name: dotted namespace path}``, read from the declaring class. + """Return ``{field: namespace}`` from collected metadata, class paths, or NS markers. - A field's namespace is where it is declared: each class in - ``arg_groups/fields/`` carries the ``_NS_PATH`` it stands for, and - ``ServerArgs`` composes them. Walking the MRO therefore answers "which - namespace owns this field" without a per-field marker -- the file the - declaration sits in is the marker. - - A class that is not built that way -- an ad-hoc dataclass spanning several - namespaces, which is what the config-bag tests construct -- falls back to - the per-field ``NS`` marker. A field with neither is absent from the map - (the coverage lint flags them). Non-dataclass types yield an empty map. + Unmarked fields are omitted; non-record types yield an empty map. """ if not is_record(cls): return {} - # An assembled record: the collector recorded who declared each field, - # because there are no base classes left to ask. out = dict(getattr(cls, "_NS_BY_FIELD", None) or {}) - # A class that still inherits its namespaces: nearest declaration wins, so - # walk the MRO front to back and keep the first answer. + # Nearest namespace declaration wins. for base in cls.__mro__: path = base.__dict__.get("_NS_PATH") if path is None: @@ -199,12 +169,7 @@ def resolvable_fields(cls) -> frozenset: @functools.cache def fallbacks_of(cls) -> dict: - """``{field_name: value}`` for every field of ``cls`` that declares one. - - Read the same way `resolvable_fields` reads its flag, so a fallback lives - beside the help text of the field it belongs to rather than in whatever - hook used to fill it in. - """ + """Return declared scalar fallbacks for fields whose input default is None.""" if not is_record(cls): return {} hints = get_type_hints(cls, include_extras=True) @@ -212,11 +177,7 @@ def fallbacks_of(cls) -> dict: for field in record_fields(cls): _, arg = _unwrap_annotated(hints.get(field.name, field.type)) if arg is not None and arg.fallback is not None: - # Two things `with_fallback` relies on and cannot check itself, - # asserted where a new declaration passes through. This function is - # cached, so a mutable fallback would hand one shared object to - # every reader; and a field whose dataclass default is not `None` - # can never reach the fallback, which makes the declaration dead. + # Reject shared mutable fallbacks and defaults that make the fallback unreachable. assert not isinstance(arg.fallback, (list, dict, set)), ( f"{cls.__name__}.{field.name}: a mutable fallback would be " "shared by every reader -- use a scalar" @@ -230,16 +191,10 @@ def fallbacks_of(cls) -> dict: def with_fallback(cls, name: str, value: Any) -> Any: - """``value``, or the declared fallback when nothing has answered. + """Apply a declared fallback when ``value`` is None. - `resolution_result` calls this as its last step -- the effective surface, - which the config bags and `/server_info` read through. Deliberately not the - views a pass reads *while deciding*: `model_overrides/inkling.py` branches - on `if cfg.swa_full_tokens_ratio is None`, and a fallback answering there - would make that branch dead. `test_declared_fallbacks.py` pins both halves. - - A mutable fallback would need copying per read, for the reason a dataclass - spells this `default_factory`. Every declared one is a scalar. + Used by ``resolution_result``, but not mid-resolution views: handlers must + still distinguish unset inputs. Fallbacks must be immutable scalars. """ if value is not None: return value @@ -247,12 +202,7 @@ def with_fallback(cls, name: str, value: Any) -> Any: def record_fields(cls): - """The declared fields of a record, Struct or dataclass. - - `ServerArgs` and the namespace classes are `msgspec.Struct`; the config-bag - tests build ad-hoc dataclasses spanning namespaces, and the helpers here are - driven with both. Anything else yields nothing. - """ + """Return Struct or dataclass fields, or an empty tuple for other types.""" if isinstance(cls, type) and issubclass(cls, msgspec.Struct): return msgspec.structs.fields(cls) if dataclasses.is_dataclass(cls): @@ -326,12 +276,7 @@ def _infer_type_func(tp): def _field_default(field): - """Return the default value for a field, or `_MISSING`. - - The two record shapes spell "no default" differently -- a Struct field says - `msgspec.NODEFAULT`, a dataclass field `dataclasses.MISSING` -- so both are - normalized here and every caller below tests against `_MISSING` alone. - """ + """Return a field default, normalizing Struct and dataclass missing sentinels.""" absent = (_MISSING, msgspec.NODEFAULT) if field.default not in absent: return field.default diff --git a/python/sglang/srt/arg_groups/choices.py b/python/sglang/srt/arg_groups/choices.py index 1ac5b914cb..558cc8ab52 100644 --- a/python/sglang/srt/arg_groups/choices.py +++ b/python/sglang/srt/arg_groups/choices.py @@ -1,10 +1,4 @@ -"""Enumerated choices shared by the config field declarations. - -These lived in ``server_args.py`` beside the fields that name them. The fields -moved to ``arg_groups/fields/``, and ``server_args`` imports the field modules, -so the lists cannot stay there without a cycle. ``server_args`` re-exports them -for the handful of modules that import them from their old home. -""" +"""Shared config choices, re-exported by server_args for plugin compatibility.""" LOAD_FORMAT_CHOICES = [ "auto", diff --git a/python/sglang/srt/arg_groups/deepseek_v4_hook.py b/python/sglang/srt/arg_groups/deepseek_v4_hook.py index 6f901263ec..c2cd5c0526 100644 --- a/python/sglang/srt/arg_groups/deepseek_v4_hook.py +++ b/python/sglang/srt/arg_groups/deepseek_v4_hook.py @@ -21,16 +21,7 @@ logger = logging.getLogger(__name__) def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None: - """Residual imperative arm of the DeepSeek V4 defaults. - - The attention/page/window/MoE-runner declarations moved to the override - registry (arg_groups/overrides.py: _deepseek_v4_overrides) and the - kv-cache dtype default to the resolution pipeline - (_deepseek_v4_kv_cache_dtype, invoked below at its legacy slot). This - keeps, at the legacy slot: the ROCm env fill (env-write policy), the - max_running_requests fill (the speculative hook is a later writer of - that field) and the validations. - """ + """Apply DeepSeek V4 environment defaults, request limits, and validation.""" cfg = resolving_view(server_args) # FlashMLA sparse prefill (SGLANG_OPT_FLASHMLA_SPARSE_PREFILL, default on) diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py index 99879192e0..e20da4374b 100644 --- a/python/sglang/srt/arg_groups/field_order.py +++ b/python/sglang/srt/arg_groups/field_order.py @@ -1,14 +1,6 @@ -"""The order ``ServerArgs`` presents its fields in, frozen. +"""Field order preserving the ServerArgs positional constructor signature. -A dataclass turns field order into a positional constructor signature, so -``ServerArgs(model_path, tokenizer_path)`` has to keep meaning what it means. -Grouping the declarations by namespace would move the second argument onto -another field, silently. - -A compatibility record and nothing else -- a field's namespace is the module it -is declared in, and only ``collect_input_fields`` reads this. A name that is not -here sorts after every name that is, which is the only backward-compatible -position for a new field anyway. +``collect_input_fields`` appends unlisted fields after these entries. """ # fmt: off diff --git a/python/sglang/srt/arg_groups/fields/__init__.py b/python/sglang/srt/arg_groups/fields/__init__.py index b933476055..490fa9e4d6 100644 --- a/python/sglang/srt/arg_groups/fields/__init__.py +++ b/python/sglang/srt/arg_groups/fields/__init__.py @@ -1,9 +1,7 @@ -"""Config field declarations, one module per top-level namespace. +"""Namespace declarations for input and derived configuration. -Each class carries the ``_NS_PATH`` it stands for, so the module a field is -declared in *is* its namespace. ``ServerArgs`` is assembled from the classes -that declare **operator input**; the derived fields, which nobody can type, -are declared beside them but are not collected into the record. +``ServerArgs`` collects input fields from classes carrying ``_NS_PATH``. +Derived declarations stay on those classes for publication into config bags. """ from __future__ import annotations @@ -20,30 +18,11 @@ from sglang.srt.arg_groups.field_order import POSITIONAL_FIELD_ORDER def collect_input_fields( sources: List[type], ) -> Tuple[Dict[str, Any], Dict[str, Any], Dict[str, str]]: - """The annotations and defaults of every field these classes declare. + """Return resolved annotations, defaults, and a field-to-namespace map. - Each source's annotations are resolved **in its own module** and handed on - as type objects. Strings never travel: an annotation carried across as text - would be re-evaluated where it lands, and the composing module does not - import the names the declarations use -- that is the whole point of them - living where they do. - - Returns the resolved annotations, the defaults, and ``{field: namespace}`` - -- the last because the assembled record has no base classes to read a - ``_NS_PATH`` off, and the namespace is still the class that declared it. - - Which classes are passed here is the record's contract: the record holds - what the operator asked for, so a class of derived fields is simply not in - the list, and the rule is one readable call rather than an invariant spread - across a base-class list. - - The result is ordered by ``POSITIONAL_FIELD_ORDER``, not by namespace: a - dataclass turns field order into a positional constructor signature, and - grouping fields that used to be interleaved would silently move - ``ServerArgs(model_path, tokenizer_path)``'s second argument onto another - field. Anything the record declares that the frozen order does not name - goes after it, in declaration order -- the only backward-compatible place - for a new field. + Resolve annotations in their declaring module so imported type names remain + available. Preserve ``POSITIONAL_FIELD_ORDER`` for constructor compatibility; + append new fields in declaration order. """ annotations: Dict[str, Any] = {} defaults: Dict[str, Any] = {} diff --git a/python/sglang/srt/arg_groups/fields/device.py b/python/sglang/srt/arg_groups/fields/device.py index 1463bf6cd5..614860cc9d 100644 --- a/python/sglang/srt/arg_groups/fields/device.py +++ b/python/sglang/srt/arg_groups/fields/device.py @@ -1,11 +1,4 @@ -"""Config fields of the ``device`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``device`` bag, which is what ``get_device()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``device`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/disagg.py b/python/sglang/srt/arg_groups/fields/disagg.py index ad80c251a8..2451ce6329 100644 --- a/python/sglang/srt/arg_groups/fields/disagg.py +++ b/python/sglang/srt/arg_groups/fields/disagg.py @@ -1,11 +1,4 @@ -"""Config fields of the ``disagg`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``disagg`` bag, which is what ``get_disagg()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``disagg`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index cb763cbd6e..7da4dca844 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -1,11 +1,4 @@ -"""Config fields of the ``exec`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``exec`` bag, which is what ``get_exec()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``exec`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/lora.py b/python/sglang/srt/arg_groups/fields/lora.py index 674aa504f5..c7ea9ea8f0 100644 --- a/python/sglang/srt/arg_groups/fields/lora.py +++ b/python/sglang/srt/arg_groups/fields/lora.py @@ -1,11 +1,4 @@ -"""Config fields of the ``lora`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``lora`` bag, which is what ``get_lora()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``lora`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/memory.py b/python/sglang/srt/arg_groups/fields/memory.py index c0bd80d6ec..992caf7222 100644 --- a/python/sglang/srt/arg_groups/fields/memory.py +++ b/python/sglang/srt/arg_groups/fields/memory.py @@ -1,11 +1,4 @@ -"""Config fields of the ``memory`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``memory`` bag, which is what ``get_memory()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``memory`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/mm.py b/python/sglang/srt/arg_groups/fields/mm.py index 0ea162eeff..e9da9deaf2 100644 --- a/python/sglang/srt/arg_groups/fields/mm.py +++ b/python/sglang/srt/arg_groups/fields/mm.py @@ -1,11 +1,4 @@ -"""Config fields of the ``mm`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``mm`` bag, which is what ``get_mm()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``mm`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/model.py b/python/sglang/srt/arg_groups/fields/model.py index 1f2c42bed1..b6ca074691 100644 --- a/python/sglang/srt/arg_groups/fields/model.py +++ b/python/sglang/srt/arg_groups/fields/model.py @@ -1,11 +1,4 @@ -"""Config fields of the ``model`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``model`` bag, which is what ``get_model()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``model`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/observability.py b/python/sglang/srt/arg_groups/fields/observability.py index 902bcc85ad..70ef022a24 100644 --- a/python/sglang/srt/arg_groups/fields/observability.py +++ b/python/sglang/srt/arg_groups/fields/observability.py @@ -1,11 +1,4 @@ -"""Config fields of the ``observability`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``observability`` bag, which is what ``get_observability()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``observability`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/parallel.py b/python/sglang/srt/arg_groups/fields/parallel.py index a2bfd2a843..239da21a60 100644 --- a/python/sglang/srt/arg_groups/fields/parallel.py +++ b/python/sglang/srt/arg_groups/fields/parallel.py @@ -1,11 +1,4 @@ -"""Config fields of the ``parallel`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``parallel`` bag, which is what ``get_parallel()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``parallel`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/schedule.py b/python/sglang/srt/arg_groups/fields/schedule.py index 0d835e8599..8e8fbe5c8c 100644 --- a/python/sglang/srt/arg_groups/fields/schedule.py +++ b/python/sglang/srt/arg_groups/fields/schedule.py @@ -1,11 +1,4 @@ -"""Config fields of the ``schedule`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``schedule`` bag, which is what ``get_schedule()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``schedule`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/serving.py b/python/sglang/srt/arg_groups/fields/serving.py index abc81835a1..d5664ae689 100644 --- a/python/sglang/srt/arg_groups/fields/serving.py +++ b/python/sglang/srt/arg_groups/fields/serving.py @@ -1,11 +1,4 @@ -"""Config fields of the ``serving`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``serving`` bag, which is what ``get_serving()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``serving`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/fields/spec.py b/python/sglang/srt/arg_groups/fields/spec.py index 5ae41ce085..ac91102fdc 100644 --- a/python/sglang/srt/arg_groups/fields/spec.py +++ b/python/sglang/srt/arg_groups/fields/spec.py @@ -1,11 +1,4 @@ -"""Config fields of the ``spec`` namespace. - -One class per namespace. The class *is* the namespace: a field declared here -lands in the ``spec`` bag, which is what ``get_spec()`` returns, so a reader -spells it exactly as before. ``ServerArgs`` composes these classes, so the -record stays one flat object -- the split moves where declarations live, not -how config is shaped at runtime. -""" +"""Config fields of the ``spec`` namespace.""" from __future__ import annotations diff --git a/python/sglang/srt/arg_groups/model_override_base.py b/python/sglang/srt/arg_groups/model_override_base.py index 2d764449dc..5cc545a89f 100644 --- a/python/sglang/srt/arg_groups/model_override_base.py +++ b/python/sglang/srt/arg_groups/model_override_base.py @@ -1,11 +1,7 @@ -"""What the per-model override declarations are written against. +"""Registry and read views shared by model override providers. -The declarations themselves live one directory down, in -``arg_groups/model_overrides/``: one module per model family, mirroring the -``models/`` naming. This module is what they all import -- the registry they -register into, the read-only views they are handed, and the few accessors that -answer questions about the model. It deliberately depends on nothing in -``overrides.py``, so a family module never has to import its way back up. +Model-family modules in ``model_overrides/`` import this module without +creating a dependency back to ``overrides.py``. """ import logging @@ -18,10 +14,9 @@ from sglang.srt.utils.common import is_mps, is_no_spec_infer_or_topk_one logger = logging.getLogger(__name__) -# Constant per-architecture overrides (populated by the migration sweeps). +# Constant per-architecture overrides. MODEL_OVERRIDES: Dict[str, Dict[str, Any]] = { - # These models run in bfloat16 regardless of the requested dtype - # (faithful port of the legacy unconditional arch branch). + # These models require bfloat16 regardless of the requested dtype. "MistralLarge3ForCausalLM": {"dtype": "bfloat16"}, "PixtralForConditionalGeneration": {"dtype": "bfloat16"}, } @@ -31,9 +26,7 @@ MODEL_OVERRIDES: Dict[str, Dict[str, Any]] = { _MODEL_OVERRIDE_FNS: Dict[str, List[Callable[..., dict]]] = {} -# Predicate-keyed providers, in registration order — for legacy branches -# matched by substring/predicate on the architecture string rather than an -# exact name (e.g. '"Step3p5ForCausalLM" in model_arch'). +# Predicate-keyed providers, in registration order. _PREDICATE_OVERRIDE_FNS: List[Tuple[Callable[[str], bool], Callable[..., dict]]] = [] @@ -78,12 +71,7 @@ def _invoke_provider( class ResolvedView: - """Read-only view of the resolving configuration handed to post-process - passes: the accumulated declarations overlaid on the pristine - ``server_args`` (residual imperative writes of non-resolved fields show - through the fallthrough) — exactly the state the legacy handler at the - same slot observed. Writes are rejected: passes return declarations. - """ + """Read-only snapshot of declarations over raw inputs, used by post-process passes.""" __slots__ = ("_server_args", "_overlay") @@ -104,14 +92,9 @@ class ResolvedView: class ResolvingConfig: - """Live read view of the resolution result: the declaration stash over the - record's fields, looked up per read. + """Live view of declarations over raw inputs. - ``ResolvedView`` snapshots the overlay when it is built, which is what a - post-process pass wants -- it reads the state at its slot. A resolver that - reads *after* declaring, or after calling something that declares, needs the - current answer instead, so this one walks the stash on every read. It falls - through to the field, which is where the raw input lives. + Unlike ``ResolvedView``, each read observes declarations made since construction. """ __slots__ = ("_server_args",) @@ -140,10 +123,7 @@ def resolving_view(server_args: Any) -> ResolvingConfig: def _declaration_overlay(server_args: Any) -> Dict[str, Any]: - """What the declarations say so far, last writer wins. - - Nothing writes the fields, so a mid-resolution reader needs this to see a - decision at all; the fields keep what the caller supplied.""" + """Merge declarations in order, with the last writer winning.""" overlay: Dict[str, Any] = {} for _source, declared in getattr(server_args, "_resolved_overrides", None) or (): overlay.update(declared) @@ -151,13 +131,7 @@ def _declaration_overlay(server_args: Any) -> Dict[str, Any]: def resolved_view(server_args: Any) -> ResolvedView: - """Read-only view of the resolving configuration: the declarations - overlaid on the fields, snapshotted per call. - - For mid-resolution code that is not a pass (``__post_init__`` handlers and - hooks) that must answer with what resolution decided -- a declaration-only resolver (a model-specific - override, a registry entry) never writes the field, so a field read there - answers with the raw input.""" + """Snapshot current declarations over the record fields in a read-only view.""" return ResolvedView(server_args, overlay=_declaration_overlay(server_args)) @@ -200,12 +174,7 @@ def record_of(view: Any) -> Any: def is_attention_backend_not_set(cfg: Any): - """None of the three attention backends has been decided yet. - - Takes the view rather than the record: every read is a view read, and the - callers that hold a view (the override providers) would otherwise have to - reach back through it for a record. - """ + """Return whether all three attention backends are unset in the resolving view.""" return ( cfg.attention_backend is None and cfg.prefill_attention_backend is None @@ -221,12 +190,7 @@ def use_mla_backend(server_args: Any): def model_config_of(server_args: Any): - """The model configuration this record describes, built once and memoised. - - Takes a view as readily as the record: a view is a read overlay of one - record, the memo has to live on that record either way, and the callers - that hold a view would otherwise all have to unwrap it themselves. - """ + """Build and memoize ModelConfig on the underlying record, accepting records or views.""" if isinstance(server_args, (ResolvedView, ResolvingConfig)): server_args = record_of(server_args) # Lazy init to avoid circular import @@ -258,13 +222,7 @@ def model_config_of(server_args: Any): def ep_joiner_of(cfg: Any) -> bool: - """Whether this process was launched as an elastic-EP joiner. - - The one definition. After publish the answer is a bag leaf -- - `get_exec().moe.is_ep_joiner` -- computed from this function by the - declaration in `arg_groups/fields/exec_.py`; resolution needs it before - there is a bag to read, which is why it is still a function. - """ + """Return whether this process was launched as an elastic-EP joiner.""" return cfg.ep_join_mode in ("scale", "recover") @@ -279,13 +237,7 @@ def startup_weight_load_overlap_of(cfg: Any) -> bool: def mamba_extra_buffer_of(cfg: Any) -> bool: - """The predicate, read off a config-shaped object mid-resolution. - - This is the one definition. After publish the answer is a bag leaf -- - ``get_exec().mamba.enable_mamba_extra_buffer`` -- computed from this same - function by the declaration in ``arg_groups/fields/exec_.py``. Resolution - needs it before there is a bag to read, which is why it is still a - function.""" + """Return whether the resolved Mamba cache strategy requires an extra buffer.""" return cfg.disable_radix_cache is False and cfg.mamba_radix_cache_strategy in ( "extra_buffer", "extra_buffer_lazy", diff --git a/python/sglang/srt/arg_groups/model_overrides/bailing_moe_v3.py b/python/sglang/srt/arg_groups/model_overrides/bailing_moe_v3.py index 817b02480b..ac29611f3f 100644 --- a/python/sglang/srt/arg_groups/model_overrides/bailing_moe_v3.py +++ b/python/sglang/srt/arg_groups/model_overrides/bailing_moe_v3.py @@ -1,8 +1,4 @@ -"""Config-time override declarations for bailing_moe_v3. - -Architectures: BailingMoeV3ForCausalLM, -BailingMoeV3VLForConditionalGeneration. -""" +"""Config-time override declarations for bailing_moe_v3.""" import logging from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/cohere2_moe.py b/python/sglang/srt/arg_groups/model_overrides/cohere2_moe.py index 5b91257799..5c4bf82b6e 100644 --- a/python/sglang/srt/arg_groups/model_overrides/cohere2_moe.py +++ b/python/sglang/srt/arg_groups/model_overrides/cohere2_moe.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for cohere2_moe. - -Architectures: Cohere2MoeForCausalLM, Cohere2VisionForConditionalGeneration. -""" +"""Config-time override declarations for cohere2_moe.""" import logging from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py index 237ede9654..8693633321 100644 --- a/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/deepseek_v2.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for deepseek_v2. - -Architectures: DeepseekV32ForCausalLM, DeepseekV3ForCausalLM, Dots3NoteForCausalLM, Glm5NextForConditionalGeneration, GlmMoeDsaForCausalLM, HYV4ForCausalLM, HYV4ForCausalLMNextN, KimiK25ForConditionalGeneration, LongcatFlashForCausalLM, LongcatFlashForCausalLMNextN, MistralLarge3ForCausalLM, PixtralForConditionalGeneration. -""" +"""Config-time override declarations for deepseek_v2.""" import logging from typing import Any, Dict @@ -32,11 +29,7 @@ logger = logging.getLogger(__name__) "Dots3NoteForCausalLM", ) def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict: - """Order-safe declarations of the DeepSeek/DSA branch. The CP parallel - writes (enable_dp_attention/ep_size/moe_a2a_backend have post-monolith - writers), the kv-cache/split-backend defaults, the quant/moe block (read - before it by _set_default_dsa_kv_cache_dtype) and the env writes stay in - the branch.""" + """Declare DeepSeek/DSA defaults; ordered CP, KV-cache, and MoE passes run in model_hook.""" cfg = resolving_view(server_args) from sglang.srt.configs.model_config import ( is_deepseek_dsa, diff --git a/python/sglang/srt/arg_groups/model_overrides/deepseek_v4.py b/python/sglang/srt/arg_groups/model_overrides/deepseek_v4.py index 601c2911b3..c9a9b3fe1a 100644 --- a/python/sglang/srt/arg_groups/model_overrides/deepseek_v4.py +++ b/python/sglang/srt/arg_groups/model_overrides/deepseek_v4.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for deepseek_v4. - -Architectures: DeepseekV4ForCausalLM. -""" +"""Config-time override declarations for deepseek_v4.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/exaone.py b/python/sglang/srt/arg_groups/model_overrides/exaone.py index 17095fd5eb..7d0615f2c6 100644 --- a/python/sglang/srt/arg_groups/model_overrides/exaone.py +++ b/python/sglang/srt/arg_groups/model_overrides/exaone.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for exaone. - -Architectures: Exaone4ForCausalLM, ExaoneMoEForCausalLM. -""" +"""Config-time override declarations for exaone.""" import logging from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/falcon_h1.py b/python/sglang/srt/arg_groups/model_overrides/falcon_h1.py index 3a55aac049..28673f9272 100644 --- a/python/sglang/srt/arg_groups/model_overrides/falcon_h1.py +++ b/python/sglang/srt/arg_groups/model_overrides/falcon_h1.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for falcon_h1. - -Architectures: FalconH1ForCausalLM, JetNemotronForCausalLM, JetVLMForConditionalGeneration. -""" +"""Config-time override declarations for falcon_h1.""" from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/gemma2_gemma3.py b/python/sglang/srt/arg_groups/model_overrides/gemma2_gemma3.py index e7616205fc..90331a81cb 100644 --- a/python/sglang/srt/arg_groups/model_overrides/gemma2_gemma3.py +++ b/python/sglang/srt/arg_groups/model_overrides/gemma2_gemma3.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for gemma2_gemma3. - -Architectures: Gemma2ForCausalLM, Gemma3ForCausalLM, Gemma3ForConditionalGeneration, Gemma3nForCausalLM, Gemma3nForConditionalGeneration. -""" +"""Config-time override declarations for gemma2_gemma3.""" import logging from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/gemma4.py b/python/sglang/srt/arg_groups/model_overrides/gemma4.py index 04a7bb615c..f36e4ffbc7 100644 --- a/python/sglang/srt/arg_groups/model_overrides/gemma4.py +++ b/python/sglang/srt/arg_groups/model_overrides/gemma4.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for gemma4. - -Architectures: Gemma4ForCausalLM, Gemma4ForConditionalGeneration, Gemma4UnifiedForConditionalGeneration. -""" +"""Config-time override declarations for gemma4.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/gigachat35.py b/python/sglang/srt/arg_groups/model_overrides/gigachat35.py index 77a19e8b2c..cb50400ea9 100644 --- a/python/sglang/srt/arg_groups/model_overrides/gigachat35.py +++ b/python/sglang/srt/arg_groups/model_overrides/gigachat35.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for gigachat35. - -Architectures: GigaChat35ForCausalLM, GigaChat35ForCausalLMNextN. -""" +"""Config-time override declarations for gigachat35.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/glm4_moe.py b/python/sglang/srt/arg_groups/model_overrides/glm4_moe.py index ebf50fa4fc..c8540f49fd 100644 --- a/python/sglang/srt/arg_groups/model_overrides/glm4_moe.py +++ b/python/sglang/srt/arg_groups/model_overrides/glm4_moe.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for glm4_moe. - -Architectures: Glm4MoeForCausalLM. -""" +"""Config-time override declarations for glm4_moe.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/gpt_oss.py b/python/sglang/srt/arg_groups/model_overrides/gpt_oss.py index 0bee69abcb..5d94b8c7f9 100644 --- a/python/sglang/srt/arg_groups/model_overrides/gpt_oss.py +++ b/python/sglang/srt/arg_groups/model_overrides/gpt_oss.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for gpt_oss. - -Architectures: GptOssForCausalLM. -""" +"""Config-time override declarations for gpt_oss.""" import logging from typing import Any, Dict @@ -46,9 +43,7 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict: # rather than landing on torch_native (no sliding window, no sinks). overrides["attention_backend"] = "triton" if get_platform().is_xpu: - # Check for bf16 dtype on Intel XPU. Reads the pristine dtype request, - # which equals the legacy mid-branch read: dtype had no earlier writer - # for this arch. + # Intel XPU requires bfloat16. if cfg.dtype == "auto": logger.warning( "GptOssForCausalLM on Intel XPU currently supports bfloat16 dtype only" diff --git a/python/sglang/srt/arg_groups/model_overrides/granitemoehybrid.py b/python/sglang/srt/arg_groups/model_overrides/granitemoehybrid.py index 29ceb5f85f..db1cfa87c1 100644 --- a/python/sglang/srt/arg_groups/model_overrides/granitemoehybrid.py +++ b/python/sglang/srt/arg_groups/model_overrides/granitemoehybrid.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for granitemoehybrid. - -Architectures: GraniteMoeHybridForCausalLM. -""" +"""Config-time override declarations for granitemoehybrid.""" from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/inkling.py b/python/sglang/srt/arg_groups/model_overrides/inkling.py index b6b1ded63d..bfc7caf1b3 100644 --- a/python/sglang/srt/arg_groups/model_overrides/inkling.py +++ b/python/sglang/srt/arg_groups/model_overrides/inkling.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for inkling. - -Architectures: InklingForConditionalGeneration, InklingForConditionalGenerationMTP. -""" +"""Config-time override declarations for inkling.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/interns2_mobius.py b/python/sglang/srt/arg_groups/model_overrides/interns2_mobius.py index bb11a3b0bd..dd7e7b6e9b 100644 --- a/python/sglang/srt/arg_groups/model_overrides/interns2_mobius.py +++ b/python/sglang/srt/arg_groups/model_overrides/interns2_mobius.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for interns2_mobius. - -Architectures: InternS2MobiusForConditionalGeneration. -""" +"""Config-time override declarations for interns2_mobius.""" from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py b/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py index 0334509431..6a0157314e 100644 --- a/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py +++ b/python/sglang/srt/arg_groups/model_overrides/kimi_k3.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for kimi_k3. - -Architectures: KimiK3ForConditionalGeneration. -""" +"""Config-time override declarations for kimi_k3.""" import inspect import logging diff --git a/python/sglang/srt/arg_groups/model_overrides/lfm2.py b/python/sglang/srt/arg_groups/model_overrides/lfm2.py index 8dc72ae9e5..a7301071b0 100644 --- a/python/sglang/srt/arg_groups/model_overrides/lfm2.py +++ b/python/sglang/srt/arg_groups/model_overrides/lfm2.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for lfm2. - -Architectures: Lfm2ForCausalLM, Lfm2MoeForCausalLM. -""" +"""Config-time override declarations for lfm2.""" from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/llama4.py b/python/sglang/srt/arg_groups/model_overrides/llama4.py index d6eff4f26f..64720a9850 100644 --- a/python/sglang/srt/arg_groups/model_overrides/llama4.py +++ b/python/sglang/srt/arg_groups/model_overrides/llama4.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for llama4. - -Architectures: Llama4ForCausalLM, Llama4ForConditionalGeneration. -""" +"""Config-time override declarations for llama4.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py b/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py index bef9a36304..c0ad3d3738 100644 --- a/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for mimo_v2. - -Architectures: MiMoV2FlashForCausalLM, MiMoV2ForCausalLM. -""" +"""Config-time override declarations for mimo_v2.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/minicpm.py b/python/sglang/srt/arg_groups/model_overrides/minicpm.py index 6cd91ef3f1..3790c30d6e 100644 --- a/python/sglang/srt/arg_groups/model_overrides/minicpm.py +++ b/python/sglang/srt/arg_groups/model_overrides/minicpm.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for minicpm. - -Architectures: MiniCPMForCausalLM, MiniCPMSALAForCausalLM. -""" +"""Config-time override declarations for minicpm.""" from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/minicpmv.py b/python/sglang/srt/arg_groups/model_overrides/minicpmv.py index c4c79fea60..9fe11f0b10 100644 --- a/python/sglang/srt/arg_groups/model_overrides/minicpmv.py +++ b/python/sglang/srt/arg_groups/model_overrides/minicpmv.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for minicpmv. - -Architectures: MiniCPMV4_6ForConditionalGeneration. -""" +"""Config-time override declarations for minicpmv.""" from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/minimax_m2.py b/python/sglang/srt/arg_groups/model_overrides/minimax_m2.py index f9e9664583..e2a36d194b 100644 --- a/python/sglang/srt/arg_groups/model_overrides/minimax_m2.py +++ b/python/sglang/srt/arg_groups/model_overrides/minimax_m2.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for minimax_m2. - -Architectures: MiniMaxM2ForCausalLM. -""" +"""Config-time override declarations for minimax_m2.""" import logging from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/minimax_m3.py b/python/sglang/srt/arg_groups/model_overrides/minimax_m3.py index dbb17ae186..41bfd58961 100644 --- a/python/sglang/srt/arg_groups/model_overrides/minimax_m3.py +++ b/python/sglang/srt/arg_groups/model_overrides/minimax_m3.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for minimax_m3. - -Architectures: MiniMaxM3SparseForCausalLM, MiniMaxM3SparseForConditionalGeneration. -""" +"""Config-time override declarations for minimax_m3.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/moss_vl.py b/python/sglang/srt/arg_groups/model_overrides/moss_vl.py index d0880e1b21..12e2bcaaa0 100644 --- a/python/sglang/srt/arg_groups/model_overrides/moss_vl.py +++ b/python/sglang/srt/arg_groups/model_overrides/moss_vl.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for moss_vl. - -Architectures: MossVLForConditionalGeneration. -""" +"""Config-time override declarations for moss_vl.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/muse_glimmer.py b/python/sglang/srt/arg_groups/model_overrides/muse_glimmer.py index 7a6f32cdbc..239ecde1d4 100644 --- a/python/sglang/srt/arg_groups/model_overrides/muse_glimmer.py +++ b/python/sglang/srt/arg_groups/model_overrides/muse_glimmer.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for muse_glimmer. - -Architectures: MuseGlimmerForCausalLM, MuseGlimmerForConditionalGeneration. -""" +"""Config-time override declarations for muse_glimmer.""" import logging from typing import Any diff --git a/python/sglang/srt/arg_groups/model_overrides/nemotron_h.py b/python/sglang/srt/arg_groups/model_overrides/nemotron_h.py index d9fa5bf3c8..d5cb122fc4 100644 --- a/python/sglang/srt/arg_groups/model_overrides/nemotron_h.py +++ b/python/sglang/srt/arg_groups/model_overrides/nemotron_h.py @@ -1,8 +1,4 @@ -"""Config-time override declarations for nemotron_h. - -Architectures: NemotronHForCausalLM, NemotronHPuzzleForCausalLM, -NemotronH_Omni_Reasoning_V3. -""" +"""Config-time override declarations for nemotron_h.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/olmo2.py b/python/sglang/srt/arg_groups/model_overrides/olmo2.py index 9dc819ca27..11ae833e84 100644 --- a/python/sglang/srt/arg_groups/model_overrides/olmo2.py +++ b/python/sglang/srt/arg_groups/model_overrides/olmo2.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for olmo2. - -Architectures: Olmo2ForCausalLM. -""" +"""Config-time override declarations for olmo2.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/qwen3_5.py b/python/sglang/srt/arg_groups/model_overrides/qwen3_5.py index 77976bbeec..0948e3dfdf 100644 --- a/python/sglang/srt/arg_groups/model_overrides/qwen3_5.py +++ b/python/sglang/srt/arg_groups/model_overrides/qwen3_5.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for qwen3_5. - -Architectures: InternS2MobiusForConditionalGeneration, InternS2PreviewForConditionalGeneration, Qwen3NextForCausalLM, Qwen3_5ForConditionalGeneration, Qwen3_5MoeForConditionalGeneration. -""" +"""Config-time override declarations for qwen3_5.""" from typing import Any @@ -39,9 +36,7 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict: use_mla_backend=use_mla_backend(server_args), model_config=model_config_of(server_args), ) - # The mamba radix-cache pass runs before this dispatch: read the - # declared strategy through the view (the legacy branch observed the - # already-written field here). + # The Mamba radix-cache pass has already resolved the strategy. if default_attn_backend == "trtllm_mha" and not ( not mamba_extra_buffer_of(resolved_view(server_args)) and not cfg.disable_radix_cache diff --git a/python/sglang/srt/arg_groups/model_overrides/qwen3_moe.py b/python/sglang/srt/arg_groups/model_overrides/qwen3_moe.py index 1747e5192d..f7d3ca0128 100644 --- a/python/sglang/srt/arg_groups/model_overrides/qwen3_moe.py +++ b/python/sglang/srt/arg_groups/model_overrides/qwen3_moe.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for qwen3_moe. - -Architectures: InternS2PreviewForConditionalGeneration, Qwen3MoeForCausalLM, Qwen3NextForCausalLM, Qwen3VLMoeForConditionalGeneration, Qwen3_5ForConditionalGeneration, Qwen3_5MoeForConditionalGeneration, Qwen4ExpForConditionalGeneration. -""" +"""Config-time override declarations for qwen3_moe.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/model_overrides/qwen3_vl.py b/python/sglang/srt/arg_groups/model_overrides/qwen3_vl.py index 406048a399..158ebba051 100644 --- a/python/sglang/srt/arg_groups/model_overrides/qwen3_vl.py +++ b/python/sglang/srt/arg_groups/model_overrides/qwen3_vl.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for qwen3_vl. - -Architectures: Qwen3VLForConditionalGeneration, Qwen3VLMoeForConditionalGeneration. -""" +"""Config-time override declarations for qwen3_vl.""" import logging from typing import Any, Optional diff --git a/python/sglang/srt/arg_groups/model_overrides/qwen4_exp.py b/python/sglang/srt/arg_groups/model_overrides/qwen4_exp.py index a35b0770fa..53d281f9be 100644 --- a/python/sglang/srt/arg_groups/model_overrides/qwen4_exp.py +++ b/python/sglang/srt/arg_groups/model_overrides/qwen4_exp.py @@ -1,7 +1,4 @@ -"""Config-time override declarations for qwen4_exp. - -Architectures: Qwen4ExpForConditionalGeneration. -""" +"""Config-time override declarations for qwen4_exp.""" import logging from typing import Any, Dict diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 5e7b6fa662..53418a5237 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -11,23 +11,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Declarative model-override registry. +"""Configuration declarations and model override dispatch. -Model-identity adjustments to the server configuration are DECLARED here and -appended to the record's declaration stash (gate order, last writer wins). -Nothing here writes back onto ``ServerArgs``: the record holds the user's raw -input, and a decision is read through ``resolution_result`` or the published -config bags — model code never mutates ``ServerArgs`` fields imperatively. That -holds without exception: a resolver this tree does not own assigns onto a -stand-in (``record_foreign_defaults``), and what it set is declared. - -Two declaration forms, keyed on ``hf_config.architectures[0]``: - -- ``MODEL_OVERRIDES``: pure-constant cases — ``arch -> {field: value}``. -- ``@register_model_override(arch)``: derived cases — a callable - ``fn(server_args, hf_config) -> dict`` that faithfully carries today's - conditional logic. ``server_args`` is pristine and must be treated - read-only: the callable returns declarations, it never writes. +Declarations preserve raw ServerArgs inputs and are applied in order, last +writer wins. Read them through ``resolution_result`` or published config bags. +Model providers use ``MODEL_OVERRIDES`` for constants or +``register_model_override`` for functions returning field-value dictionaries. """ from __future__ import annotations @@ -45,8 +34,7 @@ from sglang.srt.arg_groups.arg_utils import ( with_fallback, ) -# Re-exported for the callers that already import these names from here; the -# declarations under ``model_overrides/`` import them from the base directly. +# Compatibility re-exports; model providers import model_override_base directly. from sglang.srt.arg_groups.model_override_base import ( # noqa: F401 _MODEL_OVERRIDE_FNS, _PREDICATE_OVERRIDE_FNS, @@ -100,25 +88,10 @@ def register_post_process(fn: Callable[..., dict]) -> Callable[..., dict]: def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None: - """Invoke one pass at its legacy handler slot. + """Run a pass against the current resolution snapshot and record its decisions. - Evaluates the pass on the resolving state (a read-only view with the - accumulated declarations overlaid from the stash) and appends its - declaration to the stash, which is what the config bags are projected from. - The fields stay untouched. - - A slot that runs after resolution -- ``check_server_args`` hosts one -- lands - in the same stash, which publish projects from later, so it needs no field - write either. After *publish* there is no such later projection: the stash - would grow an entry nothing reads. - - So what is refused is the *declaration*, not the record. A pass that returns - an empty dict is a validation, and it may run on the published instance -- - it has to, because ``Engine(server_args=sa)`` after ``Engine.shutdown()`` - re-runs ``check_server_args`` on the very instance the context still holds. - A pass that returns a non-empty dict there is refused by the guard in - ``declare_resolution``, as a late declaration is -- post-publish changes go - to the bags through ``get_context().override(...)``. + After publication, only validation passes returning an empty dict are allowed; + Engine restart may re-run those validations on the same record. """ declared = fn(ResolvedView(server_args, overlay=_declaration_overlay(server_args))) @@ -133,23 +106,10 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None: def declare_resolution(server_args: Any, source: str, **fields: Any) -> None: - """Record a resolution write in the declaration stash. + """Append field decisions without changing raw inputs. - The stash *is* the resolution result: the bags are projected from it, - `resolution_result` answers from it, and no field is written. A resolver - reading a field another resolver may have decided must read `resolving_view` - (or `resolved_view(server_args)`). - - Every declaration goes through here, whenever it is made: inside - ``__post_init__``, at launcher stage (LoRA normalization, the auto-detected - parsers -- they decide what the process will run with, so they belong to the - pipeline even though they run after it), and on a copy about to cross a - process boundary. A name that is not a field is rejected here rather than - becoming an attribute nothing reads. - - Refuses the published config. The stash is projected at publish and never - again, so a declaration afterwards is a silent no-op; post-publish changes - go to the bags through ``get_context().override(...)``. + Reject unknown fields and declarations on the published record. After + publication, use ``RuntimeContext.override`` to update config bags. """ if is_record(server_args): unknown = sorted(set(fields) - field_names(type(server_args))) @@ -174,14 +134,7 @@ def declare_resolution(server_args: Any, source: str, **fields: Any) -> None: class _ForeignDefaults: - """The stand-in handed to a resolver this tree does not own. - - Reads fall through to the resolving view, so a plugin sees what resolution - has decided so far rather than the raw input -- better than what it used to - get, which was the record's own fields. Writes are captured here and - declared by the caller, so the record is never written and the write seal - has no exception. - """ + """Capture plugin writes while reading through the current resolving view.""" __slots__ = ("_cfg", "_written") @@ -202,24 +155,9 @@ class _ForeignDefaults: def record_foreign_defaults( server_args: Any, source: str, resolve: Callable[[Any], Any] ) -> Any: - """Run a resolver this tree does not own, and declare what it set. + """Run a plugin resolver, declare its field writes, and return its result. - Out-of-tree platform plugins and registered speculative algorithms are - handed a configuration and assign fields on it. That interface is not ours - to change, so the assignment stays the contract -- it just lands on a - stand-in instead of the record, and what it set is declared like any other - decision. Nothing writes the record, which is why there is no longer a - named hole in the seal. - - Returns whatever the resolver returned, so a provider with a return value - goes through the same capture. - - Non-field names are dropped: a plugin scribbling on an attribute that is - not configuration is not a decision, and it was invisible to the previous - diff for the same reason. - - A stand-in record (tests drive the hooks with a plain namespace) has no - view to read, so the resolver runs against it directly and uncaptured. + Ignore non-field writes. Non-record test objects are passed through directly. """ if not is_record(server_args): return resolve(server_args) @@ -236,20 +174,9 @@ def record_foreign_defaults( def resolution_result(server_args: Any, field: str, default: Any = None) -> Any: - """What resolution decided for ``field``: the declaration if there is one, - otherwise what the caller supplied, otherwise the field's declared - fallback. + """Read the last declaration, then raw input, then the declared fallback. - The fallback is last because it is what the field means when nobody said - anything -- an operator who types a value and a pass that decides one both - sit above it. It is read from the declaration rather than filled in by a - pass, so there is no slot to place and no second call to make idempotent. - - This is what the config projection reads. Reading the field instead would - work whatever the caller passed onto the record -- and - the point of declaring is that they will not, so the projection must not - depend on it. A config that never ran the pipeline (a mock, a partial - fixture) carries no raw snapshot; its fields are all it has. + Records without a raw-input snapshot fall back to their current fields. """ for _source, declared in reversed( getattr(server_args, "_resolved_overrides", None) or () @@ -263,12 +190,7 @@ def resolution_result(server_args: Any, field: str, default: Any = None) -> Any: def pre_capture_activation_reserve_mb_of(cfg: Any, gpu_mem: Optional[float]) -> float: - """The activation working-set reserve held back before cuda-graph capture. - - The config-shaped half of the pair; `runtime_context` carries the - published-bag half, and `TestDerivedPredicatesAgreeAcrossTiers` pins the - two equal. - """ + """Return the activation reserve in MB before CUDA graph capture.""" if cfg.disaggregation_mode == "decode": running_requests = ( cfg.max_running_requests or cfg.cuda_graph_config.decode.max_bs or 1 @@ -378,13 +300,6 @@ def collect_model_override_declarations( return declarations -# --------------------------------------------------------------------------- -# Derived per-family declarations (faithful ports of legacy arch branches). -# Callables read the PRISTINE server_args, never write; logging is kept -# verbatim from the legacy branch for operator-visible fidelity. -# --------------------------------------------------------------------------- - - # Importing the package is what registers the per-model declarations. import sglang.srt.arg_groups.model_overrides # noqa: F401 @@ -426,9 +341,7 @@ def _step3p_overrides(server_args: Any, hf_config: Any) -> dict: # --------------------------------------------------------------------------- -# Architectures whose monolith branch routes through the mamba radix cache -# handling (hybrid linear-attention models). Keep in sync with the branch -# guards in _handle_model_specific_adjustments. +# Keep this hybrid-model set in sync with model_hook.handle_model_specific_adjustments. _MAMBA_RADIX_CACHE_ARCHS = frozenset( { "KimiLinearForCausalLM", @@ -508,15 +421,7 @@ def supports_mamba_cache_extra_buffer(view: Any, model_arch: str) -> bool: @register_post_process def _mamba_radix_cache_resolution(view: Any) -> dict: - """Resolve the hybrid-mamba radix cache fields (pure). - - Slot pass: invoked at each legacy ``_handle_mamba_radix_cache`` slot — - the hybrid-spec call at the head of the monolith and the per-arch branch - calls — where it reads the mid-resolution ``page_size`` / - ``disable_overlap_schedule`` exactly as the legacy helper did. The arch - guard replicates the union of the legacy call-site guards so the pass is - self-sufficient in the end-state pass list. - """ + """Resolve hybrid-Mamba cache settings using the current page size and overlap policy.""" from sglang.srt.configs.linear_attn_model_registry import ( get_linear_attn_spec_by_arch, ) @@ -553,10 +458,7 @@ def _mamba_radix_cache_resolution(view: Any) -> dict: @register_post_process def _dsa_kv_cache_dtype_default(view: Any) -> dict: - """Slot pass in the DSA arm, ordered before the split-backend - resolution: default the kv-cache dtype from the device capability - (Blackwell FP8, Hopper bf16) and normalize the bf16 alias. Reads the - PRISTINE dsa split backends (their resolution runs after this pass).""" + """Default DSA KV-cache dtype before resolving split attention backends.""" from sglang.srt.configs.model_config import is_deepseek_dsa hf_config = model_config_of(view).hf_config @@ -658,9 +560,10 @@ def _check_tilelang_dsa_fp8_kv( @register_post_process def _dsa_split_backend_resolution(view: Any) -> dict: - """Slot pass in the DSA arm: default the DSA prefill/decode split - backends from the mid-resolution kv-cache dtype and the device - capability. The hisparse arm takes precedence under --enable-hisparse.""" + """Resolve DSA split backends from KV-cache dtype and device capability. + + HiSparse settings take precedence when enabled. + """ from sglang.srt.configs.model_config import is_deepseek_dsa hf_config = model_config_of(view).hf_config @@ -811,10 +714,7 @@ _DEEPSEEK_FAMILY_ARCHS = frozenset( @register_post_process def _deepseek_moe_quant_resolution(view: Any) -> dict: - """Slot pass invoked from inside the DeepSeek arch branch ("Set moe - backend for DeepSeek"), NOT a dispatch-time declaration: the DSA - kv-cache-dtype default earlier in the branch must read the PRISTINE - quantization, so this resolution has to stay at its legacy slot.""" + """Resolve DeepSeek MoE quantization after DSA has read the input quantization.""" hf_config = model_config_of(view).hf_config model_arch = hf_config.architectures[0] if model_arch not in _DEEPSEEK_FAMILY_ARCHS: @@ -897,10 +797,7 @@ def _deepseek_moe_quant_resolution(view: Any) -> dict: @register_post_process def _deepseek_spec_moe_resolution(view: Any) -> dict: - """Slot pass at the DeepSeek branch's HIP arm: draft (nextn) spec-MoE - backends for the DeepSeek fp4 checkpoint. Reads the mid-resolution - quantization (after _deepseek_moe_quant_resolution) and the pre-a2a - ep_size, exactly like the legacy in-branch writes.""" + """Resolve HIP draft MoE backends after quantization and before A2A adjusts EP size.""" hf_config = model_config_of(view).hf_config model_arch = hf_config.architectures[0] @@ -1008,12 +905,10 @@ _FLASHINFER_ALLREDUCE_FUSION_ARCHS = frozenset( @register_post_process def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict: - """Slot pass at the monolith tail: auto-enable FlashInfer AllReduce - Fusion on SM90/SM100 for models with explicit support. auto resolves to - mnnvl on Blackwell (single- and multi-node) and trtllm on SM90 - single-node systems. Reads the mid-resolution enable_dp_attention / - moe_a2a_backend (after the DeepSeek CP and a2a declarations), exactly - like the legacy tail block.""" + """Enable supported FlashInfer AllReduce fusion after CP and A2A resolution. + + Auto selects MNNVL on Blackwell and TRTLLM on single-node SM90. + """ hf_config = model_config_of(view).hf_config model_arch = hf_config.architectures[0] # V4.1 TP4 uses the custom push plane for decode and fused MoE finalize. @@ -1091,8 +986,7 @@ def _deterministic_sampling_backend(view: Any) -> dict: def _deterministic_is_deepseek_model(view: Any) -> bool: - """Faithful copy of the deterministic handler's arch probe (pure read; - the handler keeps its own copy for the later deepseek validation).""" + """Check the model architecture for deterministic DeepSeek backend selection.""" from sglang.srt.connector import ConnectorType from sglang.srt.utils.common import parse_connector_type @@ -1179,10 +1073,7 @@ def _attention_backend_default(view: Any) -> dict: @register_post_process def _mla_backend_page_constraints(view: Any) -> dict: - """Page-size constraints of the MLA/TRTLLM backend family (the raises and - the cutedsl prefill fallback stay in the handler; only the page snaps are - declared). The snaps chain on a local value exactly as the legacy blocks - chained on self.page_size.""" + """Resolve MLA/TRTLLM page-size constraints, chaining adjustments on the local value.""" page_size = view.page_size if ( view.attention_backend == "flashmla" @@ -1251,10 +1142,7 @@ def _mla_backend_page_constraints(view: Any) -> dict: @register_post_process def _mla_kv_cache_dtype_checks(view: Any) -> dict: - """Read-only validation pass in the attention-backend compatibility - handler: the TRT-LLM and tokenspeed MLA backends constrain the resolved - kv-cache dtype (declarations never reach the field, so the checks read - the view).""" + """Validate resolved KV-cache dtype for TRTLLM and tokenspeed MLA backends.""" if ( view.attention_backend == "trtllm_mla" or view.decode_attention_backend == "trtllm_mla" @@ -1296,10 +1184,7 @@ def _hisparse_validation(view: Any) -> dict: @register_post_process def _cutedsl_prefill_backend_fill(view: Any) -> dict: - """Slot pass in the attention-backend compatibility handler: CuteDSL MLA - is decode-only, so validate the combination and default the prefill side - to trtllm_mla. The trtllm_mha check that follows at the legacy slot reads - the resolved value through the view.""" + """Validate decode-only CuteDSL MLA and default its prefill backend to trtllm_mla.""" if not ( view.attention_backend == "cutedsl_mla" or view.decode_attention_backend == "cutedsl_mla" @@ -1484,9 +1369,7 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict: @register_post_process def _dp_lm_head_validation(view: Any) -> dict: - """Read-only validation pass: dp-attention is a prerequisite for the - dp LM head and the TP LM-head all-to-all path. Reads the mid-resolution - values through the view.""" + """Require DP attention for DP LM head and TP LM-head all-to-all.""" if view.enable_dp_lm_head: assert view.enable_dp_attention, ( "Please enable dp attention when setting enable_dp_lm_head. " @@ -1512,10 +1395,7 @@ def _dp_lm_head_validation(view: Any) -> dict: @register_post_process def _moe_runner_backend_quant_constraints(view: Any) -> dict: - """The quantization-driven moe_runner_backend resolutions at the head of - _handle_moe_kernel_config. The backend-compatibility asserts and the - disable_shared_experts_fusion writes (post-publish writers exist for that - field) stay in the handler.""" + """Resolve MoE runner backends from quantization.""" moe_runner_backend = view.moe_runner_backend if view.quantization == "nvfp4_online": if not get_platform().is_sm100: @@ -1580,10 +1460,10 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict: @register_post_process def _moe_runner_fusion_disable(view: Any) -> dict: - """FlashInfer CuteDSL / TRT-LLM / TRT-LLM-routed MoE runners require the - shared-experts fusion disabled; declared at the legacy write slots in - _handle_moe_kernel_config (before the deprecated cutlass env override, so - the runner value observed is the pre-override one).""" + """Disable shared-expert fusion for incompatible MoE runners. + + Runs before the deprecated cutlass environment override. + """ runner = view.moe_runner_backend if runner == "flashinfer_cutedsl": logger.warning( @@ -1605,10 +1485,7 @@ def _moe_runner_fusion_disable(view: Any) -> dict: @register_post_process def _a2a_fusion_adjustments(view: Any) -> dict: - """A2A-backend-driven shared-experts fusion adjustments, declared at the - legacy write slots in _handle_a2a_moe: Waterfill requires the - fusion enabled; FlashInfer, FlashInfer MegaMOE, and DeepEP v2 A2A require it disabled. - """ + """Set shared-expert fusion requirements for the selected A2A backend.""" if view.moe_a2a_backend in ("deepep", "megamoe") and view.enable_waterfill: if view.disable_shared_experts_fusion: logger.warning( @@ -1769,8 +1646,7 @@ def _dllm_page_size(view: Any) -> dict: ) return {"page_size": config.block_size} if view.page_size > config.block_size: - # Legacy scheduler-init fallback, folded into the pass: the page - # size must not exceed the dllm block size. + # The page size must not exceed the DLLM block size. logger.warning( "WARNING: " f"The page size {view.page_size} should not be larger than dllm block size {config.block_size}." @@ -1784,10 +1660,7 @@ def validate_declarations( server_args: Any, declarations: Sequence[Tuple[str, Dict[str, Any]]], ) -> None: - """Fail-fast whitelist check at declaration time: a registry typo or a - not-yet-resolvable field must be rejected at its slot, not only at - publish time. Declarations never mutate ``server_args``. - """ + """Reject unknown or non-resolvable fields before publication.""" # Non-dataclass fixtures carry no Arg metadata (mirrors the # resolvable_fields escape); only real ServerArgs is validated. if not is_record(server_args): @@ -1805,10 +1678,7 @@ def validate_declarations( @register_post_process def _hrm_text_attention_force(view: Any) -> dict: - """HRM-Text's bidirectional prefix attention only works on the Triton - backend. Invoked as the last attention declaration of the resolution - (mirroring the legacy runner-side force, which ran after the whole - pipeline).""" + """Force Triton for HRM-Text bidirectional prefix attention after backend resolution.""" if view.attention_backend not in (None, "triton"): logger.warning( f"Overriding --attention-backend " diff --git a/python/sglang/srt/arg_groups/pipeline.py b/python/sglang/srt/arg_groups/pipeline.py index acdb839217..d7490eefe3 100644 --- a/python/sglang/srt/arg_groups/pipeline.py +++ b/python/sglang/srt/arg_groups/pipeline.py @@ -1,10 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -"""The resolution pipeline: the ordered dispatcher every publishing entry runs. - -``ServerArgs.resolve_once`` is the only caller. It lives here rather than on the -record because a step decides *about* the record; none of them is a member of -it. -""" +"""Ordered configuration resolution, called by ``ServerArgs.resolve_once``.""" from __future__ import annotations @@ -22,32 +17,10 @@ from sglang.srt.arg_groups.resolution_hooks import run_hook def run_resolution_pipeline(server_args: Any) -> None: - """ - Orchestrates the handling of various server arguments, ensuring proper configuration and validation. + """Resolve and validate server arguments in dependency order. - Dispatcher style principles: - 1. Keep this function as an ordered dispatcher. Each step should be a - named call into an ``arg_groups`` family; put imports, conditionals, - mutations, and raises inside the family instead of inline here. - 2. Keep the dummy-model boundary as early as correctness allows. Only - model-independent bootstrap, API/network/protocol validation, and - errors that should fire for dummy models should run before it. - 3. Order handlers by dependency domains, not by historical insertion: - internal/bootstrap, API/network/protocol, model source/path - resolution, hardware/platform, model-specific adjustment, - parallelism, kernel/attention backend, cuda graph, memory/cache, - and advanced/debug features. - 4. Hide narrow integrations behind general handler names. The - dispatcher should say what phase is being handled, not expose a - vendor-, hook-, or feature-specific implementation detail. - 5. Give each handler one clear contract: what state it expects, what it - may mutate, and whether it validates only. Long ordering comments - belong in the helper or signal that the helper should be split. - 6. Call each step through ``run_hook(handle_x, server_args, ...)``, not - ``handle_x(server_args, ...)`` directly -- see - ``arg_groups/resolution_hooks.py``. This is every step's fixed - position in the pipeline either way; registering an override changes - what runs here, never when. + Keep model-independent validation before the dummy-model return. Call steps + through ``run_hook`` so plugins can replace them without changing their order. """ # What the caller asked for, before any handler runs; this plus the @@ -128,20 +101,14 @@ def run_resolution_pipeline(server_args: Any) -> None: run_hook(handle_model_source_paths, server_args) - # Validate mm_process_config. run_hook(handle_multimodal, server_args) - # Validate SSL arguments early. run_hook(handle_ssl_validation, server_args) - # Validate transcription/ASR-specific server args. run_hook(handle_asr_validation, server_args) - # Handle deprecated arguments. run_hook(handle_deprecated_args, server_args) - # Handle deprecated environment variables for prefill delayer. run_hook(handle_prefill_delayer_env_compat, server_args) - # Set missing default values. run_hook(handle_missing_default_values, server_args) # expert_pack may replace a raw GGUF input with its generated local @@ -207,7 +174,6 @@ def run_resolution_pipeline(server_args: Any) -> None: # handle_gpu_memory_settings so the chunk size feeds memory budgeting. run_hook(apply_glm5_chunked_prefill_default, server_args) - # Handle device-specific backends. from sglang.srt.arg_groups.platform_hook import ( handle_amd_specifics, handle_cpu_backends, @@ -231,12 +197,10 @@ def run_resolution_pipeline(server_args: Any) -> None: run_hook(handle_platform_defaults, server_args) - # Handle memory-related, chunked prefill, and CUDA graph batch size configurations. from sglang.srt.arg_groups.memory_hook import handle_gpu_memory_settings run_hook(handle_gpu_memory_settings, server_args) - # Apply model-specific adjustments. from sglang.srt.arg_groups.model_hook import ( handle_model_capability_adjustments, handle_model_specific_adjustments, @@ -247,7 +211,6 @@ def run_resolution_pipeline(server_args: Any) -> None: # After the model overrides: Qwen4-Exp declares the PLE offload default there. run_hook(handle_offload_compatibility, server_args) - # Set kernel backends. run_post_process_pass(server_args, _sampling_backend_default) # Must run before _handle_attention_backend_compatibility so the # deterministic backend is set before auto-detection fills it in. @@ -291,19 +254,14 @@ def run_resolution_pipeline(server_args: Any) -> None: # _validate_prefill_only_disable_kv_cache_args(). run_hook(handle_prefill_only_disable_kv_cache, server_args) - # Handle Hicache settings. run_hook(handle_hicache, server_args) - # Handle data parallelism. run_hook(handle_data_parallelism, server_args) - # Normalize load balancing defaults. run_hook(handle_load_balance_method, server_args) - # Handle context parallelism. run_hook(handle_context_parallelism, server_args) - # Handle MoE configurations. from sglang.srt.arg_groups.moe_hook import ( handle_a2a_moe, handle_moe_kernel_config, @@ -320,11 +278,8 @@ def run_resolution_pipeline(server_args: Any) -> None: run_hook(handle_elastic_ep, server_args) run_hook(validate_experimental_sgl_marlin, server_args) - # Handle pipeline parallelism. run_post_process_pass(server_args, _pipeline_parallel_overlap_disable) - # Handle speculative decoding logic. - from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding run_hook(handle_speculative_decoding, server_args) @@ -343,26 +298,20 @@ def run_resolution_pipeline(server_args: Any) -> None: run_hook(validate_mega_moe_token_budget_for_model, server_args) - # Handle model loading format. run_hook(handle_load_format, server_args) - # Handle Encoder disaggregation. run_hook(handle_encoder_disaggregation, server_args) - # Validate tokenizer settings. run_hook(handle_tokenizer_batching, server_args) - # Propagate environment variables. run_hook(handle_environment_variables, server_args) - # Validate cache settings. run_hook(handle_cache_compatibility, server_args) run_hook(handle_page_major_kv_layout, server_args) run_hook(handle_unified_memory_pool, server_args) - # Handle diffusion LLM inference. from sglang.srt.arg_groups.dllm_hook import handle_dllm_inference run_hook(handle_dllm_inference, server_args) @@ -370,14 +319,11 @@ def run_resolution_pipeline(server_args: Any) -> None: # Handle crash dump environment variables (must run before CUDA init). run_hook(handle_crash_dump_env, server_args) - # Handle debug utilities. run_hook(handle_debug_utils, server_args) - # Handle any other necessary validations. run_hook(handle_other_validations, server_args) - # Model-capability adjustments that legacy code applied at model-load - # time; last declarations of the resolution, mirroring that order. + # Apply model-capability constraints after backend selection. run_hook(handle_model_capability_adjustments, server_args) finalize_cuda_graph_prefill_max_context(server_args) diff --git a/python/sglang/srt/arg_groups/resolution_hooks.py b/python/sglang/srt/arg_groups/resolution_hooks.py index c94cc9873a..9422d8025e 100644 --- a/python/sglang/srt/arg_groups/resolution_hooks.py +++ b/python/sglang/srt/arg_groups/resolution_hooks.py @@ -1,45 +1,16 @@ -"""Out-of-tree replacement for a named step of the resolution pipeline. +"""Plugin overrides for named resolution steps. -`run_resolution_pipeline` calls its steps by name, hardcoded, with no -per-step dispatch through `self` -- there is nothing on `ServerArgs` left to -subclass in order to change how one step decides. This is the replacement -for that: a decorator that wraps whatever currently runs under a given name, -and a dispatcher the pipeline calls instead of the bare function. The name -itself is never spelled out at the call site -- `run_hook` reads it off the -function it was handed -- so there is exactly one place a step's name is -written by hand: the whitelist below, and whatever a downstream registrant -passes to the decorator. - -Whitelisted names only, the same discipline `Arg(resolvable=True)` uses for -declarable fields: this project has already been bitten once by an -unqualified name collision in this exact pipeline (`_parse_cuda_graph_config` -and `_handle_cuda_graph_config` merged under one rename and the dispatcher -called itself). A name that is not on the list fails loudly at import time, -not silently at the call site three modules away. - -Every step takes exactly `(server_args)`, `handle_hardware_runtime_validation` -included -- it does not read `server_args` (see the comment at its -definition), but it takes the parameter anyway so `run_hook` never has to -special-case an arity. An override's own signature always matches: `def -mine(server_args, previous)`. +Overrides receive ``(server_args, previous)`` and may call ``previous`` to +wrap the existing step. Only whitelisted names are accepted. Registration +changes the implementation at that step's existing pipeline position. """ from __future__ import annotations from typing import Any, Callable, Dict, FrozenSet, List -# Every step this pipeline runs by name that a downstream package may replace. -# Add a name here only alongside the call site's own switch to `run_hook` -- -# an entry with no matching `run_hook(...)` call is a name nothing will ever -# look up. `test_every_whitelisted_hook_has_a_call_site` in -# test_resolution_hook_registry.py holds the two directions of this equal by -# construction, so this list cannot drift from `pipeline.py` silently. -# -# `handle_offload_compatibility` runs at two different points in the pipeline -# (once before model-specific adjustments, once after -- see the comments at -# its call sites). An override registered for it applies at both, identically; -# there is no way to target "just the second call" through this mechanism, -# because the name is all `run_hook` has to key on. +# Names must match run_hook call sites in pipeline.py. An override applies +# to every invocation of that name, including repeated steps. _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset( { "handle_mega_moe", @@ -125,42 +96,16 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset( } ) -# name -> registered overrides, oldest first. Each takes `(server_args, -# previous)`, where `previous` is the callable it wraps -- the built-in on -# the first registration, the previous registrant's own wrapper on every one -# after. Process-global as a `dict[str, list]` so a test isolates it the way -# test_model_overrides.py isolates `_MODEL_OVERRIDE_FNS`: -# `patch.dict(..., clear=True)`. +# Step name -> wrappers in registration order. _HOOKS: Dict[str, List[Callable[[Any, Callable[[Any], None]], None]]] = {} def register_resolution_hook(name: str): - """Replace (or wrap) the pipeline step named ``name``. + """Register ``fn(server_args, previous)`` for a pipeline step. - The decorated function is called as ``fn(server_args, previous)``. - ``previous`` is a plain ``server_args -> None`` callable: the built-in - step on the first registration for this name, or the previous - registrant's own wrapper on every registration after that. Call it to - run what would have run without this override -- the `super().handle_x()` - shape, expressed as an explicit argument instead of a method-resolution - lookup, because there is no class hierarchy here for `super()` to walk. - Not calling it is a full replacement. - - Registering twice for the same name does not replace the first - registration; it wraps it. The **last** registration is outermost -- - runs first, and decides whether/when its `previous` (everything - registered before it, down to the built-in) runs at all. Two downstream - packages that both target the same name compose in whichever order they - happened to import in; if that order matters to you, make one of them - import the other first. - - This changes *what* runs at the step's existing position in the - pipeline, never *when*: the call site in `pipeline.py` is unmoved, so - every other step keeps the order it already had. A wrapped step's own - declarations reach `resolution_result` the same way any declaration - does -- only readers from this position onward see them; a step that - already ran and read the old value before this one's `previous` (or the - built-in) declared its replacement has already made its decision on it. + ``previous(server_args)`` calls the previously registered wrapper, or the + built-in step for the first registration. Omitting it replaces the step. + The last registration runs outermost, so plugin import order matters. """ if name not in _OVERRIDABLE_HOOKS: raise ValueError( @@ -178,22 +123,10 @@ def register_resolution_hook(name: str): def run_hook(builtin: Callable[[Any], None], server_args: Any) -> None: - """Run ``builtin`` -- the registered chain if anything overrode it under - its name, ``builtin`` directly otherwise. + """Run registered wrappers for ``builtin.__name__``, or the built-in step. - The name is ``builtin.__name__``, not a second argument: the call site - already has the function in scope (a function-local import, same as - every other step), and spelling the name out again next to it is the - exact "two copies that can silently disagree" shape this project keeps - removing elsewhere. ``builtin`` must therefore be a plain, named - function -- every real call site is -- not a lambda or a bound method. - - Called from the step's fixed position in `run_resolution_pipeline`. The - chain is rebuilt from the registry on every call rather than cached at - registration time, because at registration time (import time, before any - `ServerArgs` exists) there is no `server_args` yet and the first - registrant's `previous` cannot be bound to anything real until a call - actually happens. + ``builtin`` must be a named function. Wrappers receive the same server + arguments as the built-in step. """ step = builtin for fn in _HOOKS.get(builtin.__name__, ()): diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index 8395248230..ed8ac84d61 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -11,35 +11,16 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""A single structured accessor for process-static runtime state. +"""Process-wide configuration, parallel state, flags, and resources. -``get_parallel()`` returns a ``ParallelContext`` for configuration, ranks, and -process-group handles. Reads use scoped overrides, then permanent overrides, -then the published configuration. ``publish`` records ranks from ``SpawnRanks``; -distributed initialization records group handles. +``get_server_args()`` retains the raw input for diagnostics. Namespace +accessors such as ``get_exec()`` return resolved configuration published +from declarations; ``RuntimeContext.override`` changes those bags. +``get_parallel()`` also exposes ranks and process groups, with scoped and +permanent overrides taking precedence over published values. -``get_server_args()`` returns the process-wide ``ServerArgs``. This is the -user's raw input, kept **read-only** for debug and reproduction; what -resolution decided lives in the declarations (``resolution_result``) and, for -business code, in the namespace bags below -- never on this object's fields. The context owns the storage: -publishing goes through ``RuntimeContext.set_server_args`` (the legacy -``set_global_server_args_for_scheduler`` is a thin shim over this slot; -``get_global_server_args`` is retired and raises). - -``get_exec()`` / ``get_memory()`` / ``get_schedule()`` / ``get_device()`` / -``get_model()`` / ``get_spec()`` / ``get_lora()`` / ``get_mm()`` / -``get_disagg()`` / ``get_serving()`` / ``get_observability()`` return the -resolved **config namespace bags** — the single source of truth for config, -snapshotted from ``server_args`` at publish, one bag per namespace class in -``arg_groups/fields/`` (multi-level under ``exec.*``). Reads are attribute -chains (``get_exec().moe.moe_runner_backend``); bags are read-only by bare -assignment (written via ``override``). - -``get_flags()`` returns the runtime-flags tier: state that is **not** a pure -function of config (the capture lifecycle, ACTIVE MoE backend, DP runtime) — -never a mirror of config. Flags live in typed dataclass groups; reads and -writes are plain attribute access, and each group offers a transactional, -test-only ``override(**kw)``. +``get_flags()`` holds mutable runtime state; ``get_resources()`` owns process +handles, and ``get_forward()`` holds per-forward state. """ from __future__ import annotations @@ -612,13 +593,7 @@ class SpFlags(_FlagGroupBase): class Flags(_FlagGroupBase): - """Root of the runtime-flags tier. - - Resolved configuration lives in the config bags below (projected from the - declarations at publish) — this tier only carries genuine runtime - state whose value is not a function of the configuration alone, grouped - by lifecycle (``capture``) or subsystem (``moe`` / ``dp`` / ``sp``). - """ + """Mutable runtime state; resolved configuration lives in the config bags.""" capture: CaptureFlags = msgspec.field(default_factory=CaptureFlags) moe: MoeFlags = msgspec.field(default_factory=MoeFlags) @@ -662,20 +637,12 @@ class Resources(_FlagGroupBase): class ForwardFlags: - """Per-forward runtime flags with one API and two backings. + """Scoped per-forward flags. - Flags read only from eager Python are backed by context variables, so - nested scopes and threads stay isolated (a new thread sees the defaults). - Flags that are read or written *inside torch.compile-traced model code* - (``_GRAPH_VISIBLE``) are backed by plain dict slots instead: dynamo - cannot trace ``ContextVar.get``/``set``, while plain reads it guards on - — the storage form these flags had before joining the tier. Their - writers and readers are single-threaded per process (TBO interleaves - ubatches on one thread; attention-TP input scattering excludes TBO), so - context isolation is not needed for correctness. - - ``scoped(**kw)`` — the one regular write path — restores on exit for - both backings. ``set()`` exists for the legacy unscoped setters' shims. + Eager-only flags use ContextVars for thread and nested-scope isolation. + ``_GRAPH_VISIBLE`` flags use plain dict slots because Dynamo cannot trace + ContextVar access. Their readers and writers run on one thread per process. + ``scoped`` restores both backings on exit; ``set`` supports unscoped setters. """ _DEFAULTS = { @@ -792,24 +759,12 @@ class ForwardFlags: class _ConfigBag: - """A resolved-config namespace bag. + """Resolved namespace with nested bags and guarded writes. - Values are snapshotted from ``server_args`` at ``publish`` and this bag is - the **single source of truth** for its fields thereafter. Read is plain - attribute access; the bag is read-only by bare assignment. The sanctioned - writers are ``get_context().override(source, ...)`` (permanent) and - the scoped ``.override(**kw)`` context manager (tests). Sub-namespaces - (e.g. ``exec.moe``) are nested ``_ConfigBag`` instances reached by attribute. - - Leaves and sub-bags are stored as **real instance attributes** (in - ``__dict__``), so ``bag.leaf`` / ``bag.sub`` is a plain attribute load that - ``torch.compile`` / dynamo can trace — config reads inside a compiled model - forward (e.g. ``get_exec().comm.enable_symm_mem`` in the embedding layer) - must not graph-break. ``_fields`` / ``_subs`` keep the authoritative - name→value maps used for override routing, membership, and scoped restore; - ``__getattr__`` is only a fallback for genuinely absent names. (Deliberately - no ``__slots__``: leaves are dynamic, and the ``__dict__`` is what makes the - reads traceable.) + Publication snapshots resolved values; ``RuntimeContext.override`` updates + them permanently and ``override`` restores scoped changes on exit. + Leaves and sub-bags are real instance attributes so Dynamo can trace reads. + ``_fields`` and ``_subs`` track names for routing and restoration. """ def __init__(self, path: str): @@ -837,14 +792,12 @@ class _ConfigBag: ) def _set(self, name: str, value: Any) -> None: - """Internal write (publish + override) that bypasses the read-only guard. - Updates both the bookkeeping map and the real attribute (traceable read).""" + """Update the leaf map and traceable attribute, bypassing the write guard.""" object.__getattribute__(self, "_fields")[name] = value object.__setattr__(self, name, value) def _set_sub(self, name: str, sub: _ConfigBag) -> None: - """Register a nested bag as both a bookkeeping entry and a real - attribute (so ``bag.sub`` is a plain, traceable attribute load).""" + """Register a nested bag in the lookup map and as a traceable attribute.""" object.__getattribute__(self, "_subs")[name] = sub object.__setattr__(self, name, sub) @@ -853,13 +806,10 @@ class _ConfigBag: @contextmanager def override(self, **kwargs): - """Scoped, transactional override of this bag's own leaves (keys - validated before any write; restored on exit). + """Temporarily override this bag's leaves, validating all keys before writing. - For a window where one runner's value differs from the process's — a - draft model loading under ``--speculative-draft-load-format`` while the - target keeps ``--load-format`` — and for tests forcing a code path. - A permanent change goes through ``get_context().override``.""" + Restores on exit; use ``RuntimeContext.override`` for permanent changes. + """ fields = object.__getattribute__(self, "_fields") unknown = set(kwargs) - set(fields) if unknown: @@ -876,16 +826,7 @@ class _ConfigBag: def _build_config_bags(server_args: Any) -> dict: - """Snapshot the resolution result into the namespace bag tree. - - The tree is ``namespace_of``: each field is placed by the namespace class - that declares it (``arg_groups/fields/``). Each leaf comes from - ``resolution_result`` -- the declaration if resolution made one, else what - the caller supplied. Returns ``{top_level_name: _ConfigBag}``, arbitrarily - nested (``exec.moe.eplb.…``). Only dataclass fields are placed, so derived - properties and methods are naturally excluded (they stay on the bag). A - name used as both a leaf and a subgroup at the same level is a hard error - — no silent shadowing.""" + """Project resolved fields by namespace, rejecting leaf/subgroup name collisions.""" from sglang.srt.arg_groups.arg_utils import namespace_of from sglang.srt.arg_groups.overrides import resolution_result @@ -894,10 +835,6 @@ def _build_config_bags(server_args: Any) -> dict: for field, path in namespace_of(type(server_args)).items(): value = resolution_result(server_args, field, _MISSING) if value is _MISSING: - # Every placed field is a dataclass field, so a resolved config - # always carries it; a miss means a malformed/partial config object - # was published. Fail loud here rather than silently omitting the - # leaf (which surfaces later as a confusing "not a published leaf"). raise AttributeError( f"config field {field!r} belongs to namespace {path!r} but is absent from " f"the published {type(server_args).__name__}; cannot project its bag leaf" @@ -930,18 +867,9 @@ def _build_config_bags(server_args: Any) -> dict: def _install_derived_leaves(tops: dict, server_args: Any) -> None: - """Compute the declared config-derived fields into their bags. + """Compute declared derived leaves once at publication. - A `Derived(fn=...)` is a pure function of the published configuration, so it - is computed once, here, and stored as an ordinary leaf: readers get a plain - attribute load, and there is one answer rather than a pre-publish spelling - and a post-publish one that have to be kept saying the same thing. - - The function is handed the whole resolved config, not the bag it lands in. - A derivation is free to span namespaces and they do -- the mamba - extra-buffer predicate reads `memory.disable_radix_cache` alongside its own - `exec.mamba` strategy -- which is exactly why it cannot be written as a - method on either bag. + Derivations receive the whole resolved config because they may span namespaces. """ import importlib @@ -990,9 +918,6 @@ def _resolved_or_field(server_args: Any, name: str, default: Any) -> Any: decided = resolution_result(server_args, name) if decided is not None: return decided - # The default is for the callers that hand over something record-shaped but - # not a record -- the fake configs the context tests publish, and `object()` - # for the sentinel publish. A real ServerArgs always has the field. return getattr(server_args, name, default) @@ -1023,10 +948,7 @@ class RuntimeContext: self.forward = ForwardFlags() def get_stream(self, name: str) -> Any: - """Named process-level side stream: get-or-create, shared by - name (the keyed-lazy pattern of the persistent buffers). Creation is - a driver call that must stay outside cuda-graph capture — call sites - lease their stream at init/warmup time.""" + """Get or create a named process stream. Call before CUDA graph capture.""" from sglang.srt.arg_groups.overrides import resolution_result stream = self.resources.streams.get(name) @@ -1049,9 +971,7 @@ class RuntimeContext: return stream def get_buffer(self, name: str, factory: Any) -> Any: - """Named process-level persistent buffer: get-or-create via - ``factory()``, shared by name (the keyed-lazy pattern of the - persistent buffers / named streams).""" + """Get or create a named persistent buffer using ``factory()``.""" buf = self.resources.buffers.get(name) if buf is None: buf = factory() @@ -1068,24 +988,11 @@ class RuntimeContext: return server_args def set_server_args(self, server_args: ServerArgs) -> None: - """Publish the process-wide ``ServerArgs`` into the context-owned slot. - - Overwrite-allowed: a re-publish replaces the slot (test kits re-publish - per test; production ordering discipline lives at the call-sites, e.g. - the draft-worker guard in ``ModelRunner.__init__``). The published - object is the raw input; the resolution it carries is its declaration - stash, which is what the bags are projected from. - """ - # Seed the capture tier for the new lifecycle (defaults for sentinel - # and mock publishes, which carry no config). Through the resolution, - # not the field: the field is the operator's input. + """Replace the raw input and rebuild resolved config bags, clearing override provenance.""" self.flags.capture.enable_torch_compile = bool( _resolved_or_field(server_args, "enable_torch_compile", False) ) self._server_args = server_args - # Snapshot resolved config into the namespace bags (the single source of - # truth for config reads). Placed by `namespace_of`; a mock/partial - # config that declares no namespace yields an empty tree (no bags). # Preserve the launcher-assigned device when rebuilding config bags. stated = {} if self._config_bags is not None: @@ -1109,19 +1016,13 @@ class RuntimeContext: "max_speculative_num_draft_tokens", max_draft_tokens_of(server_args), ) - # Wire the published `parallel` bag onto the live wrapper: it is the slot - # the `config` property reads, which is how config-only leaves like - # pp_max_micro_batch_size are spelled. self.parallel._config = self._config_bags.get("parallel") # A direct install is roleless; ``publish`` assigns the role afterwards. self._overrides_log = [] self._publish_role = None def config_bag(self, name: str) -> _ConfigBag: - """Return the top-level config namespace bag (``device`` / ``model`` / - ``exec`` / ``schedule`` / ``memory`` / ``spec`` / ``lora`` / ``mm`` / - ``disagg`` / ``serving`` / ``observability``). Fails closed until - ``publish`` / ``set_server_args`` has projected it.""" + """Return a top-level namespace bag; raise if it has not been published.""" bags = self._config_bags if not bags or name not in bags: raise ValueError(f"config namespace {name!r} not published") @@ -1159,16 +1060,10 @@ class RuntimeContext: ) def override(self, source: str, **fields) -> None: - """The business mutation entry: write resolved config - leaves onto the namespace bags — the single source of truth. It does - **not** touch ``server_args`` (the pristine startup record) and there is - no write-through, so the old "wrote one store, read another" desync class - cannot occur. + """Update resolved config leaves without changing the raw input. - Each flat field name is routed to its bag by ``namespace_of`` (flat - names are unique across namespaces). Validation is all-or-nothing: an - unknown / unprojected field aborts before any write. ``source`` is - recorded for provenance / reproduction. + Validate all field names and target bags before writing. Record ``source`` + and the changed fields for provenance. """ if not fields: return @@ -1204,12 +1099,7 @@ class RuntimeContext: self._overrides_log.append((source, dict(fields))) def config_leaf(self, name: str): - """One resolved config leaf by field name — the read side of ``override``. - - Callers that hold a field name rather than a namespace (a readback - endpoint, a control-plane handler) would otherwise have to know which - bag it lives in. - """ + """Read a resolved config leaf by its flat field name.""" bags = self._config_bags if bags is None: raise ValueError("config not published; cannot read a config leaf") @@ -1227,36 +1117,16 @@ class RuntimeContext: return getattr(bag, name) def overrides_log(self) -> list: - """Provenance of post-publish ``override`` calls: ``[(source, {field: value})]``. - - Returns deep-ish copies (source, dict(fields)) so callers inspecting the - log cannot mutate the recorded provenance in place.""" + """Return ``(source, fields)`` entries with shallow copies of the field dicts.""" return [(source, dict(fields)) for source, fields in self._overrides_log] def resolved_server_args_dict(self, base: dict | None = None) -> dict: - """Serialize the *resolved* config: the pristine ``server_args`` fields - with every post-publish ``override`` overlaid. + """Overlay this process's runtime overrides on the resolved startup config. - ``get_internal_state`` reports this, and ``/server_info`` carries it in - the ``internal_states`` block, so scheduler-side runtime changes show up - in a readback: HiCache attach/detach, the generated forward-pass-metrics - endpoint, tunables set via ``/set_internal_state``. - - ``base`` defaults to ``server_args.resolved_dict()`` -- the record's - fields as resolution decided them, nested dataclasses expanded. (It used - to be ``dict(vars(server_args))``, which carried the private resolution - bookkeeping and the ``model_config`` memo into the readback.) Override - leaves are flat ``ServerArgs`` field names, so overlaying them onto the - top level of the base is exact. - - The log is per process: it carries what *this* process overrode. A - weight reload records ``model_path`` and ``load_format`` from the - scheduler process (``ModelRunner.update_model_fields``); the tokenizer - process records only ``load_format`` and keeps ``model_path`` / - ``served_model_name`` as ``TokenizerManager`` attributes, which - ``TokenizerManager.resolved_config_dict`` overlays on top of this dump. - The top-level ``/server_info`` fields are the startup record, not this - dump. + ``base`` defaults to ``server_args.resolved_dict()``. Scheduler readbacks + include these overrides in ``/server_info``'s ``internal_states`` block; + the top-level fields describe startup config. TokenizerManager separately + overlays its per-instance model identity. """ d = self.server_args.resolved_dict() if base is None else dict(base) for _source, fields in self._overrides_log: @@ -1264,24 +1134,10 @@ class RuntimeContext: return d def override_server_args(self, **fields) -> _ServerArgsOverride: - """Test-only scoped override for the config tier — the sibling of - ``get_parallel().override()`` and the flag groups' ``override()``: - tests force execution paths by overriding the context instead of - hand-building config objects. + """Publish a temporary dummy config for tests and restore the previous context. - ``install()`` (or entering it as a context manager) publishes a fresh - dummy-boundary ``ServerArgs`` carrying ``fields`` and returns it; - ``restore()`` (or exiting) reinstates whatever the slot held before. - - This is the sanctioned way for a test to get a published context, and - it stays. The transitional reason it was introduced for — production - code branching on raw ``server_args`` fields at runtime — is gone (the - read ratchet pins business reads at zero), but a test that exercises - bag readers still needs bags, and the bag tree is projected *from an - instance*: something has to publish one. Prefer the finer-grained - scoped overrides (``get_exec().override(...)``, the flag groups' - ``override``) on top of a published context when a test only needs to - force one leaf. + Use as a context manager or call ``install`` / ``restore`` explicitly. + Prefer a bag's scoped ``override`` when only existing leaves need changing. """ return _ServerArgsOverride(self, fields) @@ -1341,19 +1197,11 @@ class _ServerArgsOverride: raise ValueError( f"override_server_args: unknown ServerArgs field(s): {sorted(unknown)}" ) - # Declared so the projection sees it; late, because the record is - # resolved already and not yet published. - # Split on whether the name is a field, not on whether it starts with - # an underscore: `_speculative_draft_quantization_explicitly_set` is a - # real field, and seeding it as a raw attribute would leave the earlier - # declaration authoritative, so `resolution_result` and the bag would - # both keep answering the pre-override value. + # Declare config fields even when underscore-prefixed; only non-fields seed caches. fields = set(type(server_args).__struct_fields__) declared = {n: v for n, v in self._fields.items() if n in fields} if declared: declare_resolution(server_args, "override_server_args", **declared) - # What is left seeds the record's own private caches (`_model_config` - # and friends), which are not configuration and never were. seeds = {n: v for n, v in self._fields.items() if n not in fields} for name, value in seeds.items(): msgspec.Struct.__setattr__(server_args, name, value) @@ -1462,7 +1310,7 @@ def get_observability() -> _ConfigBag: return _CONTEXT.config_bag("observability") -# --- Per-role namespace sets (2c) ------------------------------------------- +# --- Per-role namespace sets ------------------------------------------- # # ``publish(role=...)`` records which process type installed the config; this # table declares which top-level config namespaces each role reads. ``None`` @@ -1483,22 +1331,10 @@ ROLE_NAMESPACE_SETS: dict[str, frozenset[str] | None] = { # Reads (almost) everything by design — the model-executing process. "scheduler": None, "test": None, - # The DP controller's static read set, checked against the module: the - # elastic-EP gate, the load-balance method, the watchdog timeout, and the - # disaggregation mode. - # `observability` and `serving` were added when the controller's metrics - # gate, tracing setup and worker-port broadcast stopped reading the record: - # under `enforce` the set is what the process may read, so a conversion - # that reaches a new namespace has to widen it in the same commit. "dp_controller": frozenset( {"exec", "parallel", "device", "disagg", "observability", "serving"} ), - # Record-mode audit (2026-08-06, text model, /generate + /get_server_info + - # /v1/models): reads exactly {"serving"} — the per-instance managers read - # self.server_args by design. Still declared full, because that run did not - # exercise the multimodal processors, LoRA/score endpoints, the disagg - # roles, or the gRPC bridge; narrowing needs those shapes audited too, and - # a wrong set fails a request rather than a test. + # Keep unrestricted until multimodal, LoRA, disaggregation, and gRPC reads are audited. "tokenizer": None, # Deployment shapes not exercised locally; audit before restricting. "detokenizer": None, @@ -1712,16 +1548,9 @@ def _attention_ranks(parallel, tp_rank: int) -> dict: def assert_published(server_args, *, role: str) -> RuntimeContext: - """This record, under this role, is already published -- or fail loud. + """Require this record and role to have been published by the process entry. - Publishing is the process entry's job: `run_scheduler_process`, - `init_multi_tokenizer`, a spawned encoder worker, the benchmark work - functions. A constructor arriving here unpublished means one of those - entries is missing. - - A `publish` at this point re-projects the bags over a live process, - discarding every `override()` taken since and the provenance log with it, - so this raises. + Re-publishing here would discard runtime overrides and their provenance. """ if _CONTEXT._server_args is server_args and _CONTEXT._publish_role == role: return _CONTEXT @@ -1866,13 +1695,9 @@ def reset_context() -> None: def remote_instance_transfer_engine_enabled(load_format: str | None = None) -> bool: - """Whether remote-instance weight loading runs over the transfer engine. + """Check transfer-engine loading against current model config. - Every input is a ``model`` leaf, so this derives from the bags and follows a - post-publish override; ``ServerArgs.remote_instance_weight_loader_use_transfer_engine`` - is the pre-publish equivalent, and both go through the same helper. - ``load_format`` is the caller's own (a draft runner loading under - ``--speculative-draft-load-format`` has one the process record does not). + ``load_format`` can override the process value for a draft runner. """ from sglang.srt.arg_groups.overrides import remote_instance_transfer_engine_of @@ -1880,15 +1705,9 @@ def remote_instance_transfer_engine_enabled(load_format: str | None = None) -> b def max_prefill_buffer_tokens() -> int: - """The prefill-buffer ceiling: ``chunked_prefill_size``, except PP dynamic - chunking can grow chunks toward ``max_prefill_tokens`` and probe at 1.25x. + """Return the registered prefill ceiling, or derive it from current config. - The default derives from published leaves (``schedule`` plus the configured - PP size), so it follows post-publish overrides; - ``overrides.max_prefill_buffer_tokens`` is the pre-publish equivalent and - ``TestDerivedPredicatesAgreeAcrossTiers`` pins the two equal. Records with - a registered ceiling provider (see ``register_prefill_buffer_ceiling``) - answer through it. + PP dynamic chunking can grow toward ``max_prefill_tokens`` and probe at 1.25x. """ schedule = get_schedule() chunked = ( @@ -1905,14 +1724,7 @@ def max_prefill_buffer_tokens() -> int: def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float: - """The activation working-set reserve held back before cuda-graph capture. - - Derived from published leaves across four bags (``disagg`` / ``schedule`` / - ``exec.graph`` / ``spec``) plus the configured parallel sizes, so it follows - a post-publish override; ``pre_capture_activation_reserve_mb_of`` in - ``arg_groups.overrides`` is the config-shaped equivalent and - ``TestDerivedPredicatesAgreeAcrossTiers`` pins the two equal. - """ + """Return the activation reserve in MB before CUDA graph capture, using current config.""" schedule = get_schedule() if get_disagg().disaggregation_mode == "decode": running_requests = ( @@ -1936,12 +1748,7 @@ def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float: return reserved_mem -# --- Platform facts ----------------------------------------------------------- -# -# One address for what kind of machine this is, so a reader asks -# `get_platform().is_sm100` and an override is stated once instead of patched -# into every module that imported a probe. True before publish, so the context -# probes when no override is installed; `utils.common` holds the implementation. +# Platform probes work before publication and support scoped overrides. _PLATFORM_PROBES: Dict[str, str] = { "is_cuda": "is_cuda", @@ -2072,14 +1879,7 @@ def override_platform(**facts: Any) -> _PlatformOverride: return _PlatformOverride(**facts) -# --- Derived config accessors ------------------------------------------------ -# -# A few values are computed from several config fields plus the HF config, so -# they are derived accessors rather than namespace leaves. Business code must -# not reach for the startup record to get them: these accessors are the named -# home, and this module — which owns the slot — is the only place that reads -# it. Each one keeps the pre-publish function's exact semantics, including which model -# config it derives from (always the process's, i.e. the target's). +# Derived config accessors use the process model (the target during speculation). def mamba_cache_chunk_size() -> int: @@ -2146,8 +1946,6 @@ def attention_backends() -> tuple: """ from sglang.srt.arg_groups.overrides import attention_backends_of - # All three leaves live in the same bag, so the resolution pipeline's own - # helper applies directly -- one definition of the fallback rule. return attention_backends_of(get_exec().kernel) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 10d9ff06de..a49b230099 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -11,25 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Server argument declarations, resolution, and CLI registration. - -Keep this file in the following top-level order: - -1. Imports and the module logger. -2. Public extension-point choice lists, with each legacy ``add_*`` alias - immediately below the choice list it extends. -3. Shared (non-extensible) choice lists, scalar defaults, and deprecated - aliases. A choice list used by only one field belongs inline in that field. -4. ``ServerArgs``: fields first, then resolution/validation helpers, then CLI - registration and small query helpers. New resolution steps are appended at - the end of ``arg_groups.pipeline.run_resolution_pipeline``, immediately - marked complete, unless an earlier dependency is documented explicitly. -5. Module-level ``ServerArgs`` construction/runtime shims. -6. Networking constants and ``PortArgs``. - -Model- or vendor-specific utilities belong in ``sglang.srt.arg_groups`` (or -their owning subsystem), not before ``ServerArgs`` in this module. -""" +"""Server argument construction, resolution, CLI registration, and network ports.""" from __future__ import annotations @@ -96,16 +78,11 @@ def _real_kv_hash_modes(): return list(RealKvHashMode) -# Re-exported. These were importable from this module while the field -# declarations that used them lived here; the declarations moved to -# `arg_groups/fields/` but out-of-tree code -- and `tokenizer_control_mixin` -# for `LoRARef` -- still reaches them through `sglang.srt.server_args`. +# Compatibility re-exports for callers importing through server_args. from sglang.srt.arg_groups.arg_utils import NS, A, Arg # noqa: F401 from sglang.srt.arg_groups.argparse_actions import LoRAPathAction # noqa: F401 -# Re-exported for out-of-tree plugins, which have always reached these -# through `sglang.srt.server_args`. The lists and their adders moved to -# `arg_groups/choices.py` with the field declarations that name them. +# Public choice lists and plugin registration helpers. from sglang.srt.arg_groups.choices import ( # noqa: F401 ATTENTION_BACKEND_CHOICES, CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS, @@ -219,10 +196,7 @@ def __getattr__(name: str) -> Any: def _plain(value: Any) -> Any: - """``asdict``'s conversion, applied to one value: a record -- Struct or - dataclass, since nested config values are both -- becomes a dict, containers - recurse, everything else is deep-copied (a caller mutating the dump must not - reach the live configuration).""" + """Convert nested Structs and dataclasses to dicts, copying all values.""" if not isinstance(value, type) and is_record(value): return { field.name: _plain(getattr(value, field.name)) @@ -238,74 +212,23 @@ def _plain(value: Any) -> Any: class ServerArgs: - """Server-wide configuration for SGLang. + """Raw server configuration, sealed when resolution starts. - Adding new arguments - -------------------- - 1. **Place the field in the right section.** Arguments are grouped by - comment blocks (``# Model and tokenizer``, ``# LoRA``, etc.). - Add new fields to the matching section, or create a new section - with a ``# ---`` banner when none fits. - - 2. **Use the ``A[T, ...]`` annotation.** ``A`` is an alias for - ``typing.Annotated``. The primary CLI flag is auto-derived from the - field name (``tp_size`` → ``--tp-size``). Use ``aliases`` for - longer alternate names - (``aliases=["--tensor-parallel-size"]``):: - - # Bare string — simplest form (just help text): - host: A[str, "The host of the HTTP server."] = "127.0.0.1" - trust_remote_code: A[bool, "Whether to allow custom models."] = False - - # Arg(...) — when you need choices, aliases, type_parser, etc.: - load_format: A[str, Arg(help="...", choices=CHOICES)] = "auto" - model_path: A[str, Arg(help="...", aliases=["--model"])] - - See ``Arg`` in ``arg_groups/arg_utils.py`` for the full list of - supported metadata (``choices``, ``aliases``, ``type_parser``, - ``nargs``, ``const``, ``action``, ``no_cli``, …). - - 3. **Manual entries in ``add_cli_args`` — only for special cases.** - A few arguments cannot use the annotation style and must be - registered manually in ``add_cli_args``: - - - **Deprecated flags** that redirect to another field via one of the - ``Deprecated*Action`` classes in ``arg_groups/argparse_actions.py`` - (that module's header says which shape fits which migration). - - **Dynamic choices** computed at runtime (e.g. ``reasoning_parser`` - whose choices come from a plugin registry). - - The ``--config`` meta-argument (not a dataclass field). - - Everything else should use the ``A[T, ...]`` annotation. - - The fields live in ``arg_groups/fields/`` -- one class per config - namespace, assembled here rather than inherited, so what the record holds - is one readable call and not a property of a base-class list. The order - they appear in is ``POSITIONAL_FIELD_ORDER``, not the order the namespaces - are listed in: field order *is* the positional constructor signature. + Add fields to the matching namespace in ``arg_groups/fields/`` using + ``A[T, help]`` or ``A[T, Arg(...)]``; see ``arg_utils.Arg`` for CLI metadata. + Only dynamic choices, deprecated flags, and ``--config`` need manual + registration in ``add_cli_args``. ``POSITIONAL_FIELD_ORDER`` preserves + the positional constructor signature when these namespaces are assembled. """ def __post_init__(self): - """Construction leaves the record at what the caller asked for. - - Resolution is a separate act, entered through ``resolve_once``: the - launcher runs it once per engine, and every publishing process asks the - gate on the way in. A record that is only constructed -- a fixture, a - config being inspected, one being handed to a subprocess that will - resolve it itself -- stays raw. - """ + """Leave construction unresolved; launchers and publishers call ``resolve_once``.""" def resolve_once(self) -> None: - """Run the resolution pipeline, unless this record has been through it. + """Resolve once, preserving declarations across pickling to child processes. - Resolution is a deterministic function of the raw inputs -- two records - built from the same arguments declare the same things -- but the - handlers do not survive a second pass over their own output: DP - attention halves ``chunked_prefill_size`` again on every re-entry. - - The publishing entry of every process calls this. In a child the record - arrived by pickle and brought its declarations along, so the child has - nothing left to derive and projects what the parent decided. + Handlers are not idempotent over their own output. A failed resolution + cannot be retried on the same record. """ if getattr(self, "_resolution_finished", False): return @@ -318,63 +241,30 @@ class ServerArgs: ) from sglang.srt.arg_groups.pipeline import run_resolution_pipeline - # Sealed for the duration, not just afterwards: everything below this - # line reads the input and declares against it. No exceptions -- even a - # resolver from outside this tree assigns onto a stand-in, not here. self._input_frozen = True try: run_resolution_pipeline(self) except BaseException: - # The handlers that ran already declared, and they are not - # idempotent over their own output. self._resolution_failed = True raise finally: self._input_frozen = False - # Set here too, because the dummy/absent-model path returns before the - # end of the pipeline that normally sets it: the gate is about whether - # the handlers ran, not how far they got. + # Also mark the dummy/absent-model path, which returns early from the pipeline. self._resolution_finished = True @property def launch_command(self) -> str | None: - """How this record was created, verbatim. - - `resolved_dict` answers with what resolution decided; this answers with - what the operator asked for, which is a different question and the one - "why is this server configured like this?" usually means. The two are - not derivable from each other: a field the operator never set reads the - same as one they set to the value resolution would have picked anyway. - - The launcher stores the arguments it parsed; the in-process `Engine` - stores the call that built the record, since there was no command line. - `None` on a record built directly (a fixture, a subprocess copy that - predates this, a config being inspected). - """ + """Original CLI arguments or Engine constructor call; ``None`` for direct construction.""" return getattr(self, "_launch_command", None) def resolved_dict(self) -> dict[str, Any]: - """This configuration as a plain dict of resolved field values. - - What the whole-object readbacks report (`/server_info` and its gRPC and - in-process twins). A plain `asdict` reads the fields, which - carry the raw input; this reads the declarations, so it answers with what - resolution decided. Nested dataclass fields are expanded - the way `asdict` expands them; the private resolution bookkeeping and the - `model_config` memo are not fields and do not appear. - """ + """Serialize resolved field values, expanding nested records and excluding bookkeeping.""" return { field.name: _plain(resolution_result(self, field.name)) for field in record_fields(type(self)) } - # ------------------------------------------------------------------ - # CUDA graph configuration resolution - # ------------------------------------------------------------------ - - # ===== END TO BE REFACTORED ==== - LANGUAGE_MODEL_ONLY_ARCHITECTURES = ( "MuseGlimmerForConditionalGeneration", "Cosmos3ForConditionalGeneration", @@ -392,7 +282,6 @@ class ServerArgs: @staticmethod def add_cli_args(parser: argparse.ArgumentParser): - # Auto-derived from Annotated[..., Arg(...)] field metadata. add_cli_args_from_dataclass(parser, ServerArgs) # --- Fields with dynamic choices (computed at add_cli_args time) --- @@ -493,17 +382,7 @@ class ServerArgs: return self.url(port=self.engine_info_bootstrap_port) def __setattr__(self, name, value): - # The record holds the operator's input. It is writable while the - # caller is still assembling it and sealed from the moment resolution - # starts: a resolver that writes a field would overwrite the very thing - # the record exists to remember, and the decision it meant to record - # belongs in the stash, where it carries a source and does not destroy - # the input it was derived from. - # Underscore names are mostly the record's own bookkeeping -- - # `_input_frozen`, `_raw_input`, `_resolved_overrides`, the memo slots - # -- which resolution writes on purpose. A *field* spelled that way is - # still configuration, so the test cannot be on spelling alone or that - # one leaf stays writable on a read-only record. + # Seal configuration fields, including underscore-prefixed ones, but allow bookkeeping. if not name.startswith("_") or name in _underscore_field_names(): if getattr(self, "_input_frozen", False): raise AttributeError( @@ -527,14 +406,9 @@ class ServerArgs: msgspec.Struct.__setattr__(self, name, value) def __reduce__(self): - """Pickle the record *and* what resolution left on it. + """Preserve resolution bookkeeping as well as Struct fields when pickling. - A Struct pickles its fields; everything else lives in the `dict=True` - namespace and would be dropped, which for this record means the input - snapshot, the declaration stash and the resolution flags -- the whole - reason a child can publish what its parent decided without resolving - again. Reconstruction restores the fields first and the bookkeeping - after, so the seal is re-armed only once the fields are in place. + Restore fields before bookkeeping so the write seal is re-armed last. """ return ( _rebuild_server_args, @@ -552,12 +426,7 @@ class ServerArgs: return remote_instance_transfer_engine_of(resolving_view(self), load_format) -# The namespaces whose *input* fields make up the record. A namespace declares -# its input and derived fields side by side; only the input half is collected, -# which is why this is a call and not a base-class list. -# -# A set, not an order: `collect_input_fields` orders by `field_order.py`, which -# is what keeps the positional constructor stable. +# Collect input fields only; field_order.py preserves positional argument order. _INPUT_NAMESPACES = [ @@ -586,16 +455,9 @@ _INPUT_NAMESPACES = [ _annotations, _defaults, _namespaces = collect_input_fields(_INPUT_NAMESPACES) ServerArgs.__annotations__ = {**_annotations, **ServerArgs.__annotations__} -# The assembled record has no base classes, so it carries the map the classes -# used to answer through their `_NS_PATH`. ServerArgs._NS_BY_FIELD = _namespaces -# The classes themselves, so the bag projection can find the declarations -# that are not fields -- the derived half of each namespace. ServerArgs._NAMESPACES = _INPUT_NAMESPACES -# `dict=True` so the record can carry what is not configuration -- the input -# snapshot, the declaration stash, the resolution flags, the memo slots. A -# Struct has no `__dict__` without it, and those are exactly the underscore -# names `_underscore_field_names()` is careful *not* to include. +# dict=True retains resolution bookkeeping and memoized values outside the fields. ServerArgs = msgspec.defstruct( "ServerArgs", [ @@ -661,14 +523,7 @@ def m3_fp8_attn_gemm_enabled(args) -> bool: @functools.lru_cache(maxsize=1) def _underscore_field_names() -> frozenset: - """Real dataclass fields whose names start with an underscore. - - The read-only guard exempts underscore names because they are the record's - own bookkeeping (the stash, the flags, the cache keys). A *field* that - happens to start with an underscore is still resolved configuration -- - `_speculative_draft_quantization_explicitly_set` is one -- and exempting it - by spelling would leave exactly one leaf writable on a read-only record. - """ + """Configuration fields that must stay sealed despite their underscore prefix.""" return frozenset( field.name for field in record_fields(ServerArgs) if field.name.startswith("_") ) @@ -695,15 +550,7 @@ def set_global_server_args_for_tokenizer(server_args: ServerArgs): def get_global_server_args() -> NoReturn: - """Retired. It raises, because what it used to return is the problem: the - record answers with the operator's *input*, so a caller reading a field - resolution decided got a stale value and no error. - - The name survives so that a caller importing it from this module lands on - a message instead of an ImportError. Annotated ``NoReturn`` so a type - checker rejects the call rather than accepting the attribute access after - it. The message lives once, in the exception. - """ + """Retired accessor retained to raise a migration error for existing imports.""" raise RuntimeError( "get_global_server_args() is retired. Read the value that is in effect " "from its namespace bag -- `get_exec().kernel.attention_backend`, " @@ -721,13 +568,7 @@ def _rebuild_server_args(cls, fields, bookkeeping): def _declared_default(name: str): - """The declared default of a field, for a manual `add_argument`. - - `ServerArgs.` used to answer with it. The record is a Struct now, so - that expression returns the slot descriptor instead -- which argparse - happily stores as the default, and the first reader gets a - `member_descriptor` where it expected a string. - """ + """Return a field default; Struct class attributes are slot descriptors.""" return next( field.default for field in msgspec.structs.fields(ServerArgs)