[Refactor] Trim server configuration and runtime context comments (#40971)
This commit is contained in:
@@ -72,23 +72,12 @@ class Arg(msgspec.Struct, frozen=True):
|
||||
action: Any | None = None
|
||||
action_kwargs: dict | None = None
|
||||
const: Any | None = None
|
||||
# When True, this field is skipped by add_cli_args_from_dataclass.
|
||||
# Use for fields that have no CLI surface (e.g. injected via Python only).
|
||||
# Skip automatic CLI registration.
|
||||
no_cli: bool = False
|
||||
# When True, config resolution (model overrides and post-process passes)
|
||||
# may decide this field: the declaration stash accepts the name, and
|
||||
# `resolution_result` and the config bags answer with the decision. The
|
||||
# field keeps what the operator passed.
|
||||
# Allow resolution passes to declare a value for this field.
|
||||
resolvable: bool = False
|
||||
# What the field means when nobody said anything -- the bottom of the read
|
||||
# chain: override, decision, input, then this. Not the dataclass default,
|
||||
# which stays `None` because that is how the record spells "not typed".
|
||||
# `None` here means the field declares no fallback, which is the same
|
||||
# answer the read chain gives without one.
|
||||
#
|
||||
# Only a value fixed for the life of the configuration belongs here. One
|
||||
# that depends on the machine, on another field, or on anything impure is a
|
||||
# decision, and decisions stay in a hook where their order is visible.
|
||||
# Scalar fallback when neither resolution nor input supplies a value.
|
||||
# Machine- or config-dependent defaults belong in resolution hooks.
|
||||
fallback: Any = None
|
||||
|
||||
|
||||
@@ -112,44 +101,25 @@ class Derived(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
class NS(msgspec.Struct, frozen=True):
|
||||
"""Namespace-path marker for a ServerArgs field, attached alongside the
|
||||
field's metadata in ``Annotated``:
|
||||
"""Per-field namespace marker for records spanning multiple namespaces.
|
||||
|
||||
field: A[int, "help", NS("parallel")] = 1
|
||||
field: A[str, Arg(help="…"), NS("exec.moe")] = "auto"
|
||||
|
||||
``ServerArgs`` no longer uses it: its fields are declared in the
|
||||
``arg_groups/fields/`` classes, each of which carries the ``_NS_PATH`` it
|
||||
stands for, so the module a declaration lives in *is* its namespace. What
|
||||
is left for this marker is the case a class cannot express -- one ad-hoc
|
||||
dataclass whose fields span several namespaces, which is what the
|
||||
config-bag tests build."""
|
||||
Example: ``field: A[int, "help", NS("parallel")] = 1``.
|
||||
ServerArgs instead collects each namespace class's ``_NS_PATH``.
|
||||
"""
|
||||
|
||||
path: str
|
||||
|
||||
|
||||
@functools.cache
|
||||
def namespace_of(cls) -> dict:
|
||||
"""``{field_name: dotted namespace path}``, read from the declaring class.
|
||||
"""Return ``{field: namespace}`` from collected metadata, class paths, or NS markers.
|
||||
|
||||
A field's namespace is where it is declared: each class in
|
||||
``arg_groups/fields/`` carries the ``_NS_PATH`` it stands for, and
|
||||
``ServerArgs`` composes them. Walking the MRO therefore answers "which
|
||||
namespace owns this field" without a per-field marker -- the file the
|
||||
declaration sits in is the marker.
|
||||
|
||||
A class that is not built that way -- an ad-hoc dataclass spanning several
|
||||
namespaces, which is what the config-bag tests construct -- falls back to
|
||||
the per-field ``NS`` marker. A field with neither is absent from the map
|
||||
(the coverage lint flags them). Non-dataclass types yield an empty map.
|
||||
Unmarked fields are omitted; non-record types yield an empty map.
|
||||
"""
|
||||
if not is_record(cls):
|
||||
return {}
|
||||
# An assembled record: the collector recorded who declared each field,
|
||||
# because there are no base classes left to ask.
|
||||
out = dict(getattr(cls, "_NS_BY_FIELD", None) or {})
|
||||
# A class that still inherits its namespaces: nearest declaration wins, so
|
||||
# walk the MRO front to back and keep the first answer.
|
||||
# Nearest namespace declaration wins.
|
||||
for base in cls.__mro__:
|
||||
path = base.__dict__.get("_NS_PATH")
|
||||
if path is None:
|
||||
@@ -199,12 +169,7 @@ def resolvable_fields(cls) -> frozenset:
|
||||
|
||||
@functools.cache
|
||||
def fallbacks_of(cls) -> dict:
|
||||
"""``{field_name: value}`` for every field of ``cls`` that declares one.
|
||||
|
||||
Read the same way `resolvable_fields` reads its flag, so a fallback lives
|
||||
beside the help text of the field it belongs to rather than in whatever
|
||||
hook used to fill it in.
|
||||
"""
|
||||
"""Return declared scalar fallbacks for fields whose input default is None."""
|
||||
if not is_record(cls):
|
||||
return {}
|
||||
hints = get_type_hints(cls, include_extras=True)
|
||||
@@ -212,11 +177,7 @@ def fallbacks_of(cls) -> dict:
|
||||
for field in record_fields(cls):
|
||||
_, arg = _unwrap_annotated(hints.get(field.name, field.type))
|
||||
if arg is not None and arg.fallback is not None:
|
||||
# Two things `with_fallback` relies on and cannot check itself,
|
||||
# asserted where a new declaration passes through. This function is
|
||||
# cached, so a mutable fallback would hand one shared object to
|
||||
# every reader; and a field whose dataclass default is not `None`
|
||||
# can never reach the fallback, which makes the declaration dead.
|
||||
# Reject shared mutable fallbacks and defaults that make the fallback unreachable.
|
||||
assert not isinstance(arg.fallback, (list, dict, set)), (
|
||||
f"{cls.__name__}.{field.name}: a mutable fallback would be "
|
||||
"shared by every reader -- use a scalar"
|
||||
@@ -230,16 +191,10 @@ def fallbacks_of(cls) -> dict:
|
||||
|
||||
|
||||
def with_fallback(cls, name: str, value: Any) -> Any:
|
||||
"""``value``, or the declared fallback when nothing has answered.
|
||||
"""Apply a declared fallback when ``value`` is None.
|
||||
|
||||
`resolution_result` calls this as its last step -- the effective surface,
|
||||
which the config bags and `/server_info` read through. Deliberately not the
|
||||
views a pass reads *while deciding*: `model_overrides/inkling.py` branches
|
||||
on `if cfg.swa_full_tokens_ratio is None`, and a fallback answering there
|
||||
would make that branch dead. `test_declared_fallbacks.py` pins both halves.
|
||||
|
||||
A mutable fallback would need copying per read, for the reason a dataclass
|
||||
spells this `default_factory`. Every declared one is a scalar.
|
||||
Used by ``resolution_result``, but not mid-resolution views: handlers must
|
||||
still distinguish unset inputs. Fallbacks must be immutable scalars.
|
||||
"""
|
||||
if value is not None:
|
||||
return value
|
||||
@@ -247,12 +202,7 @@ def with_fallback(cls, name: str, value: Any) -> Any:
|
||||
|
||||
|
||||
def record_fields(cls):
|
||||
"""The declared fields of a record, Struct or dataclass.
|
||||
|
||||
`ServerArgs` and the namespace classes are `msgspec.Struct`; the config-bag
|
||||
tests build ad-hoc dataclasses spanning namespaces, and the helpers here are
|
||||
driven with both. Anything else yields nothing.
|
||||
"""
|
||||
"""Return Struct or dataclass fields, or an empty tuple for other types."""
|
||||
if isinstance(cls, type) and issubclass(cls, msgspec.Struct):
|
||||
return msgspec.structs.fields(cls)
|
||||
if dataclasses.is_dataclass(cls):
|
||||
@@ -326,12 +276,7 @@ def _infer_type_func(tp):
|
||||
|
||||
|
||||
def _field_default(field):
|
||||
"""Return the default value for a field, or `_MISSING`.
|
||||
|
||||
The two record shapes spell "no default" differently -- a Struct field says
|
||||
`msgspec.NODEFAULT`, a dataclass field `dataclasses.MISSING` -- so both are
|
||||
normalized here and every caller below tests against `_MISSING` alone.
|
||||
"""
|
||||
"""Return a field default, normalizing Struct and dataclass missing sentinels."""
|
||||
absent = (_MISSING, msgspec.NODEFAULT)
|
||||
if field.default not in absent:
|
||||
return field.default
|
||||
|
||||
@@ -1,10 +1,4 @@
|
||||
"""Enumerated choices shared by the config field declarations.
|
||||
|
||||
These lived in ``server_args.py`` beside the fields that name them. The fields
|
||||
moved to ``arg_groups/fields/``, and ``server_args`` imports the field modules,
|
||||
so the lists cannot stay there without a cycle. ``server_args`` re-exports them
|
||||
for the handful of modules that import them from their old home.
|
||||
"""
|
||||
"""Shared config choices, re-exported by server_args for plugin compatibility."""
|
||||
|
||||
LOAD_FORMAT_CHOICES = [
|
||||
"auto",
|
||||
|
||||
@@ -21,16 +21,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def apply_deepseek_v4_defaults(server_args: ServerArgs, model_arch: str) -> None:
|
||||
"""Residual imperative arm of the DeepSeek V4 defaults.
|
||||
|
||||
The attention/page/window/MoE-runner declarations moved to the override
|
||||
registry (arg_groups/overrides.py: _deepseek_v4_overrides) and the
|
||||
kv-cache dtype default to the resolution pipeline
|
||||
(_deepseek_v4_kv_cache_dtype, invoked below at its legacy slot). This
|
||||
keeps, at the legacy slot: the ROCm env fill (env-write policy), the
|
||||
max_running_requests fill (the speculative hook is a later writer of
|
||||
that field) and the validations.
|
||||
"""
|
||||
"""Apply DeepSeek V4 environment defaults, request limits, and validation."""
|
||||
cfg = resolving_view(server_args)
|
||||
|
||||
# FlashMLA sparse prefill (SGLANG_OPT_FLASHMLA_SPARSE_PREFILL, default on)
|
||||
|
||||
@@ -1,14 +1,6 @@
|
||||
"""The order ``ServerArgs`` presents its fields in, frozen.
|
||||
"""Field order preserving the ServerArgs positional constructor signature.
|
||||
|
||||
A dataclass turns field order into a positional constructor signature, so
|
||||
``ServerArgs(model_path, tokenizer_path)`` has to keep meaning what it means.
|
||||
Grouping the declarations by namespace would move the second argument onto
|
||||
another field, silently.
|
||||
|
||||
A compatibility record and nothing else -- a field's namespace is the module it
|
||||
is declared in, and only ``collect_input_fields`` reads this. A name that is not
|
||||
here sorts after every name that is, which is the only backward-compatible
|
||||
position for a new field anyway.
|
||||
``collect_input_fields`` appends unlisted fields after these entries.
|
||||
"""
|
||||
|
||||
# fmt: off
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
"""Config field declarations, one module per top-level namespace.
|
||||
"""Namespace declarations for input and derived configuration.
|
||||
|
||||
Each class carries the ``_NS_PATH`` it stands for, so the module a field is
|
||||
declared in *is* its namespace. ``ServerArgs`` is assembled from the classes
|
||||
that declare **operator input**; the derived fields, which nobody can type,
|
||||
are declared beside them but are not collected into the record.
|
||||
``ServerArgs`` collects input fields from classes carrying ``_NS_PATH``.
|
||||
Derived declarations stay on those classes for publication into config bags.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -20,30 +18,11 @@ from sglang.srt.arg_groups.field_order import POSITIONAL_FIELD_ORDER
|
||||
def collect_input_fields(
|
||||
sources: List[type],
|
||||
) -> Tuple[Dict[str, Any], Dict[str, Any], Dict[str, str]]:
|
||||
"""The annotations and defaults of every field these classes declare.
|
||||
"""Return resolved annotations, defaults, and a field-to-namespace map.
|
||||
|
||||
Each source's annotations are resolved **in its own module** and handed on
|
||||
as type objects. Strings never travel: an annotation carried across as text
|
||||
would be re-evaluated where it lands, and the composing module does not
|
||||
import the names the declarations use -- that is the whole point of them
|
||||
living where they do.
|
||||
|
||||
Returns the resolved annotations, the defaults, and ``{field: namespace}``
|
||||
-- the last because the assembled record has no base classes to read a
|
||||
``_NS_PATH`` off, and the namespace is still the class that declared it.
|
||||
|
||||
Which classes are passed here is the record's contract: the record holds
|
||||
what the operator asked for, so a class of derived fields is simply not in
|
||||
the list, and the rule is one readable call rather than an invariant spread
|
||||
across a base-class list.
|
||||
|
||||
The result is ordered by ``POSITIONAL_FIELD_ORDER``, not by namespace: a
|
||||
dataclass turns field order into a positional constructor signature, and
|
||||
grouping fields that used to be interleaved would silently move
|
||||
``ServerArgs(model_path, tokenizer_path)``'s second argument onto another
|
||||
field. Anything the record declares that the frozen order does not name
|
||||
goes after it, in declaration order -- the only backward-compatible place
|
||||
for a new field.
|
||||
Resolve annotations in their declaring module so imported type names remain
|
||||
available. Preserve ``POSITIONAL_FIELD_ORDER`` for constructor compatibility;
|
||||
append new fields in declaration order.
|
||||
"""
|
||||
annotations: Dict[str, Any] = {}
|
||||
defaults: Dict[str, Any] = {}
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``device`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``device`` bag, which is what ``get_device()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``device`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``disagg`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``disagg`` bag, which is what ``get_disagg()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``disagg`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``exec`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``exec`` bag, which is what ``get_exec()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``exec`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``lora`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``lora`` bag, which is what ``get_lora()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``lora`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``memory`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``memory`` bag, which is what ``get_memory()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``memory`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``mm`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``mm`` bag, which is what ``get_mm()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``mm`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``model`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``model`` bag, which is what ``get_model()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``model`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``observability`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``observability`` bag, which is what ``get_observability()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``observability`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``parallel`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``parallel`` bag, which is what ``get_parallel()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``parallel`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``schedule`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``schedule`` bag, which is what ``get_schedule()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``schedule`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``serving`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``serving`` bag, which is what ``get_serving()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``serving`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
"""Config fields of the ``spec`` namespace.
|
||||
|
||||
One class per namespace. The class *is* the namespace: a field declared here
|
||||
lands in the ``spec`` bag, which is what ``get_spec()`` returns, so a reader
|
||||
spells it exactly as before. ``ServerArgs`` composes these classes, so the
|
||||
record stays one flat object -- the split moves where declarations live, not
|
||||
how config is shaped at runtime.
|
||||
"""
|
||||
"""Config fields of the ``spec`` namespace."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
"""What the per-model override declarations are written against.
|
||||
"""Registry and read views shared by model override providers.
|
||||
|
||||
The declarations themselves live one directory down, in
|
||||
``arg_groups/model_overrides/``: one module per model family, mirroring the
|
||||
``models/`` naming. This module is what they all import -- the registry they
|
||||
register into, the read-only views they are handed, and the few accessors that
|
||||
answer questions about the model. It deliberately depends on nothing in
|
||||
``overrides.py``, so a family module never has to import its way back up.
|
||||
Model-family modules in ``model_overrides/`` import this module without
|
||||
creating a dependency back to ``overrides.py``.
|
||||
"""
|
||||
|
||||
import logging
|
||||
@@ -18,10 +14,9 @@ from sglang.srt.utils.common import is_mps, is_no_spec_infer_or_topk_one
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Constant per-architecture overrides (populated by the migration sweeps).
|
||||
# Constant per-architecture overrides.
|
||||
MODEL_OVERRIDES: Dict[str, Dict[str, Any]] = {
|
||||
# These models run in bfloat16 regardless of the requested dtype
|
||||
# (faithful port of the legacy unconditional arch branch).
|
||||
# These models require bfloat16 regardless of the requested dtype.
|
||||
"MistralLarge3ForCausalLM": {"dtype": "bfloat16"},
|
||||
"PixtralForConditionalGeneration": {"dtype": "bfloat16"},
|
||||
}
|
||||
@@ -31,9 +26,7 @@ MODEL_OVERRIDES: Dict[str, Dict[str, Any]] = {
|
||||
_MODEL_OVERRIDE_FNS: Dict[str, List[Callable[..., dict]]] = {}
|
||||
|
||||
|
||||
# Predicate-keyed providers, in registration order — for legacy branches
|
||||
# matched by substring/predicate on the architecture string rather than an
|
||||
# exact name (e.g. '"Step3p5ForCausalLM" in model_arch').
|
||||
# Predicate-keyed providers, in registration order.
|
||||
_PREDICATE_OVERRIDE_FNS: List[Tuple[Callable[[str], bool], Callable[..., dict]]] = []
|
||||
|
||||
|
||||
@@ -78,12 +71,7 @@ def _invoke_provider(
|
||||
|
||||
|
||||
class ResolvedView:
|
||||
"""Read-only view of the resolving configuration handed to post-process
|
||||
passes: the accumulated declarations overlaid on the pristine
|
||||
``server_args`` (residual imperative writes of non-resolved fields show
|
||||
through the fallthrough) — exactly the state the legacy handler at the
|
||||
same slot observed. Writes are rejected: passes return declarations.
|
||||
"""
|
||||
"""Read-only snapshot of declarations over raw inputs, used by post-process passes."""
|
||||
|
||||
__slots__ = ("_server_args", "_overlay")
|
||||
|
||||
@@ -104,14 +92,9 @@ class ResolvedView:
|
||||
|
||||
|
||||
class ResolvingConfig:
|
||||
"""Live read view of the resolution result: the declaration stash over the
|
||||
record's fields, looked up per read.
|
||||
"""Live view of declarations over raw inputs.
|
||||
|
||||
``ResolvedView`` snapshots the overlay when it is built, which is what a
|
||||
post-process pass wants -- it reads the state at its slot. A resolver that
|
||||
reads *after* declaring, or after calling something that declares, needs the
|
||||
current answer instead, so this one walks the stash on every read. It falls
|
||||
through to the field, which is where the raw input lives.
|
||||
Unlike ``ResolvedView``, each read observes declarations made since construction.
|
||||
"""
|
||||
|
||||
__slots__ = ("_server_args",)
|
||||
@@ -140,10 +123,7 @@ def resolving_view(server_args: Any) -> ResolvingConfig:
|
||||
|
||||
|
||||
def _declaration_overlay(server_args: Any) -> Dict[str, Any]:
|
||||
"""What the declarations say so far, last writer wins.
|
||||
|
||||
Nothing writes the fields, so a mid-resolution reader needs this to see a
|
||||
decision at all; the fields keep what the caller supplied."""
|
||||
"""Merge declarations in order, with the last writer winning."""
|
||||
overlay: Dict[str, Any] = {}
|
||||
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
|
||||
overlay.update(declared)
|
||||
@@ -151,13 +131,7 @@ def _declaration_overlay(server_args: Any) -> Dict[str, Any]:
|
||||
|
||||
|
||||
def resolved_view(server_args: Any) -> ResolvedView:
|
||||
"""Read-only view of the resolving configuration: the declarations
|
||||
overlaid on the fields, snapshotted per call.
|
||||
|
||||
For mid-resolution code that is not a pass (``__post_init__`` handlers and
|
||||
hooks) that must answer with what resolution decided -- a declaration-only resolver (a model-specific
|
||||
override, a registry entry) never writes the field, so a field read there
|
||||
answers with the raw input."""
|
||||
"""Snapshot current declarations over the record fields in a read-only view."""
|
||||
return ResolvedView(server_args, overlay=_declaration_overlay(server_args))
|
||||
|
||||
|
||||
@@ -200,12 +174,7 @@ def record_of(view: Any) -> Any:
|
||||
|
||||
|
||||
def is_attention_backend_not_set(cfg: Any):
|
||||
"""None of the three attention backends has been decided yet.
|
||||
|
||||
Takes the view rather than the record: every read is a view read, and the
|
||||
callers that hold a view (the override providers) would otherwise have to
|
||||
reach back through it for a record.
|
||||
"""
|
||||
"""Return whether all three attention backends are unset in the resolving view."""
|
||||
return (
|
||||
cfg.attention_backend is None
|
||||
and cfg.prefill_attention_backend is None
|
||||
@@ -221,12 +190,7 @@ def use_mla_backend(server_args: Any):
|
||||
|
||||
|
||||
def model_config_of(server_args: Any):
|
||||
"""The model configuration this record describes, built once and memoised.
|
||||
|
||||
Takes a view as readily as the record: a view is a read overlay of one
|
||||
record, the memo has to live on that record either way, and the callers
|
||||
that hold a view would otherwise all have to unwrap it themselves.
|
||||
"""
|
||||
"""Build and memoize ModelConfig on the underlying record, accepting records or views."""
|
||||
if isinstance(server_args, (ResolvedView, ResolvingConfig)):
|
||||
server_args = record_of(server_args)
|
||||
# Lazy init to avoid circular import
|
||||
@@ -258,13 +222,7 @@ def model_config_of(server_args: Any):
|
||||
|
||||
|
||||
def ep_joiner_of(cfg: Any) -> bool:
|
||||
"""Whether this process was launched as an elastic-EP joiner.
|
||||
|
||||
The one definition. After publish the answer is a bag leaf --
|
||||
`get_exec().moe.is_ep_joiner` -- computed from this function by the
|
||||
declaration in `arg_groups/fields/exec_.py`; resolution needs it before
|
||||
there is a bag to read, which is why it is still a function.
|
||||
"""
|
||||
"""Return whether this process was launched as an elastic-EP joiner."""
|
||||
return cfg.ep_join_mode in ("scale", "recover")
|
||||
|
||||
|
||||
@@ -279,13 +237,7 @@ def startup_weight_load_overlap_of(cfg: Any) -> bool:
|
||||
|
||||
|
||||
def mamba_extra_buffer_of(cfg: Any) -> bool:
|
||||
"""The predicate, read off a config-shaped object mid-resolution.
|
||||
|
||||
This is the one definition. After publish the answer is a bag leaf --
|
||||
``get_exec().mamba.enable_mamba_extra_buffer`` -- computed from this same
|
||||
function by the declaration in ``arg_groups/fields/exec_.py``. Resolution
|
||||
needs it before there is a bag to read, which is why it is still a
|
||||
function."""
|
||||
"""Return whether the resolved Mamba cache strategy requires an extra buffer."""
|
||||
return cfg.disable_radix_cache is False and cfg.mamba_radix_cache_strategy in (
|
||||
"extra_buffer",
|
||||
"extra_buffer_lazy",
|
||||
|
||||
@@ -1,8 +1,4 @@
|
||||
"""Config-time override declarations for bailing_moe_v3.
|
||||
|
||||
Architectures: BailingMoeV3ForCausalLM,
|
||||
BailingMoeV3VLForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for bailing_moe_v3."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for cohere2_moe.
|
||||
|
||||
Architectures: Cohere2MoeForCausalLM, Cohere2VisionForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for cohere2_moe."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for deepseek_v2.
|
||||
|
||||
Architectures: DeepseekV32ForCausalLM, DeepseekV3ForCausalLM, Dots3NoteForCausalLM, Glm5NextForConditionalGeneration, GlmMoeDsaForCausalLM, HYV4ForCausalLM, HYV4ForCausalLMNextN, KimiK25ForConditionalGeneration, LongcatFlashForCausalLM, LongcatFlashForCausalLMNextN, MistralLarge3ForCausalLM, PixtralForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for deepseek_v2."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
@@ -32,11 +29,7 @@ logger = logging.getLogger(__name__)
|
||||
"Dots3NoteForCausalLM",
|
||||
)
|
||||
def _deepseek_family_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
"""Order-safe declarations of the DeepSeek/DSA branch. The CP parallel
|
||||
writes (enable_dp_attention/ep_size/moe_a2a_backend have post-monolith
|
||||
writers), the kv-cache/split-backend defaults, the quant/moe block (read
|
||||
before it by _set_default_dsa_kv_cache_dtype) and the env writes stay in
|
||||
the branch."""
|
||||
"""Declare DeepSeek/DSA defaults; ordered CP, KV-cache, and MoE passes run in model_hook."""
|
||||
cfg = resolving_view(server_args)
|
||||
from sglang.srt.configs.model_config import (
|
||||
is_deepseek_dsa,
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for deepseek_v4.
|
||||
|
||||
Architectures: DeepseekV4ForCausalLM.
|
||||
"""
|
||||
"""Config-time override declarations for deepseek_v4."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for exaone.
|
||||
|
||||
Architectures: Exaone4ForCausalLM, ExaoneMoEForCausalLM.
|
||||
"""
|
||||
"""Config-time override declarations for exaone."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for falcon_h1.
|
||||
|
||||
Architectures: FalconH1ForCausalLM, JetNemotronForCausalLM, JetVLMForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for falcon_h1."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for gemma2_gemma3.
|
||||
|
||||
Architectures: Gemma2ForCausalLM, Gemma3ForCausalLM, Gemma3ForConditionalGeneration, Gemma3nForCausalLM, Gemma3nForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for gemma2_gemma3."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for gemma4.
|
||||
|
||||
Architectures: Gemma4ForCausalLM, Gemma4ForConditionalGeneration, Gemma4UnifiedForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for gemma4."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for gigachat35.
|
||||
|
||||
Architectures: GigaChat35ForCausalLM, GigaChat35ForCausalLMNextN.
|
||||
"""
|
||||
"""Config-time override declarations for gigachat35."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for glm4_moe.
|
||||
|
||||
Architectures: Glm4MoeForCausalLM.
|
||||
"""
|
||||
"""Config-time override declarations for glm4_moe."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for gpt_oss.
|
||||
|
||||
Architectures: GptOssForCausalLM.
|
||||
"""
|
||||
"""Config-time override declarations for gpt_oss."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
@@ -46,9 +43,7 @@ def _gpt_oss_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
# rather than landing on torch_native (no sliding window, no sinks).
|
||||
overrides["attention_backend"] = "triton"
|
||||
if get_platform().is_xpu:
|
||||
# Check for bf16 dtype on Intel XPU. Reads the pristine dtype request,
|
||||
# which equals the legacy mid-branch read: dtype had no earlier writer
|
||||
# for this arch.
|
||||
# Intel XPU requires bfloat16.
|
||||
if cfg.dtype == "auto":
|
||||
logger.warning(
|
||||
"GptOssForCausalLM on Intel XPU currently supports bfloat16 dtype only"
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for granitemoehybrid.
|
||||
|
||||
Architectures: GraniteMoeHybridForCausalLM.
|
||||
"""
|
||||
"""Config-time override declarations for granitemoehybrid."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for inkling.
|
||||
|
||||
Architectures: InklingForConditionalGeneration, InklingForConditionalGenerationMTP.
|
||||
"""
|
||||
"""Config-time override declarations for inkling."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for interns2_mobius.
|
||||
|
||||
Architectures: InternS2MobiusForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for interns2_mobius."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for kimi_k3.
|
||||
|
||||
Architectures: KimiK3ForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for kimi_k3."""
|
||||
|
||||
import inspect
|
||||
import logging
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for lfm2.
|
||||
|
||||
Architectures: Lfm2ForCausalLM, Lfm2MoeForCausalLM.
|
||||
"""
|
||||
"""Config-time override declarations for lfm2."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for llama4.
|
||||
|
||||
Architectures: Llama4ForCausalLM, Llama4ForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for llama4."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for mimo_v2.
|
||||
|
||||
Architectures: MiMoV2FlashForCausalLM, MiMoV2ForCausalLM.
|
||||
"""
|
||||
"""Config-time override declarations for mimo_v2."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for minicpm.
|
||||
|
||||
Architectures: MiniCPMForCausalLM, MiniCPMSALAForCausalLM.
|
||||
"""
|
||||
"""Config-time override declarations for minicpm."""
|
||||
|
||||
from typing import Any, Dict
|
||||
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for minicpmv.
|
||||
|
||||
Architectures: MiniCPMV4_6ForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for minicpmv."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for minimax_m2.
|
||||
|
||||
Architectures: MiniMaxM2ForCausalLM.
|
||||
"""
|
||||
"""Config-time override declarations for minimax_m2."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for minimax_m3.
|
||||
|
||||
Architectures: MiniMaxM3SparseForCausalLM, MiniMaxM3SparseForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for minimax_m3."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for moss_vl.
|
||||
|
||||
Architectures: MossVLForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for moss_vl."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for muse_glimmer.
|
||||
|
||||
Architectures: MuseGlimmerForCausalLM, MuseGlimmerForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for muse_glimmer."""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
@@ -1,8 +1,4 @@
|
||||
"""Config-time override declarations for nemotron_h.
|
||||
|
||||
Architectures: NemotronHForCausalLM, NemotronHPuzzleForCausalLM,
|
||||
NemotronH_Omni_Reasoning_V3.
|
||||
"""
|
||||
"""Config-time override declarations for nemotron_h."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for olmo2.
|
||||
|
||||
Architectures: Olmo2ForCausalLM.
|
||||
"""
|
||||
"""Config-time override declarations for olmo2."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for qwen3_5.
|
||||
|
||||
Architectures: InternS2MobiusForConditionalGeneration, InternS2PreviewForConditionalGeneration, Qwen3NextForCausalLM, Qwen3_5ForConditionalGeneration, Qwen3_5MoeForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for qwen3_5."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
@@ -39,9 +36,7 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
use_mla_backend=use_mla_backend(server_args),
|
||||
model_config=model_config_of(server_args),
|
||||
)
|
||||
# The mamba radix-cache pass runs before this dispatch: read the
|
||||
# declared strategy through the view (the legacy branch observed the
|
||||
# already-written field here).
|
||||
# The Mamba radix-cache pass has already resolved the strategy.
|
||||
if default_attn_backend == "trtllm_mha" and not (
|
||||
not mamba_extra_buffer_of(resolved_view(server_args))
|
||||
and not cfg.disable_radix_cache
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for qwen3_moe.
|
||||
|
||||
Architectures: InternS2PreviewForConditionalGeneration, Qwen3MoeForCausalLM, Qwen3NextForCausalLM, Qwen3VLMoeForConditionalGeneration, Qwen3_5ForConditionalGeneration, Qwen3_5MoeForConditionalGeneration, Qwen4ExpForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for qwen3_moe."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for qwen3_vl.
|
||||
|
||||
Architectures: Qwen3VLForConditionalGeneration, Qwen3VLMoeForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for qwen3_vl."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Optional
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
"""Config-time override declarations for qwen4_exp.
|
||||
|
||||
Architectures: Qwen4ExpForConditionalGeneration.
|
||||
"""
|
||||
"""Config-time override declarations for qwen4_exp."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict
|
||||
|
||||
@@ -11,23 +11,12 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Declarative model-override registry.
|
||||
"""Configuration declarations and model override dispatch.
|
||||
|
||||
Model-identity adjustments to the server configuration are DECLARED here and
|
||||
appended to the record's declaration stash (gate order, last writer wins).
|
||||
Nothing here writes back onto ``ServerArgs``: the record holds the user's raw
|
||||
input, and a decision is read through ``resolution_result`` or the published
|
||||
config bags — model code never mutates ``ServerArgs`` fields imperatively. That
|
||||
holds without exception: a resolver this tree does not own assigns onto a
|
||||
stand-in (``record_foreign_defaults``), and what it set is declared.
|
||||
|
||||
Two declaration forms, keyed on ``hf_config.architectures[0]``:
|
||||
|
||||
- ``MODEL_OVERRIDES``: pure-constant cases — ``arch -> {field: value}``.
|
||||
- ``@register_model_override(arch)``: derived cases — a callable
|
||||
``fn(server_args, hf_config) -> dict`` that faithfully carries today's
|
||||
conditional logic. ``server_args`` is pristine and must be treated
|
||||
read-only: the callable returns declarations, it never writes.
|
||||
Declarations preserve raw ServerArgs inputs and are applied in order, last
|
||||
writer wins. Read them through ``resolution_result`` or published config bags.
|
||||
Model providers use ``MODEL_OVERRIDES`` for constants or
|
||||
``register_model_override`` for functions returning field-value dictionaries.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -45,8 +34,7 @@ from sglang.srt.arg_groups.arg_utils import (
|
||||
with_fallback,
|
||||
)
|
||||
|
||||
# Re-exported for the callers that already import these names from here; the
|
||||
# declarations under ``model_overrides/`` import them from the base directly.
|
||||
# Compatibility re-exports; model providers import model_override_base directly.
|
||||
from sglang.srt.arg_groups.model_override_base import ( # noqa: F401
|
||||
_MODEL_OVERRIDE_FNS,
|
||||
_PREDICATE_OVERRIDE_FNS,
|
||||
@@ -100,25 +88,10 @@ def register_post_process(fn: Callable[..., dict]) -> Callable[..., dict]:
|
||||
|
||||
|
||||
def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
"""Invoke one pass at its legacy handler slot.
|
||||
"""Run a pass against the current resolution snapshot and record its decisions.
|
||||
|
||||
Evaluates the pass on the resolving state (a read-only view with the
|
||||
accumulated declarations overlaid from the stash) and appends its
|
||||
declaration to the stash, which is what the config bags are projected from.
|
||||
The fields stay untouched.
|
||||
|
||||
A slot that runs after resolution -- ``check_server_args`` hosts one -- lands
|
||||
in the same stash, which publish projects from later, so it needs no field
|
||||
write either. After *publish* there is no such later projection: the stash
|
||||
would grow an entry nothing reads.
|
||||
|
||||
So what is refused is the *declaration*, not the record. A pass that returns
|
||||
an empty dict is a validation, and it may run on the published instance --
|
||||
it has to, because ``Engine(server_args=sa)`` after ``Engine.shutdown()``
|
||||
re-runs ``check_server_args`` on the very instance the context still holds.
|
||||
A pass that returns a non-empty dict there is refused by the guard in
|
||||
``declare_resolution``, as a late declaration is -- post-publish changes go
|
||||
to the bags through ``get_context().override(...)``.
|
||||
After publication, only validation passes returning an empty dict are allowed;
|
||||
Engine restart may re-run those validations on the same record.
|
||||
"""
|
||||
|
||||
declared = fn(ResolvedView(server_args, overlay=_declaration_overlay(server_args)))
|
||||
@@ -133,23 +106,10 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
|
||||
|
||||
def declare_resolution(server_args: Any, source: str, **fields: Any) -> None:
|
||||
"""Record a resolution write in the declaration stash.
|
||||
"""Append field decisions without changing raw inputs.
|
||||
|
||||
The stash *is* the resolution result: the bags are projected from it,
|
||||
`resolution_result` answers from it, and no field is written. A resolver
|
||||
reading a field another resolver may have decided must read `resolving_view`
|
||||
(or `resolved_view(server_args)`).
|
||||
|
||||
Every declaration goes through here, whenever it is made: inside
|
||||
``__post_init__``, at launcher stage (LoRA normalization, the auto-detected
|
||||
parsers -- they decide what the process will run with, so they belong to the
|
||||
pipeline even though they run after it), and on a copy about to cross a
|
||||
process boundary. A name that is not a field is rejected here rather than
|
||||
becoming an attribute nothing reads.
|
||||
|
||||
Refuses the published config. The stash is projected at publish and never
|
||||
again, so a declaration afterwards is a silent no-op; post-publish changes
|
||||
go to the bags through ``get_context().override(...)``.
|
||||
Reject unknown fields and declarations on the published record. After
|
||||
publication, use ``RuntimeContext.override`` to update config bags.
|
||||
"""
|
||||
if is_record(server_args):
|
||||
unknown = sorted(set(fields) - field_names(type(server_args)))
|
||||
@@ -174,14 +134,7 @@ def declare_resolution(server_args: Any, source: str, **fields: Any) -> None:
|
||||
|
||||
|
||||
class _ForeignDefaults:
|
||||
"""The stand-in handed to a resolver this tree does not own.
|
||||
|
||||
Reads fall through to the resolving view, so a plugin sees what resolution
|
||||
has decided so far rather than the raw input -- better than what it used to
|
||||
get, which was the record's own fields. Writes are captured here and
|
||||
declared by the caller, so the record is never written and the write seal
|
||||
has no exception.
|
||||
"""
|
||||
"""Capture plugin writes while reading through the current resolving view."""
|
||||
|
||||
__slots__ = ("_cfg", "_written")
|
||||
|
||||
@@ -202,24 +155,9 @@ class _ForeignDefaults:
|
||||
def record_foreign_defaults(
|
||||
server_args: Any, source: str, resolve: Callable[[Any], Any]
|
||||
) -> Any:
|
||||
"""Run a resolver this tree does not own, and declare what it set.
|
||||
"""Run a plugin resolver, declare its field writes, and return its result.
|
||||
|
||||
Out-of-tree platform plugins and registered speculative algorithms are
|
||||
handed a configuration and assign fields on it. That interface is not ours
|
||||
to change, so the assignment stays the contract -- it just lands on a
|
||||
stand-in instead of the record, and what it set is declared like any other
|
||||
decision. Nothing writes the record, which is why there is no longer a
|
||||
named hole in the seal.
|
||||
|
||||
Returns whatever the resolver returned, so a provider with a return value
|
||||
goes through the same capture.
|
||||
|
||||
Non-field names are dropped: a plugin scribbling on an attribute that is
|
||||
not configuration is not a decision, and it was invisible to the previous
|
||||
diff for the same reason.
|
||||
|
||||
A stand-in record (tests drive the hooks with a plain namespace) has no
|
||||
view to read, so the resolver runs against it directly and uncaptured.
|
||||
Ignore non-field writes. Non-record test objects are passed through directly.
|
||||
"""
|
||||
if not is_record(server_args):
|
||||
return resolve(server_args)
|
||||
@@ -236,20 +174,9 @@ def record_foreign_defaults(
|
||||
|
||||
|
||||
def resolution_result(server_args: Any, field: str, default: Any = None) -> Any:
|
||||
"""What resolution decided for ``field``: the declaration if there is one,
|
||||
otherwise what the caller supplied, otherwise the field's declared
|
||||
fallback.
|
||||
"""Read the last declaration, then raw input, then the declared fallback.
|
||||
|
||||
The fallback is last because it is what the field means when nobody said
|
||||
anything -- an operator who types a value and a pass that decides one both
|
||||
sit above it. It is read from the declaration rather than filled in by a
|
||||
pass, so there is no slot to place and no second call to make idempotent.
|
||||
|
||||
This is what the config projection reads. Reading the field instead would
|
||||
work whatever the caller passed onto the record -- and
|
||||
the point of declaring is that they will not, so the projection must not
|
||||
depend on it. A config that never ran the pipeline (a mock, a partial
|
||||
fixture) carries no raw snapshot; its fields are all it has.
|
||||
Records without a raw-input snapshot fall back to their current fields.
|
||||
"""
|
||||
for _source, declared in reversed(
|
||||
getattr(server_args, "_resolved_overrides", None) or ()
|
||||
@@ -263,12 +190,7 @@ def resolution_result(server_args: Any, field: str, default: Any = None) -> Any:
|
||||
|
||||
|
||||
def pre_capture_activation_reserve_mb_of(cfg: Any, gpu_mem: Optional[float]) -> float:
|
||||
"""The activation working-set reserve held back before cuda-graph capture.
|
||||
|
||||
The config-shaped half of the pair; `runtime_context` carries the
|
||||
published-bag half, and `TestDerivedPredicatesAgreeAcrossTiers` pins the
|
||||
two equal.
|
||||
"""
|
||||
"""Return the activation reserve in MB before CUDA graph capture."""
|
||||
if cfg.disaggregation_mode == "decode":
|
||||
running_requests = (
|
||||
cfg.max_running_requests or cfg.cuda_graph_config.decode.max_bs or 1
|
||||
@@ -378,13 +300,6 @@ def collect_model_override_declarations(
|
||||
return declarations
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Derived per-family declarations (faithful ports of legacy arch branches).
|
||||
# Callables read the PRISTINE server_args, never write; logging is kept
|
||||
# verbatim from the legacy branch for operator-visible fidelity.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Importing the package is what registers the per-model declarations.
|
||||
import sglang.srt.arg_groups.model_overrides # noqa: F401
|
||||
|
||||
@@ -426,9 +341,7 @@ def _step3p_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Architectures whose monolith branch routes through the mamba radix cache
|
||||
# handling (hybrid linear-attention models). Keep in sync with the branch
|
||||
# guards in _handle_model_specific_adjustments.
|
||||
# Keep this hybrid-model set in sync with model_hook.handle_model_specific_adjustments.
|
||||
_MAMBA_RADIX_CACHE_ARCHS = frozenset(
|
||||
{
|
||||
"KimiLinearForCausalLM",
|
||||
@@ -508,15 +421,7 @@ def supports_mamba_cache_extra_buffer(view: Any, model_arch: str) -> bool:
|
||||
|
||||
@register_post_process
|
||||
def _mamba_radix_cache_resolution(view: Any) -> dict:
|
||||
"""Resolve the hybrid-mamba radix cache fields (pure).
|
||||
|
||||
Slot pass: invoked at each legacy ``_handle_mamba_radix_cache`` slot —
|
||||
the hybrid-spec call at the head of the monolith and the per-arch branch
|
||||
calls — where it reads the mid-resolution ``page_size`` /
|
||||
``disable_overlap_schedule`` exactly as the legacy helper did. The arch
|
||||
guard replicates the union of the legacy call-site guards so the pass is
|
||||
self-sufficient in the end-state pass list.
|
||||
"""
|
||||
"""Resolve hybrid-Mamba cache settings using the current page size and overlap policy."""
|
||||
from sglang.srt.configs.linear_attn_model_registry import (
|
||||
get_linear_attn_spec_by_arch,
|
||||
)
|
||||
@@ -553,10 +458,7 @@ def _mamba_radix_cache_resolution(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _dsa_kv_cache_dtype_default(view: Any) -> dict:
|
||||
"""Slot pass in the DSA arm, ordered before the split-backend
|
||||
resolution: default the kv-cache dtype from the device capability
|
||||
(Blackwell FP8, Hopper bf16) and normalize the bf16 alias. Reads the
|
||||
PRISTINE dsa split backends (their resolution runs after this pass)."""
|
||||
"""Default DSA KV-cache dtype before resolving split attention backends."""
|
||||
from sglang.srt.configs.model_config import is_deepseek_dsa
|
||||
|
||||
hf_config = model_config_of(view).hf_config
|
||||
@@ -658,9 +560,10 @@ def _check_tilelang_dsa_fp8_kv(
|
||||
|
||||
@register_post_process
|
||||
def _dsa_split_backend_resolution(view: Any) -> dict:
|
||||
"""Slot pass in the DSA arm: default the DSA prefill/decode split
|
||||
backends from the mid-resolution kv-cache dtype and the device
|
||||
capability. The hisparse arm takes precedence under --enable-hisparse."""
|
||||
"""Resolve DSA split backends from KV-cache dtype and device capability.
|
||||
|
||||
HiSparse settings take precedence when enabled.
|
||||
"""
|
||||
from sglang.srt.configs.model_config import is_deepseek_dsa
|
||||
|
||||
hf_config = model_config_of(view).hf_config
|
||||
@@ -811,10 +714,7 @@ _DEEPSEEK_FAMILY_ARCHS = frozenset(
|
||||
|
||||
@register_post_process
|
||||
def _deepseek_moe_quant_resolution(view: Any) -> dict:
|
||||
"""Slot pass invoked from inside the DeepSeek arch branch ("Set moe
|
||||
backend for DeepSeek"), NOT a dispatch-time declaration: the DSA
|
||||
kv-cache-dtype default earlier in the branch must read the PRISTINE
|
||||
quantization, so this resolution has to stay at its legacy slot."""
|
||||
"""Resolve DeepSeek MoE quantization after DSA has read the input quantization."""
|
||||
hf_config = model_config_of(view).hf_config
|
||||
model_arch = hf_config.architectures[0]
|
||||
if model_arch not in _DEEPSEEK_FAMILY_ARCHS:
|
||||
@@ -897,10 +797,7 @@ def _deepseek_moe_quant_resolution(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _deepseek_spec_moe_resolution(view: Any) -> dict:
|
||||
"""Slot pass at the DeepSeek branch's HIP arm: draft (nextn) spec-MoE
|
||||
backends for the DeepSeek fp4 checkpoint. Reads the mid-resolution
|
||||
quantization (after _deepseek_moe_quant_resolution) and the pre-a2a
|
||||
ep_size, exactly like the legacy in-branch writes."""
|
||||
"""Resolve HIP draft MoE backends after quantization and before A2A adjusts EP size."""
|
||||
|
||||
hf_config = model_config_of(view).hf_config
|
||||
model_arch = hf_config.architectures[0]
|
||||
@@ -1008,12 +905,10 @@ _FLASHINFER_ALLREDUCE_FUSION_ARCHS = frozenset(
|
||||
|
||||
@register_post_process
|
||||
def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
|
||||
"""Slot pass at the monolith tail: auto-enable FlashInfer AllReduce
|
||||
Fusion on SM90/SM100 for models with explicit support. auto resolves to
|
||||
mnnvl on Blackwell (single- and multi-node) and trtllm on SM90
|
||||
single-node systems. Reads the mid-resolution enable_dp_attention /
|
||||
moe_a2a_backend (after the DeepSeek CP and a2a declarations), exactly
|
||||
like the legacy tail block."""
|
||||
"""Enable supported FlashInfer AllReduce fusion after CP and A2A resolution.
|
||||
|
||||
Auto selects MNNVL on Blackwell and TRTLLM on single-node SM90.
|
||||
"""
|
||||
hf_config = model_config_of(view).hf_config
|
||||
model_arch = hf_config.architectures[0]
|
||||
# V4.1 TP4 uses the custom push plane for decode and fused MoE finalize.
|
||||
@@ -1091,8 +986,7 @@ def _deterministic_sampling_backend(view: Any) -> dict:
|
||||
|
||||
|
||||
def _deterministic_is_deepseek_model(view: Any) -> bool:
|
||||
"""Faithful copy of the deterministic handler's arch probe (pure read;
|
||||
the handler keeps its own copy for the later deepseek validation)."""
|
||||
"""Check the model architecture for deterministic DeepSeek backend selection."""
|
||||
from sglang.srt.connector import ConnectorType
|
||||
from sglang.srt.utils.common import parse_connector_type
|
||||
|
||||
@@ -1179,10 +1073,7 @@ def _attention_backend_default(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _mla_backend_page_constraints(view: Any) -> dict:
|
||||
"""Page-size constraints of the MLA/TRTLLM backend family (the raises and
|
||||
the cutedsl prefill fallback stay in the handler; only the page snaps are
|
||||
declared). The snaps chain on a local value exactly as the legacy blocks
|
||||
chained on self.page_size."""
|
||||
"""Resolve MLA/TRTLLM page-size constraints, chaining adjustments on the local value."""
|
||||
page_size = view.page_size
|
||||
if (
|
||||
view.attention_backend == "flashmla"
|
||||
@@ -1251,10 +1142,7 @@ def _mla_backend_page_constraints(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _mla_kv_cache_dtype_checks(view: Any) -> dict:
|
||||
"""Read-only validation pass in the attention-backend compatibility
|
||||
handler: the TRT-LLM and tokenspeed MLA backends constrain the resolved
|
||||
kv-cache dtype (declarations never reach the field, so the checks read
|
||||
the view)."""
|
||||
"""Validate resolved KV-cache dtype for TRTLLM and tokenspeed MLA backends."""
|
||||
if (
|
||||
view.attention_backend == "trtllm_mla"
|
||||
or view.decode_attention_backend == "trtllm_mla"
|
||||
@@ -1296,10 +1184,7 @@ def _hisparse_validation(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _cutedsl_prefill_backend_fill(view: Any) -> dict:
|
||||
"""Slot pass in the attention-backend compatibility handler: CuteDSL MLA
|
||||
is decode-only, so validate the combination and default the prefill side
|
||||
to trtllm_mla. The trtllm_mha check that follows at the legacy slot reads
|
||||
the resolved value through the view."""
|
||||
"""Validate decode-only CuteDSL MLA and default its prefill backend to trtllm_mla."""
|
||||
if not (
|
||||
view.attention_backend == "cutedsl_mla"
|
||||
or view.decode_attention_backend == "cutedsl_mla"
|
||||
@@ -1484,9 +1369,7 @@ def _tp_lm_head_all_to_all_default(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _dp_lm_head_validation(view: Any) -> dict:
|
||||
"""Read-only validation pass: dp-attention is a prerequisite for the
|
||||
dp LM head and the TP LM-head all-to-all path. Reads the mid-resolution
|
||||
values through the view."""
|
||||
"""Require DP attention for DP LM head and TP LM-head all-to-all."""
|
||||
if view.enable_dp_lm_head:
|
||||
assert view.enable_dp_attention, (
|
||||
"Please enable dp attention when setting enable_dp_lm_head. "
|
||||
@@ -1512,10 +1395,7 @@ def _dp_lm_head_validation(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
||||
"""The quantization-driven moe_runner_backend resolutions at the head of
|
||||
_handle_moe_kernel_config. The backend-compatibility asserts and the
|
||||
disable_shared_experts_fusion writes (post-publish writers exist for that
|
||||
field) stay in the handler."""
|
||||
"""Resolve MoE runner backends from quantization."""
|
||||
moe_runner_backend = view.moe_runner_backend
|
||||
if view.quantization == "nvfp4_online":
|
||||
if not get_platform().is_sm100:
|
||||
@@ -1580,10 +1460,10 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _moe_runner_fusion_disable(view: Any) -> dict:
|
||||
"""FlashInfer CuteDSL / TRT-LLM / TRT-LLM-routed MoE runners require the
|
||||
shared-experts fusion disabled; declared at the legacy write slots in
|
||||
_handle_moe_kernel_config (before the deprecated cutlass env override, so
|
||||
the runner value observed is the pre-override one)."""
|
||||
"""Disable shared-expert fusion for incompatible MoE runners.
|
||||
|
||||
Runs before the deprecated cutlass environment override.
|
||||
"""
|
||||
runner = view.moe_runner_backend
|
||||
if runner == "flashinfer_cutedsl":
|
||||
logger.warning(
|
||||
@@ -1605,10 +1485,7 @@ def _moe_runner_fusion_disable(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _a2a_fusion_adjustments(view: Any) -> dict:
|
||||
"""A2A-backend-driven shared-experts fusion adjustments, declared at the
|
||||
legacy write slots in _handle_a2a_moe: Waterfill requires the
|
||||
fusion enabled; FlashInfer, FlashInfer MegaMOE, and DeepEP v2 A2A require it disabled.
|
||||
"""
|
||||
"""Set shared-expert fusion requirements for the selected A2A backend."""
|
||||
if view.moe_a2a_backend in ("deepep", "megamoe") and view.enable_waterfill:
|
||||
if view.disable_shared_experts_fusion:
|
||||
logger.warning(
|
||||
@@ -1769,8 +1646,7 @@ def _dllm_page_size(view: Any) -> dict:
|
||||
)
|
||||
return {"page_size": config.block_size}
|
||||
if view.page_size > config.block_size:
|
||||
# Legacy scheduler-init fallback, folded into the pass: the page
|
||||
# size must not exceed the dllm block size.
|
||||
# The page size must not exceed the DLLM block size.
|
||||
logger.warning(
|
||||
"WARNING: "
|
||||
f"The page size {view.page_size} should not be larger than dllm block size {config.block_size}."
|
||||
@@ -1784,10 +1660,7 @@ def validate_declarations(
|
||||
server_args: Any,
|
||||
declarations: Sequence[Tuple[str, Dict[str, Any]]],
|
||||
) -> None:
|
||||
"""Fail-fast whitelist check at declaration time: a registry typo or a
|
||||
not-yet-resolvable field must be rejected at its slot, not only at
|
||||
publish time. Declarations never mutate ``server_args``.
|
||||
"""
|
||||
"""Reject unknown or non-resolvable fields before publication."""
|
||||
# Non-dataclass fixtures carry no Arg metadata (mirrors the
|
||||
# resolvable_fields escape); only real ServerArgs is validated.
|
||||
if not is_record(server_args):
|
||||
@@ -1805,10 +1678,7 @@ def validate_declarations(
|
||||
|
||||
@register_post_process
|
||||
def _hrm_text_attention_force(view: Any) -> dict:
|
||||
"""HRM-Text's bidirectional prefix attention only works on the Triton
|
||||
backend. Invoked as the last attention declaration of the resolution
|
||||
(mirroring the legacy runner-side force, which ran after the whole
|
||||
pipeline)."""
|
||||
"""Force Triton for HRM-Text bidirectional prefix attention after backend resolution."""
|
||||
if view.attention_backend not in (None, "triton"):
|
||||
logger.warning(
|
||||
f"Overriding --attention-backend "
|
||||
|
||||
@@ -1,10 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The resolution pipeline: the ordered dispatcher every publishing entry runs.
|
||||
|
||||
``ServerArgs.resolve_once`` is the only caller. It lives here rather than on the
|
||||
record because a step decides *about* the record; none of them is a member of
|
||||
it.
|
||||
"""
|
||||
"""Ordered configuration resolution, called by ``ServerArgs.resolve_once``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -22,32 +17,10 @@ from sglang.srt.arg_groups.resolution_hooks import run_hook
|
||||
|
||||
|
||||
def run_resolution_pipeline(server_args: Any) -> None:
|
||||
"""
|
||||
Orchestrates the handling of various server arguments, ensuring proper configuration and validation.
|
||||
"""Resolve and validate server arguments in dependency order.
|
||||
|
||||
Dispatcher style principles:
|
||||
1. Keep this function as an ordered dispatcher. Each step should be a
|
||||
named call into an ``arg_groups`` family; put imports, conditionals,
|
||||
mutations, and raises inside the family instead of inline here.
|
||||
2. Keep the dummy-model boundary as early as correctness allows. Only
|
||||
model-independent bootstrap, API/network/protocol validation, and
|
||||
errors that should fire for dummy models should run before it.
|
||||
3. Order handlers by dependency domains, not by historical insertion:
|
||||
internal/bootstrap, API/network/protocol, model source/path
|
||||
resolution, hardware/platform, model-specific adjustment,
|
||||
parallelism, kernel/attention backend, cuda graph, memory/cache,
|
||||
and advanced/debug features.
|
||||
4. Hide narrow integrations behind general handler names. The
|
||||
dispatcher should say what phase is being handled, not expose a
|
||||
vendor-, hook-, or feature-specific implementation detail.
|
||||
5. Give each handler one clear contract: what state it expects, what it
|
||||
may mutate, and whether it validates only. Long ordering comments
|
||||
belong in the helper or signal that the helper should be split.
|
||||
6. Call each step through ``run_hook(handle_x, server_args, ...)``, not
|
||||
``handle_x(server_args, ...)`` directly -- see
|
||||
``arg_groups/resolution_hooks.py``. This is every step's fixed
|
||||
position in the pipeline either way; registering an override changes
|
||||
what runs here, never when.
|
||||
Keep model-independent validation before the dummy-model return. Call steps
|
||||
through ``run_hook`` so plugins can replace them without changing their order.
|
||||
"""
|
||||
|
||||
# What the caller asked for, before any handler runs; this plus the
|
||||
@@ -128,20 +101,14 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
|
||||
run_hook(handle_model_source_paths, server_args)
|
||||
|
||||
# Validate mm_process_config.
|
||||
run_hook(handle_multimodal, server_args)
|
||||
# Validate SSL arguments early.
|
||||
run_hook(handle_ssl_validation, server_args)
|
||||
# Validate transcription/ASR-specific server args.
|
||||
run_hook(handle_asr_validation, server_args)
|
||||
|
||||
# Handle deprecated arguments.
|
||||
run_hook(handle_deprecated_args, server_args)
|
||||
|
||||
# Handle deprecated environment variables for prefill delayer.
|
||||
run_hook(handle_prefill_delayer_env_compat, server_args)
|
||||
|
||||
# Set missing default values.
|
||||
run_hook(handle_missing_default_values, server_args)
|
||||
|
||||
# expert_pack may replace a raw GGUF input with its generated local
|
||||
@@ -207,7 +174,6 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
# handle_gpu_memory_settings so the chunk size feeds memory budgeting.
|
||||
run_hook(apply_glm5_chunked_prefill_default, server_args)
|
||||
|
||||
# Handle device-specific backends.
|
||||
from sglang.srt.arg_groups.platform_hook import (
|
||||
handle_amd_specifics,
|
||||
handle_cpu_backends,
|
||||
@@ -231,12 +197,10 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
|
||||
run_hook(handle_platform_defaults, server_args)
|
||||
|
||||
# Handle memory-related, chunked prefill, and CUDA graph batch size configurations.
|
||||
from sglang.srt.arg_groups.memory_hook import handle_gpu_memory_settings
|
||||
|
||||
run_hook(handle_gpu_memory_settings, server_args)
|
||||
|
||||
# Apply model-specific adjustments.
|
||||
from sglang.srt.arg_groups.model_hook import (
|
||||
handle_model_capability_adjustments,
|
||||
handle_model_specific_adjustments,
|
||||
@@ -247,7 +211,6 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
# After the model overrides: Qwen4-Exp declares the PLE offload default there.
|
||||
run_hook(handle_offload_compatibility, server_args)
|
||||
|
||||
# Set kernel backends.
|
||||
run_post_process_pass(server_args, _sampling_backend_default)
|
||||
# Must run before _handle_attention_backend_compatibility so the
|
||||
# deterministic backend is set before auto-detection fills it in.
|
||||
@@ -291,19 +254,14 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
# _validate_prefill_only_disable_kv_cache_args().
|
||||
run_hook(handle_prefill_only_disable_kv_cache, server_args)
|
||||
|
||||
# Handle Hicache settings.
|
||||
run_hook(handle_hicache, server_args)
|
||||
|
||||
# Handle data parallelism.
|
||||
run_hook(handle_data_parallelism, server_args)
|
||||
|
||||
# Normalize load balancing defaults.
|
||||
run_hook(handle_load_balance_method, server_args)
|
||||
|
||||
# Handle context parallelism.
|
||||
run_hook(handle_context_parallelism, server_args)
|
||||
|
||||
# Handle MoE configurations.
|
||||
from sglang.srt.arg_groups.moe_hook import (
|
||||
handle_a2a_moe,
|
||||
handle_moe_kernel_config,
|
||||
@@ -320,11 +278,8 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
run_hook(handle_elastic_ep, server_args)
|
||||
run_hook(validate_experimental_sgl_marlin, server_args)
|
||||
|
||||
# Handle pipeline parallelism.
|
||||
run_post_process_pass(server_args, _pipeline_parallel_overlap_disable)
|
||||
|
||||
# Handle speculative decoding logic.
|
||||
|
||||
from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding
|
||||
|
||||
run_hook(handle_speculative_decoding, server_args)
|
||||
@@ -343,26 +298,20 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
|
||||
run_hook(validate_mega_moe_token_budget_for_model, server_args)
|
||||
|
||||
# Handle model loading format.
|
||||
run_hook(handle_load_format, server_args)
|
||||
|
||||
# Handle Encoder disaggregation.
|
||||
run_hook(handle_encoder_disaggregation, server_args)
|
||||
|
||||
# Validate tokenizer settings.
|
||||
run_hook(handle_tokenizer_batching, server_args)
|
||||
|
||||
# Propagate environment variables.
|
||||
run_hook(handle_environment_variables, server_args)
|
||||
|
||||
# Validate cache settings.
|
||||
run_hook(handle_cache_compatibility, server_args)
|
||||
|
||||
run_hook(handle_page_major_kv_layout, server_args)
|
||||
|
||||
run_hook(handle_unified_memory_pool, server_args)
|
||||
|
||||
# Handle diffusion LLM inference.
|
||||
from sglang.srt.arg_groups.dllm_hook import handle_dllm_inference
|
||||
|
||||
run_hook(handle_dllm_inference, server_args)
|
||||
@@ -370,14 +319,11 @@ def run_resolution_pipeline(server_args: Any) -> None:
|
||||
# Handle crash dump environment variables (must run before CUDA init).
|
||||
run_hook(handle_crash_dump_env, server_args)
|
||||
|
||||
# Handle debug utilities.
|
||||
run_hook(handle_debug_utils, server_args)
|
||||
|
||||
# Handle any other necessary validations.
|
||||
run_hook(handle_other_validations, server_args)
|
||||
|
||||
# Model-capability adjustments that legacy code applied at model-load
|
||||
# time; last declarations of the resolution, mirroring that order.
|
||||
# Apply model-capability constraints after backend selection.
|
||||
run_hook(handle_model_capability_adjustments, server_args)
|
||||
|
||||
finalize_cuda_graph_prefill_max_context(server_args)
|
||||
|
||||
@@ -1,45 +1,16 @@
|
||||
"""Out-of-tree replacement for a named step of the resolution pipeline.
|
||||
"""Plugin overrides for named resolution steps.
|
||||
|
||||
`run_resolution_pipeline` calls its steps by name, hardcoded, with no
|
||||
per-step dispatch through `self` -- there is nothing on `ServerArgs` left to
|
||||
subclass in order to change how one step decides. This is the replacement
|
||||
for that: a decorator that wraps whatever currently runs under a given name,
|
||||
and a dispatcher the pipeline calls instead of the bare function. The name
|
||||
itself is never spelled out at the call site -- `run_hook` reads it off the
|
||||
function it was handed -- so there is exactly one place a step's name is
|
||||
written by hand: the whitelist below, and whatever a downstream registrant
|
||||
passes to the decorator.
|
||||
|
||||
Whitelisted names only, the same discipline `Arg(resolvable=True)` uses for
|
||||
declarable fields: this project has already been bitten once by an
|
||||
unqualified name collision in this exact pipeline (`_parse_cuda_graph_config`
|
||||
and `_handle_cuda_graph_config` merged under one rename and the dispatcher
|
||||
called itself). A name that is not on the list fails loudly at import time,
|
||||
not silently at the call site three modules away.
|
||||
|
||||
Every step takes exactly `(server_args)`, `handle_hardware_runtime_validation`
|
||||
included -- it does not read `server_args` (see the comment at its
|
||||
definition), but it takes the parameter anyway so `run_hook` never has to
|
||||
special-case an arity. An override's own signature always matches: `def
|
||||
mine(server_args, previous)`.
|
||||
Overrides receive ``(server_args, previous)`` and may call ``previous`` to
|
||||
wrap the existing step. Only whitelisted names are accepted. Registration
|
||||
changes the implementation at that step's existing pipeline position.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Callable, Dict, FrozenSet, List
|
||||
|
||||
# Every step this pipeline runs by name that a downstream package may replace.
|
||||
# Add a name here only alongside the call site's own switch to `run_hook` --
|
||||
# an entry with no matching `run_hook(...)` call is a name nothing will ever
|
||||
# look up. `test_every_whitelisted_hook_has_a_call_site` in
|
||||
# test_resolution_hook_registry.py holds the two directions of this equal by
|
||||
# construction, so this list cannot drift from `pipeline.py` silently.
|
||||
#
|
||||
# `handle_offload_compatibility` runs at two different points in the pipeline
|
||||
# (once before model-specific adjustments, once after -- see the comments at
|
||||
# its call sites). An override registered for it applies at both, identically;
|
||||
# there is no way to target "just the second call" through this mechanism,
|
||||
# because the name is all `run_hook` has to key on.
|
||||
# Names must match run_hook call sites in pipeline.py. An override applies
|
||||
# to every invocation of that name, including repeated steps.
|
||||
_OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
|
||||
{
|
||||
"handle_mega_moe",
|
||||
@@ -125,42 +96,16 @@ _OVERRIDABLE_HOOKS: FrozenSet[str] = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
# name -> registered overrides, oldest first. Each takes `(server_args,
|
||||
# previous)`, where `previous` is the callable it wraps -- the built-in on
|
||||
# the first registration, the previous registrant's own wrapper on every one
|
||||
# after. Process-global as a `dict[str, list]` so a test isolates it the way
|
||||
# test_model_overrides.py isolates `_MODEL_OVERRIDE_FNS`:
|
||||
# `patch.dict(..., clear=True)`.
|
||||
# Step name -> wrappers in registration order.
|
||||
_HOOKS: Dict[str, List[Callable[[Any, Callable[[Any], None]], None]]] = {}
|
||||
|
||||
|
||||
def register_resolution_hook(name: str):
|
||||
"""Replace (or wrap) the pipeline step named ``name``.
|
||||
"""Register ``fn(server_args, previous)`` for a pipeline step.
|
||||
|
||||
The decorated function is called as ``fn(server_args, previous)``.
|
||||
``previous`` is a plain ``server_args -> None`` callable: the built-in
|
||||
step on the first registration for this name, or the previous
|
||||
registrant's own wrapper on every registration after that. Call it to
|
||||
run what would have run without this override -- the `super().handle_x()`
|
||||
shape, expressed as an explicit argument instead of a method-resolution
|
||||
lookup, because there is no class hierarchy here for `super()` to walk.
|
||||
Not calling it is a full replacement.
|
||||
|
||||
Registering twice for the same name does not replace the first
|
||||
registration; it wraps it. The **last** registration is outermost --
|
||||
runs first, and decides whether/when its `previous` (everything
|
||||
registered before it, down to the built-in) runs at all. Two downstream
|
||||
packages that both target the same name compose in whichever order they
|
||||
happened to import in; if that order matters to you, make one of them
|
||||
import the other first.
|
||||
|
||||
This changes *what* runs at the step's existing position in the
|
||||
pipeline, never *when*: the call site in `pipeline.py` is unmoved, so
|
||||
every other step keeps the order it already had. A wrapped step's own
|
||||
declarations reach `resolution_result` the same way any declaration
|
||||
does -- only readers from this position onward see them; a step that
|
||||
already ran and read the old value before this one's `previous` (or the
|
||||
built-in) declared its replacement has already made its decision on it.
|
||||
``previous(server_args)`` calls the previously registered wrapper, or the
|
||||
built-in step for the first registration. Omitting it replaces the step.
|
||||
The last registration runs outermost, so plugin import order matters.
|
||||
"""
|
||||
if name not in _OVERRIDABLE_HOOKS:
|
||||
raise ValueError(
|
||||
@@ -178,22 +123,10 @@ def register_resolution_hook(name: str):
|
||||
|
||||
|
||||
def run_hook(builtin: Callable[[Any], None], server_args: Any) -> None:
|
||||
"""Run ``builtin`` -- the registered chain if anything overrode it under
|
||||
its name, ``builtin`` directly otherwise.
|
||||
"""Run registered wrappers for ``builtin.__name__``, or the built-in step.
|
||||
|
||||
The name is ``builtin.__name__``, not a second argument: the call site
|
||||
already has the function in scope (a function-local import, same as
|
||||
every other step), and spelling the name out again next to it is the
|
||||
exact "two copies that can silently disagree" shape this project keeps
|
||||
removing elsewhere. ``builtin`` must therefore be a plain, named
|
||||
function -- every real call site is -- not a lambda or a bound method.
|
||||
|
||||
Called from the step's fixed position in `run_resolution_pipeline`. The
|
||||
chain is rebuilt from the registry on every call rather than cached at
|
||||
registration time, because at registration time (import time, before any
|
||||
`ServerArgs` exists) there is no `server_args` yet and the first
|
||||
registrant's `previous` cannot be bound to anything real until a call
|
||||
actually happens.
|
||||
``builtin`` must be a named function. Wrappers receive the same server
|
||||
arguments as the built-in step.
|
||||
"""
|
||||
step = builtin
|
||||
for fn in _HOOKS.get(builtin.__name__, ()):
|
||||
|
||||
@@ -11,35 +11,16 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""A single structured accessor for process-static runtime state.
|
||||
"""Process-wide configuration, parallel state, flags, and resources.
|
||||
|
||||
``get_parallel()`` returns a ``ParallelContext`` for configuration, ranks, and
|
||||
process-group handles. Reads use scoped overrides, then permanent overrides,
|
||||
then the published configuration. ``publish`` records ranks from ``SpawnRanks``;
|
||||
distributed initialization records group handles.
|
||||
``get_server_args()`` retains the raw input for diagnostics. Namespace
|
||||
accessors such as ``get_exec()`` return resolved configuration published
|
||||
from declarations; ``RuntimeContext.override`` changes those bags.
|
||||
``get_parallel()`` also exposes ranks and process groups, with scoped and
|
||||
permanent overrides taking precedence over published values.
|
||||
|
||||
``get_server_args()`` returns the process-wide ``ServerArgs``. This is the
|
||||
user's raw input, kept **read-only** for debug and reproduction; what
|
||||
resolution decided lives in the declarations (``resolution_result``) and, for
|
||||
business code, in the namespace bags below -- never on this object's fields. The context owns the storage:
|
||||
publishing goes through ``RuntimeContext.set_server_args`` (the legacy
|
||||
``set_global_server_args_for_scheduler`` is a thin shim over this slot;
|
||||
``get_global_server_args`` is retired and raises).
|
||||
|
||||
``get_exec()`` / ``get_memory()`` / ``get_schedule()`` / ``get_device()`` /
|
||||
``get_model()`` / ``get_spec()`` / ``get_lora()`` / ``get_mm()`` /
|
||||
``get_disagg()`` / ``get_serving()`` / ``get_observability()`` return the
|
||||
resolved **config namespace bags** — the single source of truth for config,
|
||||
snapshotted from ``server_args`` at publish, one bag per namespace class in
|
||||
``arg_groups/fields/`` (multi-level under ``exec.*``). Reads are attribute
|
||||
chains (``get_exec().moe.moe_runner_backend``); bags are read-only by bare
|
||||
assignment (written via ``override``).
|
||||
|
||||
``get_flags()`` returns the runtime-flags tier: state that is **not** a pure
|
||||
function of config (the capture lifecycle, ACTIVE MoE backend, DP runtime) —
|
||||
never a mirror of config. Flags live in typed dataclass groups; reads and
|
||||
writes are plain attribute access, and each group offers a transactional,
|
||||
test-only ``override(**kw)``.
|
||||
``get_flags()`` holds mutable runtime state; ``get_resources()`` owns process
|
||||
handles, and ``get_forward()`` holds per-forward state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -612,13 +593,7 @@ class SpFlags(_FlagGroupBase):
|
||||
|
||||
|
||||
class Flags(_FlagGroupBase):
|
||||
"""Root of the runtime-flags tier.
|
||||
|
||||
Resolved configuration lives in the config bags below (projected from the
|
||||
declarations at publish) — this tier only carries genuine runtime
|
||||
state whose value is not a function of the configuration alone, grouped
|
||||
by lifecycle (``capture``) or subsystem (``moe`` / ``dp`` / ``sp``).
|
||||
"""
|
||||
"""Mutable runtime state; resolved configuration lives in the config bags."""
|
||||
|
||||
capture: CaptureFlags = msgspec.field(default_factory=CaptureFlags)
|
||||
moe: MoeFlags = msgspec.field(default_factory=MoeFlags)
|
||||
@@ -662,20 +637,12 @@ class Resources(_FlagGroupBase):
|
||||
|
||||
|
||||
class ForwardFlags:
|
||||
"""Per-forward runtime flags with one API and two backings.
|
||||
"""Scoped per-forward flags.
|
||||
|
||||
Flags read only from eager Python are backed by context variables, so
|
||||
nested scopes and threads stay isolated (a new thread sees the defaults).
|
||||
Flags that are read or written *inside torch.compile-traced model code*
|
||||
(``_GRAPH_VISIBLE``) are backed by plain dict slots instead: dynamo
|
||||
cannot trace ``ContextVar.get``/``set``, while plain reads it guards on
|
||||
— the storage form these flags had before joining the tier. Their
|
||||
writers and readers are single-threaded per process (TBO interleaves
|
||||
ubatches on one thread; attention-TP input scattering excludes TBO), so
|
||||
context isolation is not needed for correctness.
|
||||
|
||||
``scoped(**kw)`` — the one regular write path — restores on exit for
|
||||
both backings. ``set()`` exists for the legacy unscoped setters' shims.
|
||||
Eager-only flags use ContextVars for thread and nested-scope isolation.
|
||||
``_GRAPH_VISIBLE`` flags use plain dict slots because Dynamo cannot trace
|
||||
ContextVar access. Their readers and writers run on one thread per process.
|
||||
``scoped`` restores both backings on exit; ``set`` supports unscoped setters.
|
||||
"""
|
||||
|
||||
_DEFAULTS = {
|
||||
@@ -792,24 +759,12 @@ class ForwardFlags:
|
||||
|
||||
|
||||
class _ConfigBag:
|
||||
"""A resolved-config namespace bag.
|
||||
"""Resolved namespace with nested bags and guarded writes.
|
||||
|
||||
Values are snapshotted from ``server_args`` at ``publish`` and this bag is
|
||||
the **single source of truth** for its fields thereafter. Read is plain
|
||||
attribute access; the bag is read-only by bare assignment. The sanctioned
|
||||
writers are ``get_context().override(source, ...)`` (permanent) and
|
||||
the scoped ``.override(**kw)`` context manager (tests). Sub-namespaces
|
||||
(e.g. ``exec.moe``) are nested ``_ConfigBag`` instances reached by attribute.
|
||||
|
||||
Leaves and sub-bags are stored as **real instance attributes** (in
|
||||
``__dict__``), so ``bag.leaf`` / ``bag.sub`` is a plain attribute load that
|
||||
``torch.compile`` / dynamo can trace — config reads inside a compiled model
|
||||
forward (e.g. ``get_exec().comm.enable_symm_mem`` in the embedding layer)
|
||||
must not graph-break. ``_fields`` / ``_subs`` keep the authoritative
|
||||
name→value maps used for override routing, membership, and scoped restore;
|
||||
``__getattr__`` is only a fallback for genuinely absent names. (Deliberately
|
||||
no ``__slots__``: leaves are dynamic, and the ``__dict__`` is what makes the
|
||||
reads traceable.)
|
||||
Publication snapshots resolved values; ``RuntimeContext.override`` updates
|
||||
them permanently and ``override`` restores scoped changes on exit.
|
||||
Leaves and sub-bags are real instance attributes so Dynamo can trace reads.
|
||||
``_fields`` and ``_subs`` track names for routing and restoration.
|
||||
"""
|
||||
|
||||
def __init__(self, path: str):
|
||||
@@ -837,14 +792,12 @@ class _ConfigBag:
|
||||
)
|
||||
|
||||
def _set(self, name: str, value: Any) -> None:
|
||||
"""Internal write (publish + override) that bypasses the read-only guard.
|
||||
Updates both the bookkeeping map and the real attribute (traceable read)."""
|
||||
"""Update the leaf map and traceable attribute, bypassing the write guard."""
|
||||
object.__getattribute__(self, "_fields")[name] = value
|
||||
object.__setattr__(self, name, value)
|
||||
|
||||
def _set_sub(self, name: str, sub: _ConfigBag) -> None:
|
||||
"""Register a nested bag as both a bookkeeping entry and a real
|
||||
attribute (so ``bag.sub`` is a plain, traceable attribute load)."""
|
||||
"""Register a nested bag in the lookup map and as a traceable attribute."""
|
||||
object.__getattribute__(self, "_subs")[name] = sub
|
||||
object.__setattr__(self, name, sub)
|
||||
|
||||
@@ -853,13 +806,10 @@ class _ConfigBag:
|
||||
|
||||
@contextmanager
|
||||
def override(self, **kwargs):
|
||||
"""Scoped, transactional override of this bag's own leaves (keys
|
||||
validated before any write; restored on exit).
|
||||
"""Temporarily override this bag's leaves, validating all keys before writing.
|
||||
|
||||
For a window where one runner's value differs from the process's — a
|
||||
draft model loading under ``--speculative-draft-load-format`` while the
|
||||
target keeps ``--load-format`` — and for tests forcing a code path.
|
||||
A permanent change goes through ``get_context().override``."""
|
||||
Restores on exit; use ``RuntimeContext.override`` for permanent changes.
|
||||
"""
|
||||
fields = object.__getattribute__(self, "_fields")
|
||||
unknown = set(kwargs) - set(fields)
|
||||
if unknown:
|
||||
@@ -876,16 +826,7 @@ class _ConfigBag:
|
||||
|
||||
|
||||
def _build_config_bags(server_args: Any) -> dict:
|
||||
"""Snapshot the resolution result into the namespace bag tree.
|
||||
|
||||
The tree is ``namespace_of``: each field is placed by the namespace class
|
||||
that declares it (``arg_groups/fields/``). Each leaf comes from
|
||||
``resolution_result`` -- the declaration if resolution made one, else what
|
||||
the caller supplied. Returns ``{top_level_name: _ConfigBag}``, arbitrarily
|
||||
nested (``exec.moe.eplb.…``). Only dataclass fields are placed, so derived
|
||||
properties and methods are naturally excluded (they stay on the bag). A
|
||||
name used as both a leaf and a subgroup at the same level is a hard error
|
||||
— no silent shadowing."""
|
||||
"""Project resolved fields by namespace, rejecting leaf/subgroup name collisions."""
|
||||
from sglang.srt.arg_groups.arg_utils import namespace_of
|
||||
from sglang.srt.arg_groups.overrides import resolution_result
|
||||
|
||||
@@ -894,10 +835,6 @@ def _build_config_bags(server_args: Any) -> dict:
|
||||
for field, path in namespace_of(type(server_args)).items():
|
||||
value = resolution_result(server_args, field, _MISSING)
|
||||
if value is _MISSING:
|
||||
# Every placed field is a dataclass field, so a resolved config
|
||||
# always carries it; a miss means a malformed/partial config object
|
||||
# was published. Fail loud here rather than silently omitting the
|
||||
# leaf (which surfaces later as a confusing "not a published leaf").
|
||||
raise AttributeError(
|
||||
f"config field {field!r} belongs to namespace {path!r} but is absent from "
|
||||
f"the published {type(server_args).__name__}; cannot project its bag leaf"
|
||||
@@ -930,18 +867,9 @@ def _build_config_bags(server_args: Any) -> dict:
|
||||
|
||||
|
||||
def _install_derived_leaves(tops: dict, server_args: Any) -> None:
|
||||
"""Compute the declared config-derived fields into their bags.
|
||||
"""Compute declared derived leaves once at publication.
|
||||
|
||||
A `Derived(fn=...)` is a pure function of the published configuration, so it
|
||||
is computed once, here, and stored as an ordinary leaf: readers get a plain
|
||||
attribute load, and there is one answer rather than a pre-publish spelling
|
||||
and a post-publish one that have to be kept saying the same thing.
|
||||
|
||||
The function is handed the whole resolved config, not the bag it lands in.
|
||||
A derivation is free to span namespaces and they do -- the mamba
|
||||
extra-buffer predicate reads `memory.disable_radix_cache` alongside its own
|
||||
`exec.mamba` strategy -- which is exactly why it cannot be written as a
|
||||
method on either bag.
|
||||
Derivations receive the whole resolved config because they may span namespaces.
|
||||
"""
|
||||
import importlib
|
||||
|
||||
@@ -990,9 +918,6 @@ def _resolved_or_field(server_args: Any, name: str, default: Any) -> Any:
|
||||
decided = resolution_result(server_args, name)
|
||||
if decided is not None:
|
||||
return decided
|
||||
# The default is for the callers that hand over something record-shaped but
|
||||
# not a record -- the fake configs the context tests publish, and `object()`
|
||||
# for the sentinel publish. A real ServerArgs always has the field.
|
||||
return getattr(server_args, name, default)
|
||||
|
||||
|
||||
@@ -1023,10 +948,7 @@ class RuntimeContext:
|
||||
self.forward = ForwardFlags()
|
||||
|
||||
def get_stream(self, name: str) -> Any:
|
||||
"""Named process-level side stream: get-or-create, shared by
|
||||
name (the keyed-lazy pattern of the persistent buffers). Creation is
|
||||
a driver call that must stay outside cuda-graph capture — call sites
|
||||
lease their stream at init/warmup time."""
|
||||
"""Get or create a named process stream. Call before CUDA graph capture."""
|
||||
from sglang.srt.arg_groups.overrides import resolution_result
|
||||
|
||||
stream = self.resources.streams.get(name)
|
||||
@@ -1049,9 +971,7 @@ class RuntimeContext:
|
||||
return stream
|
||||
|
||||
def get_buffer(self, name: str, factory: Any) -> Any:
|
||||
"""Named process-level persistent buffer: get-or-create via
|
||||
``factory()``, shared by name (the keyed-lazy pattern of the
|
||||
persistent buffers / named streams)."""
|
||||
"""Get or create a named persistent buffer using ``factory()``."""
|
||||
buf = self.resources.buffers.get(name)
|
||||
if buf is None:
|
||||
buf = factory()
|
||||
@@ -1068,24 +988,11 @@ class RuntimeContext:
|
||||
return server_args
|
||||
|
||||
def set_server_args(self, server_args: ServerArgs) -> None:
|
||||
"""Publish the process-wide ``ServerArgs`` into the context-owned slot.
|
||||
|
||||
Overwrite-allowed: a re-publish replaces the slot (test kits re-publish
|
||||
per test; production ordering discipline lives at the call-sites, e.g.
|
||||
the draft-worker guard in ``ModelRunner.__init__``). The published
|
||||
object is the raw input; the resolution it carries is its declaration
|
||||
stash, which is what the bags are projected from.
|
||||
"""
|
||||
# Seed the capture tier for the new lifecycle (defaults for sentinel
|
||||
# and mock publishes, which carry no config). Through the resolution,
|
||||
# not the field: the field is the operator's input.
|
||||
"""Replace the raw input and rebuild resolved config bags, clearing override provenance."""
|
||||
self.flags.capture.enable_torch_compile = bool(
|
||||
_resolved_or_field(server_args, "enable_torch_compile", False)
|
||||
)
|
||||
self._server_args = server_args
|
||||
# Snapshot resolved config into the namespace bags (the single source of
|
||||
# truth for config reads). Placed by `namespace_of`; a mock/partial
|
||||
# config that declares no namespace yields an empty tree (no bags).
|
||||
# Preserve the launcher-assigned device when rebuilding config bags.
|
||||
stated = {}
|
||||
if self._config_bags is not None:
|
||||
@@ -1109,19 +1016,13 @@ class RuntimeContext:
|
||||
"max_speculative_num_draft_tokens",
|
||||
max_draft_tokens_of(server_args),
|
||||
)
|
||||
# Wire the published `parallel` bag onto the live wrapper: it is the slot
|
||||
# the `config` property reads, which is how config-only leaves like
|
||||
# pp_max_micro_batch_size are spelled.
|
||||
self.parallel._config = self._config_bags.get("parallel")
|
||||
# A direct install is roleless; ``publish`` assigns the role afterwards.
|
||||
self._overrides_log = []
|
||||
self._publish_role = None
|
||||
|
||||
def config_bag(self, name: str) -> _ConfigBag:
|
||||
"""Return the top-level config namespace bag (``device`` / ``model`` /
|
||||
``exec`` / ``schedule`` / ``memory`` / ``spec`` / ``lora`` / ``mm`` /
|
||||
``disagg`` / ``serving`` / ``observability``). Fails closed until
|
||||
``publish`` / ``set_server_args`` has projected it."""
|
||||
"""Return a top-level namespace bag; raise if it has not been published."""
|
||||
bags = self._config_bags
|
||||
if not bags or name not in bags:
|
||||
raise ValueError(f"config namespace {name!r} not published")
|
||||
@@ -1159,16 +1060,10 @@ class RuntimeContext:
|
||||
)
|
||||
|
||||
def override(self, source: str, **fields) -> None:
|
||||
"""The business mutation entry: write resolved config
|
||||
leaves onto the namespace bags — the single source of truth. It does
|
||||
**not** touch ``server_args`` (the pristine startup record) and there is
|
||||
no write-through, so the old "wrote one store, read another" desync class
|
||||
cannot occur.
|
||||
"""Update resolved config leaves without changing the raw input.
|
||||
|
||||
Each flat field name is routed to its bag by ``namespace_of`` (flat
|
||||
names are unique across namespaces). Validation is all-or-nothing: an
|
||||
unknown / unprojected field aborts before any write. ``source`` is
|
||||
recorded for provenance / reproduction.
|
||||
Validate all field names and target bags before writing. Record ``source``
|
||||
and the changed fields for provenance.
|
||||
"""
|
||||
if not fields:
|
||||
return
|
||||
@@ -1204,12 +1099,7 @@ class RuntimeContext:
|
||||
self._overrides_log.append((source, dict(fields)))
|
||||
|
||||
def config_leaf(self, name: str):
|
||||
"""One resolved config leaf by field name — the read side of ``override``.
|
||||
|
||||
Callers that hold a field name rather than a namespace (a readback
|
||||
endpoint, a control-plane handler) would otherwise have to know which
|
||||
bag it lives in.
|
||||
"""
|
||||
"""Read a resolved config leaf by its flat field name."""
|
||||
bags = self._config_bags
|
||||
if bags is None:
|
||||
raise ValueError("config not published; cannot read a config leaf")
|
||||
@@ -1227,36 +1117,16 @@ class RuntimeContext:
|
||||
return getattr(bag, name)
|
||||
|
||||
def overrides_log(self) -> list:
|
||||
"""Provenance of post-publish ``override`` calls: ``[(source, {field: value})]``.
|
||||
|
||||
Returns deep-ish copies (source, dict(fields)) so callers inspecting the
|
||||
log cannot mutate the recorded provenance in place."""
|
||||
"""Return ``(source, fields)`` entries with shallow copies of the field dicts."""
|
||||
return [(source, dict(fields)) for source, fields in self._overrides_log]
|
||||
|
||||
def resolved_server_args_dict(self, base: dict | None = None) -> dict:
|
||||
"""Serialize the *resolved* config: the pristine ``server_args`` fields
|
||||
with every post-publish ``override`` overlaid.
|
||||
"""Overlay this process's runtime overrides on the resolved startup config.
|
||||
|
||||
``get_internal_state`` reports this, and ``/server_info`` carries it in
|
||||
the ``internal_states`` block, so scheduler-side runtime changes show up
|
||||
in a readback: HiCache attach/detach, the generated forward-pass-metrics
|
||||
endpoint, tunables set via ``/set_internal_state``.
|
||||
|
||||
``base`` defaults to ``server_args.resolved_dict()`` -- the record's
|
||||
fields as resolution decided them, nested dataclasses expanded. (It used
|
||||
to be ``dict(vars(server_args))``, which carried the private resolution
|
||||
bookkeeping and the ``model_config`` memo into the readback.) Override
|
||||
leaves are flat ``ServerArgs`` field names, so overlaying them onto the
|
||||
top level of the base is exact.
|
||||
|
||||
The log is per process: it carries what *this* process overrode. A
|
||||
weight reload records ``model_path`` and ``load_format`` from the
|
||||
scheduler process (``ModelRunner.update_model_fields``); the tokenizer
|
||||
process records only ``load_format`` and keeps ``model_path`` /
|
||||
``served_model_name`` as ``TokenizerManager`` attributes, which
|
||||
``TokenizerManager.resolved_config_dict`` overlays on top of this dump.
|
||||
The top-level ``/server_info`` fields are the startup record, not this
|
||||
dump.
|
||||
``base`` defaults to ``server_args.resolved_dict()``. Scheduler readbacks
|
||||
include these overrides in ``/server_info``'s ``internal_states`` block;
|
||||
the top-level fields describe startup config. TokenizerManager separately
|
||||
overlays its per-instance model identity.
|
||||
"""
|
||||
d = self.server_args.resolved_dict() if base is None else dict(base)
|
||||
for _source, fields in self._overrides_log:
|
||||
@@ -1264,24 +1134,10 @@ class RuntimeContext:
|
||||
return d
|
||||
|
||||
def override_server_args(self, **fields) -> _ServerArgsOverride:
|
||||
"""Test-only scoped override for the config tier — the sibling of
|
||||
``get_parallel().override()`` and the flag groups' ``override()``:
|
||||
tests force execution paths by overriding the context instead of
|
||||
hand-building config objects.
|
||||
"""Publish a temporary dummy config for tests and restore the previous context.
|
||||
|
||||
``install()`` (or entering it as a context manager) publishes a fresh
|
||||
dummy-boundary ``ServerArgs`` carrying ``fields`` and returns it;
|
||||
``restore()`` (or exiting) reinstates whatever the slot held before.
|
||||
|
||||
This is the sanctioned way for a test to get a published context, and
|
||||
it stays. The transitional reason it was introduced for — production
|
||||
code branching on raw ``server_args`` fields at runtime — is gone (the
|
||||
read ratchet pins business reads at zero), but a test that exercises
|
||||
bag readers still needs bags, and the bag tree is projected *from an
|
||||
instance*: something has to publish one. Prefer the finer-grained
|
||||
scoped overrides (``get_exec().override(...)``, the flag groups'
|
||||
``override``) on top of a published context when a test only needs to
|
||||
force one leaf.
|
||||
Use as a context manager or call ``install`` / ``restore`` explicitly.
|
||||
Prefer a bag's scoped ``override`` when only existing leaves need changing.
|
||||
"""
|
||||
return _ServerArgsOverride(self, fields)
|
||||
|
||||
@@ -1341,19 +1197,11 @@ class _ServerArgsOverride:
|
||||
raise ValueError(
|
||||
f"override_server_args: unknown ServerArgs field(s): {sorted(unknown)}"
|
||||
)
|
||||
# Declared so the projection sees it; late, because the record is
|
||||
# resolved already and not yet published.
|
||||
# Split on whether the name is a field, not on whether it starts with
|
||||
# an underscore: `_speculative_draft_quantization_explicitly_set` is a
|
||||
# real field, and seeding it as a raw attribute would leave the earlier
|
||||
# declaration authoritative, so `resolution_result` and the bag would
|
||||
# both keep answering the pre-override value.
|
||||
# Declare config fields even when underscore-prefixed; only non-fields seed caches.
|
||||
fields = set(type(server_args).__struct_fields__)
|
||||
declared = {n: v for n, v in self._fields.items() if n in fields}
|
||||
if declared:
|
||||
declare_resolution(server_args, "override_server_args", **declared)
|
||||
# What is left seeds the record's own private caches (`_model_config`
|
||||
# and friends), which are not configuration and never were.
|
||||
seeds = {n: v for n, v in self._fields.items() if n not in fields}
|
||||
for name, value in seeds.items():
|
||||
msgspec.Struct.__setattr__(server_args, name, value)
|
||||
@@ -1462,7 +1310,7 @@ def get_observability() -> _ConfigBag:
|
||||
return _CONTEXT.config_bag("observability")
|
||||
|
||||
|
||||
# --- Per-role namespace sets (2c) -------------------------------------------
|
||||
# --- Per-role namespace sets -------------------------------------------
|
||||
#
|
||||
# ``publish(role=...)`` records which process type installed the config; this
|
||||
# table declares which top-level config namespaces each role reads. ``None``
|
||||
@@ -1483,22 +1331,10 @@ ROLE_NAMESPACE_SETS: dict[str, frozenset[str] | None] = {
|
||||
# Reads (almost) everything by design — the model-executing process.
|
||||
"scheduler": None,
|
||||
"test": None,
|
||||
# The DP controller's static read set, checked against the module: the
|
||||
# elastic-EP gate, the load-balance method, the watchdog timeout, and the
|
||||
# disaggregation mode.
|
||||
# `observability` and `serving` were added when the controller's metrics
|
||||
# gate, tracing setup and worker-port broadcast stopped reading the record:
|
||||
# under `enforce` the set is what the process may read, so a conversion
|
||||
# that reaches a new namespace has to widen it in the same commit.
|
||||
"dp_controller": frozenset(
|
||||
{"exec", "parallel", "device", "disagg", "observability", "serving"}
|
||||
),
|
||||
# Record-mode audit (2026-08-06, text model, /generate + /get_server_info +
|
||||
# /v1/models): reads exactly {"serving"} — the per-instance managers read
|
||||
# self.server_args by design. Still declared full, because that run did not
|
||||
# exercise the multimodal processors, LoRA/score endpoints, the disagg
|
||||
# roles, or the gRPC bridge; narrowing needs those shapes audited too, and
|
||||
# a wrong set fails a request rather than a test.
|
||||
# Keep unrestricted until multimodal, LoRA, disaggregation, and gRPC reads are audited.
|
||||
"tokenizer": None,
|
||||
# Deployment shapes not exercised locally; audit before restricting.
|
||||
"detokenizer": None,
|
||||
@@ -1712,16 +1548,9 @@ def _attention_ranks(parallel, tp_rank: int) -> dict:
|
||||
|
||||
|
||||
def assert_published(server_args, *, role: str) -> RuntimeContext:
|
||||
"""This record, under this role, is already published -- or fail loud.
|
||||
"""Require this record and role to have been published by the process entry.
|
||||
|
||||
Publishing is the process entry's job: `run_scheduler_process`,
|
||||
`init_multi_tokenizer`, a spawned encoder worker, the benchmark work
|
||||
functions. A constructor arriving here unpublished means one of those
|
||||
entries is missing.
|
||||
|
||||
A `publish` at this point re-projects the bags over a live process,
|
||||
discarding every `override()` taken since and the provenance log with it,
|
||||
so this raises.
|
||||
Re-publishing here would discard runtime overrides and their provenance.
|
||||
"""
|
||||
if _CONTEXT._server_args is server_args and _CONTEXT._publish_role == role:
|
||||
return _CONTEXT
|
||||
@@ -1866,13 +1695,9 @@ def reset_context() -> None:
|
||||
|
||||
|
||||
def remote_instance_transfer_engine_enabled(load_format: str | None = None) -> bool:
|
||||
"""Whether remote-instance weight loading runs over the transfer engine.
|
||||
"""Check transfer-engine loading against current model config.
|
||||
|
||||
Every input is a ``model`` leaf, so this derives from the bags and follows a
|
||||
post-publish override; ``ServerArgs.remote_instance_weight_loader_use_transfer_engine``
|
||||
is the pre-publish equivalent, and both go through the same helper.
|
||||
``load_format`` is the caller's own (a draft runner loading under
|
||||
``--speculative-draft-load-format`` has one the process record does not).
|
||||
``load_format`` can override the process value for a draft runner.
|
||||
"""
|
||||
from sglang.srt.arg_groups.overrides import remote_instance_transfer_engine_of
|
||||
|
||||
@@ -1880,15 +1705,9 @@ def remote_instance_transfer_engine_enabled(load_format: str | None = None) -> b
|
||||
|
||||
|
||||
def max_prefill_buffer_tokens() -> int:
|
||||
"""The prefill-buffer ceiling: ``chunked_prefill_size``, except PP dynamic
|
||||
chunking can grow chunks toward ``max_prefill_tokens`` and probe at 1.25x.
|
||||
"""Return the registered prefill ceiling, or derive it from current config.
|
||||
|
||||
The default derives from published leaves (``schedule`` plus the configured
|
||||
PP size), so it follows post-publish overrides;
|
||||
``overrides.max_prefill_buffer_tokens`` is the pre-publish equivalent and
|
||||
``TestDerivedPredicatesAgreeAcrossTiers`` pins the two equal. Records with
|
||||
a registered ceiling provider (see ``register_prefill_buffer_ceiling``)
|
||||
answer through it.
|
||||
PP dynamic chunking can grow toward ``max_prefill_tokens`` and probe at 1.25x.
|
||||
"""
|
||||
schedule = get_schedule()
|
||||
chunked = (
|
||||
@@ -1905,14 +1724,7 @@ def max_prefill_buffer_tokens() -> int:
|
||||
|
||||
|
||||
def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float:
|
||||
"""The activation working-set reserve held back before cuda-graph capture.
|
||||
|
||||
Derived from published leaves across four bags (``disagg`` / ``schedule`` /
|
||||
``exec.graph`` / ``spec``) plus the configured parallel sizes, so it follows
|
||||
a post-publish override; ``pre_capture_activation_reserve_mb_of`` in
|
||||
``arg_groups.overrides`` is the config-shaped equivalent and
|
||||
``TestDerivedPredicatesAgreeAcrossTiers`` pins the two equal.
|
||||
"""
|
||||
"""Return the activation reserve in MB before CUDA graph capture, using current config."""
|
||||
schedule = get_schedule()
|
||||
if get_disagg().disaggregation_mode == "decode":
|
||||
running_requests = (
|
||||
@@ -1936,12 +1748,7 @@ def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float:
|
||||
return reserved_mem
|
||||
|
||||
|
||||
# --- Platform facts -----------------------------------------------------------
|
||||
#
|
||||
# One address for what kind of machine this is, so a reader asks
|
||||
# `get_platform().is_sm100` and an override is stated once instead of patched
|
||||
# into every module that imported a probe. True before publish, so the context
|
||||
# probes when no override is installed; `utils.common` holds the implementation.
|
||||
# Platform probes work before publication and support scoped overrides.
|
||||
|
||||
_PLATFORM_PROBES: Dict[str, str] = {
|
||||
"is_cuda": "is_cuda",
|
||||
@@ -2072,14 +1879,7 @@ def override_platform(**facts: Any) -> _PlatformOverride:
|
||||
return _PlatformOverride(**facts)
|
||||
|
||||
|
||||
# --- Derived config accessors ------------------------------------------------
|
||||
#
|
||||
# A few values are computed from several config fields plus the HF config, so
|
||||
# they are derived accessors rather than namespace leaves. Business code must
|
||||
# not reach for the startup record to get them: these accessors are the named
|
||||
# home, and this module — which owns the slot — is the only place that reads
|
||||
# it. Each one keeps the pre-publish function's exact semantics, including which model
|
||||
# config it derives from (always the process's, i.e. the target's).
|
||||
# Derived config accessors use the process model (the target during speculation).
|
||||
|
||||
|
||||
def mamba_cache_chunk_size() -> int:
|
||||
@@ -2146,8 +1946,6 @@ def attention_backends() -> tuple:
|
||||
"""
|
||||
from sglang.srt.arg_groups.overrides import attention_backends_of
|
||||
|
||||
# All three leaves live in the same bag, so the resolution pipeline's own
|
||||
# helper applies directly -- one definition of the fallback rule.
|
||||
return attention_backends_of(get_exec().kernel)
|
||||
|
||||
|
||||
|
||||
@@ -11,25 +11,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
"""Server argument declarations, resolution, and CLI registration.
|
||||
|
||||
Keep this file in the following top-level order:
|
||||
|
||||
1. Imports and the module logger.
|
||||
2. Public extension-point choice lists, with each legacy ``add_*`` alias
|
||||
immediately below the choice list it extends.
|
||||
3. Shared (non-extensible) choice lists, scalar defaults, and deprecated
|
||||
aliases. A choice list used by only one field belongs inline in that field.
|
||||
4. ``ServerArgs``: fields first, then resolution/validation helpers, then CLI
|
||||
registration and small query helpers. New resolution steps are appended at
|
||||
the end of ``arg_groups.pipeline.run_resolution_pipeline``, immediately
|
||||
marked complete, unless an earlier dependency is documented explicitly.
|
||||
5. Module-level ``ServerArgs`` construction/runtime shims.
|
||||
6. Networking constants and ``PortArgs``.
|
||||
|
||||
Model- or vendor-specific utilities belong in ``sglang.srt.arg_groups`` (or
|
||||
their owning subsystem), not before ``ServerArgs`` in this module.
|
||||
"""
|
||||
"""Server argument construction, resolution, CLI registration, and network ports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -96,16 +78,11 @@ def _real_kv_hash_modes():
|
||||
return list(RealKvHashMode)
|
||||
|
||||
|
||||
# Re-exported. These were importable from this module while the field
|
||||
# declarations that used them lived here; the declarations moved to
|
||||
# `arg_groups/fields/` but out-of-tree code -- and `tokenizer_control_mixin`
|
||||
# for `LoRARef` -- still reaches them through `sglang.srt.server_args`.
|
||||
# Compatibility re-exports for callers importing through server_args.
|
||||
from sglang.srt.arg_groups.arg_utils import NS, A, Arg # noqa: F401
|
||||
from sglang.srt.arg_groups.argparse_actions import LoRAPathAction # noqa: F401
|
||||
|
||||
# Re-exported for out-of-tree plugins, which have always reached these
|
||||
# through `sglang.srt.server_args`. The lists and their adders moved to
|
||||
# `arg_groups/choices.py` with the field declarations that name them.
|
||||
# Public choice lists and plugin registration helpers.
|
||||
from sglang.srt.arg_groups.choices import ( # noqa: F401
|
||||
ATTENTION_BACKEND_CHOICES,
|
||||
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS,
|
||||
@@ -219,10 +196,7 @@ def __getattr__(name: str) -> Any:
|
||||
|
||||
|
||||
def _plain(value: Any) -> Any:
|
||||
"""``asdict``'s conversion, applied to one value: a record -- Struct or
|
||||
dataclass, since nested config values are both -- becomes a dict, containers
|
||||
recurse, everything else is deep-copied (a caller mutating the dump must not
|
||||
reach the live configuration)."""
|
||||
"""Convert nested Structs and dataclasses to dicts, copying all values."""
|
||||
if not isinstance(value, type) and is_record(value):
|
||||
return {
|
||||
field.name: _plain(getattr(value, field.name))
|
||||
@@ -238,74 +212,23 @@ def _plain(value: Any) -> Any:
|
||||
|
||||
|
||||
class ServerArgs:
|
||||
"""Server-wide configuration for SGLang.
|
||||
"""Raw server configuration, sealed when resolution starts.
|
||||
|
||||
Adding new arguments
|
||||
--------------------
|
||||
1. **Place the field in the right section.** Arguments are grouped by
|
||||
comment blocks (``# Model and tokenizer``, ``# LoRA``, etc.).
|
||||
Add new fields to the matching section, or create a new section
|
||||
with a ``# ---`` banner when none fits.
|
||||
|
||||
2. **Use the ``A[T, ...]`` annotation.** ``A`` is an alias for
|
||||
``typing.Annotated``. The primary CLI flag is auto-derived from the
|
||||
field name (``tp_size`` → ``--tp-size``). Use ``aliases`` for
|
||||
longer alternate names
|
||||
(``aliases=["--tensor-parallel-size"]``)::
|
||||
|
||||
# Bare string — simplest form (just help text):
|
||||
host: A[str, "The host of the HTTP server."] = "127.0.0.1"
|
||||
trust_remote_code: A[bool, "Whether to allow custom models."] = False
|
||||
|
||||
# Arg(...) — when you need choices, aliases, type_parser, etc.:
|
||||
load_format: A[str, Arg(help="...", choices=CHOICES)] = "auto"
|
||||
model_path: A[str, Arg(help="...", aliases=["--model"])]
|
||||
|
||||
See ``Arg`` in ``arg_groups/arg_utils.py`` for the full list of
|
||||
supported metadata (``choices``, ``aliases``, ``type_parser``,
|
||||
``nargs``, ``const``, ``action``, ``no_cli``, …).
|
||||
|
||||
3. **Manual entries in ``add_cli_args`` — only for special cases.**
|
||||
A few arguments cannot use the annotation style and must be
|
||||
registered manually in ``add_cli_args``:
|
||||
|
||||
- **Deprecated flags** that redirect to another field via one of the
|
||||
``Deprecated*Action`` classes in ``arg_groups/argparse_actions.py``
|
||||
(that module's header says which shape fits which migration).
|
||||
- **Dynamic choices** computed at runtime (e.g. ``reasoning_parser``
|
||||
whose choices come from a plugin registry).
|
||||
- The ``--config`` meta-argument (not a dataclass field).
|
||||
|
||||
Everything else should use the ``A[T, ...]`` annotation.
|
||||
|
||||
The fields live in ``arg_groups/fields/`` -- one class per config
|
||||
namespace, assembled here rather than inherited, so what the record holds
|
||||
is one readable call and not a property of a base-class list. The order
|
||||
they appear in is ``POSITIONAL_FIELD_ORDER``, not the order the namespaces
|
||||
are listed in: field order *is* the positional constructor signature.
|
||||
Add fields to the matching namespace in ``arg_groups/fields/`` using
|
||||
``A[T, help]`` or ``A[T, Arg(...)]``; see ``arg_utils.Arg`` for CLI metadata.
|
||||
Only dynamic choices, deprecated flags, and ``--config`` need manual
|
||||
registration in ``add_cli_args``. ``POSITIONAL_FIELD_ORDER`` preserves
|
||||
the positional constructor signature when these namespaces are assembled.
|
||||
"""
|
||||
|
||||
def __post_init__(self):
|
||||
"""Construction leaves the record at what the caller asked for.
|
||||
|
||||
Resolution is a separate act, entered through ``resolve_once``: the
|
||||
launcher runs it once per engine, and every publishing process asks the
|
||||
gate on the way in. A record that is only constructed -- a fixture, a
|
||||
config being inspected, one being handed to a subprocess that will
|
||||
resolve it itself -- stays raw.
|
||||
"""
|
||||
"""Leave construction unresolved; launchers and publishers call ``resolve_once``."""
|
||||
|
||||
def resolve_once(self) -> None:
|
||||
"""Run the resolution pipeline, unless this record has been through it.
|
||||
"""Resolve once, preserving declarations across pickling to child processes.
|
||||
|
||||
Resolution is a deterministic function of the raw inputs -- two records
|
||||
built from the same arguments declare the same things -- but the
|
||||
handlers do not survive a second pass over their own output: DP
|
||||
attention halves ``chunked_prefill_size`` again on every re-entry.
|
||||
|
||||
The publishing entry of every process calls this. In a child the record
|
||||
arrived by pickle and brought its declarations along, so the child has
|
||||
nothing left to derive and projects what the parent decided.
|
||||
Handlers are not idempotent over their own output. A failed resolution
|
||||
cannot be retried on the same record.
|
||||
"""
|
||||
if getattr(self, "_resolution_finished", False):
|
||||
return
|
||||
@@ -318,63 +241,30 @@ class ServerArgs:
|
||||
)
|
||||
from sglang.srt.arg_groups.pipeline import run_resolution_pipeline
|
||||
|
||||
# Sealed for the duration, not just afterwards: everything below this
|
||||
# line reads the input and declares against it. No exceptions -- even a
|
||||
# resolver from outside this tree assigns onto a stand-in, not here.
|
||||
self._input_frozen = True
|
||||
try:
|
||||
run_resolution_pipeline(self)
|
||||
except BaseException:
|
||||
# The handlers that ran already declared, and they are not
|
||||
# idempotent over their own output.
|
||||
self._resolution_failed = True
|
||||
raise
|
||||
finally:
|
||||
self._input_frozen = False
|
||||
# Set here too, because the dummy/absent-model path returns before the
|
||||
# end of the pipeline that normally sets it: the gate is about whether
|
||||
# the handlers ran, not how far they got.
|
||||
# Also mark the dummy/absent-model path, which returns early from the pipeline.
|
||||
self._resolution_finished = True
|
||||
|
||||
@property
|
||||
def launch_command(self) -> str | None:
|
||||
"""How this record was created, verbatim.
|
||||
|
||||
`resolved_dict` answers with what resolution decided; this answers with
|
||||
what the operator asked for, which is a different question and the one
|
||||
"why is this server configured like this?" usually means. The two are
|
||||
not derivable from each other: a field the operator never set reads the
|
||||
same as one they set to the value resolution would have picked anyway.
|
||||
|
||||
The launcher stores the arguments it parsed; the in-process `Engine`
|
||||
stores the call that built the record, since there was no command line.
|
||||
`None` on a record built directly (a fixture, a subprocess copy that
|
||||
predates this, a config being inspected).
|
||||
"""
|
||||
"""Original CLI arguments or Engine constructor call; ``None`` for direct construction."""
|
||||
return getattr(self, "_launch_command", None)
|
||||
|
||||
def resolved_dict(self) -> dict[str, Any]:
|
||||
"""This configuration as a plain dict of resolved field values.
|
||||
|
||||
What the whole-object readbacks report (`/server_info` and its gRPC and
|
||||
in-process twins). A plain `asdict` reads the fields, which
|
||||
carry the raw input; this reads the declarations, so it answers with what
|
||||
resolution decided. Nested dataclass fields are expanded
|
||||
the way `asdict` expands them; the private resolution bookkeeping and the
|
||||
`model_config` memo are not fields and do not appear.
|
||||
"""
|
||||
"""Serialize resolved field values, expanding nested records and excluding bookkeeping."""
|
||||
|
||||
return {
|
||||
field.name: _plain(resolution_result(self, field.name))
|
||||
for field in record_fields(type(self))
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# CUDA graph configuration resolution
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
# ===== END TO BE REFACTORED ====
|
||||
|
||||
LANGUAGE_MODEL_ONLY_ARCHITECTURES = (
|
||||
"MuseGlimmerForConditionalGeneration",
|
||||
"Cosmos3ForConditionalGeneration",
|
||||
@@ -392,7 +282,6 @@ class ServerArgs:
|
||||
@staticmethod
|
||||
def add_cli_args(parser: argparse.ArgumentParser):
|
||||
|
||||
# Auto-derived from Annotated[..., Arg(...)] field metadata.
|
||||
add_cli_args_from_dataclass(parser, ServerArgs)
|
||||
|
||||
# --- Fields with dynamic choices (computed at add_cli_args time) ---
|
||||
@@ -493,17 +382,7 @@ class ServerArgs:
|
||||
return self.url(port=self.engine_info_bootstrap_port)
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
# The record holds the operator's input. It is writable while the
|
||||
# caller is still assembling it and sealed from the moment resolution
|
||||
# starts: a resolver that writes a field would overwrite the very thing
|
||||
# the record exists to remember, and the decision it meant to record
|
||||
# belongs in the stash, where it carries a source and does not destroy
|
||||
# the input it was derived from.
|
||||
# Underscore names are mostly the record's own bookkeeping --
|
||||
# `_input_frozen`, `_raw_input`, `_resolved_overrides`, the memo slots
|
||||
# -- which resolution writes on purpose. A *field* spelled that way is
|
||||
# still configuration, so the test cannot be on spelling alone or that
|
||||
# one leaf stays writable on a read-only record.
|
||||
# Seal configuration fields, including underscore-prefixed ones, but allow bookkeeping.
|
||||
if not name.startswith("_") or name in _underscore_field_names():
|
||||
if getattr(self, "_input_frozen", False):
|
||||
raise AttributeError(
|
||||
@@ -527,14 +406,9 @@ class ServerArgs:
|
||||
msgspec.Struct.__setattr__(self, name, value)
|
||||
|
||||
def __reduce__(self):
|
||||
"""Pickle the record *and* what resolution left on it.
|
||||
"""Preserve resolution bookkeeping as well as Struct fields when pickling.
|
||||
|
||||
A Struct pickles its fields; everything else lives in the `dict=True`
|
||||
namespace and would be dropped, which for this record means the input
|
||||
snapshot, the declaration stash and the resolution flags -- the whole
|
||||
reason a child can publish what its parent decided without resolving
|
||||
again. Reconstruction restores the fields first and the bookkeeping
|
||||
after, so the seal is re-armed only once the fields are in place.
|
||||
Restore fields before bookkeeping so the write seal is re-armed last.
|
||||
"""
|
||||
return (
|
||||
_rebuild_server_args,
|
||||
@@ -552,12 +426,7 @@ class ServerArgs:
|
||||
return remote_instance_transfer_engine_of(resolving_view(self), load_format)
|
||||
|
||||
|
||||
# The namespaces whose *input* fields make up the record. A namespace declares
|
||||
# its input and derived fields side by side; only the input half is collected,
|
||||
# which is why this is a call and not a base-class list.
|
||||
#
|
||||
# A set, not an order: `collect_input_fields` orders by `field_order.py`, which
|
||||
# is what keeps the positional constructor stable.
|
||||
# Collect input fields only; field_order.py preserves positional argument order.
|
||||
|
||||
|
||||
_INPUT_NAMESPACES = [
|
||||
@@ -586,16 +455,9 @@ _INPUT_NAMESPACES = [
|
||||
|
||||
_annotations, _defaults, _namespaces = collect_input_fields(_INPUT_NAMESPACES)
|
||||
ServerArgs.__annotations__ = {**_annotations, **ServerArgs.__annotations__}
|
||||
# The assembled record has no base classes, so it carries the map the classes
|
||||
# used to answer through their `_NS_PATH`.
|
||||
ServerArgs._NS_BY_FIELD = _namespaces
|
||||
# The classes themselves, so the bag projection can find the declarations
|
||||
# that are not fields -- the derived half of each namespace.
|
||||
ServerArgs._NAMESPACES = _INPUT_NAMESPACES
|
||||
# `dict=True` so the record can carry what is not configuration -- the input
|
||||
# snapshot, the declaration stash, the resolution flags, the memo slots. A
|
||||
# Struct has no `__dict__` without it, and those are exactly the underscore
|
||||
# names `_underscore_field_names()` is careful *not* to include.
|
||||
# dict=True retains resolution bookkeeping and memoized values outside the fields.
|
||||
ServerArgs = msgspec.defstruct(
|
||||
"ServerArgs",
|
||||
[
|
||||
@@ -661,14 +523,7 @@ def m3_fp8_attn_gemm_enabled(args) -> bool:
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _underscore_field_names() -> frozenset:
|
||||
"""Real dataclass fields whose names start with an underscore.
|
||||
|
||||
The read-only guard exempts underscore names because they are the record's
|
||||
own bookkeeping (the stash, the flags, the cache keys). A *field* that
|
||||
happens to start with an underscore is still resolved configuration --
|
||||
`_speculative_draft_quantization_explicitly_set` is one -- and exempting it
|
||||
by spelling would leave exactly one leaf writable on a read-only record.
|
||||
"""
|
||||
"""Configuration fields that must stay sealed despite their underscore prefix."""
|
||||
return frozenset(
|
||||
field.name for field in record_fields(ServerArgs) if field.name.startswith("_")
|
||||
)
|
||||
@@ -695,15 +550,7 @@ def set_global_server_args_for_tokenizer(server_args: ServerArgs):
|
||||
|
||||
|
||||
def get_global_server_args() -> NoReturn:
|
||||
"""Retired. It raises, because what it used to return is the problem: the
|
||||
record answers with the operator's *input*, so a caller reading a field
|
||||
resolution decided got a stale value and no error.
|
||||
|
||||
The name survives so that a caller importing it from this module lands on
|
||||
a message instead of an ImportError. Annotated ``NoReturn`` so a type
|
||||
checker rejects the call rather than accepting the attribute access after
|
||||
it. The message lives once, in the exception.
|
||||
"""
|
||||
"""Retired accessor retained to raise a migration error for existing imports."""
|
||||
raise RuntimeError(
|
||||
"get_global_server_args() is retired. Read the value that is in effect "
|
||||
"from its namespace bag -- `get_exec().kernel.attention_backend`, "
|
||||
@@ -721,13 +568,7 @@ def _rebuild_server_args(cls, fields, bookkeeping):
|
||||
|
||||
|
||||
def _declared_default(name: str):
|
||||
"""The declared default of a field, for a manual `add_argument`.
|
||||
|
||||
`ServerArgs.<field>` used to answer with it. The record is a Struct now, so
|
||||
that expression returns the slot descriptor instead -- which argparse
|
||||
happily stores as the default, and the first reader gets a
|
||||
`member_descriptor` where it expected a string.
|
||||
"""
|
||||
"""Return a field default; Struct class attributes are slot descriptors."""
|
||||
return next(
|
||||
field.default
|
||||
for field in msgspec.structs.fields(ServerArgs)
|
||||
|
||||
Reference in New Issue
Block a user