config: ServerArgs holds the raw input (#36255)
This commit is contained in:
@@ -1005,7 +1005,7 @@ def main(server_args, bench_args):
|
||||
# The decode phase has to capture the batch sizes this run benchmarks, and
|
||||
# the per-phase convenience knob loses to an explicit --cuda-graph-config
|
||||
# JSON (resolution applies that last), so the size is merged into that JSON.
|
||||
if getattr(server_args, "_declarations_materialized", False):
|
||||
if getattr(server_args, "_resolution_finished", False):
|
||||
# A record the caller already resolved: nothing will parse a raw dict
|
||||
# again, so the declaration has to be the finished typed config.
|
||||
merged = resolution_result(server_args, "cuda_graph_config")
|
||||
|
||||
@@ -75,10 +75,10 @@ class Arg:
|
||||
# When True, this field is skipped by add_cli_args_from_dataclass.
|
||||
# Use for fields that have no CLI surface (e.g. injected via Python only).
|
||||
no_cli: bool = False
|
||||
# When True, this field may be written by config resolution (model
|
||||
# overrides and post-process passes): it is part of the whitelist accepted
|
||||
# by the declaration stash, and its resolved value materializes onto the
|
||||
# field at the end of __post_init__.
|
||||
# When True, config resolution (model overrides and post-process passes)
|
||||
# may decide this field: the declaration stash accepts the name, and
|
||||
# `resolution_result` and the config bags answer with the decision. The
|
||||
# field keeps what the operator passed.
|
||||
resolvable: bool = False
|
||||
|
||||
|
||||
|
||||
@@ -14,9 +14,13 @@
|
||||
"""Declarative model-override registry.
|
||||
|
||||
Model-identity adjustments to the server configuration are DECLARED here and
|
||||
materialized onto ``server_args`` at the end of ``__post_init__`` (gate
|
||||
order, last writer wins) — model code never mutates ``ServerArgs`` fields
|
||||
imperatively.
|
||||
appended to the record's declaration stash (gate order, last writer wins).
|
||||
Nothing here writes back onto ``ServerArgs``: the record holds the user's raw
|
||||
input, and a decision is read through ``resolution_result`` or the published
|
||||
config bags — model code never mutates ``ServerArgs`` fields imperatively. The
|
||||
one channel that still leaves a field changed is ``declare_direct_writes``,
|
||||
which does not perform the write: it captures one an out-of-tree plugin already
|
||||
made, and undoing it would surprise the plugin's own reads.
|
||||
|
||||
Two declaration forms, keyed on ``hf_config.architectures[0]``:
|
||||
|
||||
@@ -206,10 +210,8 @@ def register_post_process(fn: Callable[..., dict]) -> Callable[..., dict]:
|
||||
def _declaration_overlay(server_args: Any) -> Dict[str, Any]:
|
||||
"""What the declarations say so far, last writer wins.
|
||||
|
||||
Passes declare without touching the fields until
|
||||
``materialize_declarations``, so a mid-resolution reader needs this to see
|
||||
them; handlers and hooks write as they declare, and for those the overlay
|
||||
repeats what the field already holds."""
|
||||
Nothing writes the fields, so a mid-resolution reader needs this to see a
|
||||
decision at all; the fields keep what the caller supplied."""
|
||||
overlay: Dict[str, Any] = {}
|
||||
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
|
||||
overlay.update(declared)
|
||||
@@ -222,9 +224,9 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
Evaluates the pass on the resolving state (a read-only view with the
|
||||
accumulated declarations overlaid from the stash) and appends its
|
||||
declaration to the stash. During ``__post_init__`` the fields stay
|
||||
untouched — ``materialize_declarations`` applies the whole stash once at
|
||||
the end of resolution; a pass invoked after materialization (a post-init
|
||||
slot) writes through immediately.
|
||||
untouched: the stash is what the config bags are projected from. A pass
|
||||
invoked after resolution finished (a post-init slot) writes through
|
||||
immediately, because there is no later projection to pick it up.
|
||||
"""
|
||||
declared = fn(ResolvedView(server_args, overlay=_declaration_overlay(server_args)))
|
||||
if not isinstance(declared, dict):
|
||||
@@ -244,7 +246,7 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
stash = server_args._resolved_overrides = []
|
||||
stash.append(entry)
|
||||
validate_declarations(server_args, [entry])
|
||||
if getattr(server_args, "_declarations_materialized", False):
|
||||
if getattr(server_args, "_resolution_finished", False):
|
||||
_apply_fields(server_args, declared)
|
||||
|
||||
|
||||
@@ -260,29 +262,17 @@ def _apply_fields(server_args: Any, fields: Dict[str, Any]) -> None:
|
||||
|
||||
|
||||
def declare_resolution(server_args: Any, source: str, **fields: Any) -> None:
|
||||
"""Record a resolution write in the declaration stash, and apply it now.
|
||||
"""Record a resolution write in the declaration stash.
|
||||
|
||||
The stash is what the projection reads, so a resolver that only assigns
|
||||
the field leaves that write invisible to it. The immediate write keeps the
|
||||
resolver's successors seeing the value where they read the field directly.
|
||||
The stash *is* the resolution result: the bags are projected from it,
|
||||
`resolution_result` answers from it, and no field is written. A resolver
|
||||
reading a field another resolver may have decided must read `resolving_view`
|
||||
(or `ServerArgs._resolved()`), which
|
||||
`test_resolution_reads_the_declarations` pins.
|
||||
|
||||
What it does change is which writer wins. A declaration is appended and
|
||||
replayed last, so a resolver that declares a field a *deferred* writer (a
|
||||
post-process pass, a registry entry) also decides now beats it, where its
|
||||
bare assignment used to be overwritten by that writer's declaration. A
|
||||
resolver that gates on such a field has to read the resolving view rather
|
||||
than the raw field, or it decides from a value that is already stale.
|
||||
|
||||
For resolvers inside ``__post_init__``: the handlers on ``ServerArgs``
|
||||
(through ``self._declare``) and the ``arg_groups`` hooks and hardware
|
||||
defaults they call. Resolution that has to wait for the launcher stage
|
||||
goes through ``declare_late_resolution`` instead.
|
||||
|
||||
Names arrive as keyword arguments, which accept anything; a misspelled one
|
||||
would otherwise become a new attribute that nothing ever reads, so it is
|
||||
rejected here. This is not the model-override whitelist: that one limits
|
||||
which fields a *registry entry* may reach, while a resolver writing the
|
||||
field it owns is the pipeline resolving by construction.
|
||||
For resolvers inside ``__post_init__``; launcher-stage resolution goes
|
||||
through ``declare_late_resolution``. A name that is not a field is rejected
|
||||
here rather than becoming an attribute nothing reads.
|
||||
"""
|
||||
if dataclasses.is_dataclass(type(server_args)):
|
||||
unknown = sorted(set(fields) - field_names(type(server_args)))
|
||||
@@ -293,8 +283,6 @@ def declare_resolution(server_args: Any, source: str, **fields: Any) -> None:
|
||||
stash = []
|
||||
object.__setattr__(server_args, "_resolved_overrides", stash)
|
||||
stash.append((source, dict(fields)))
|
||||
for name, value in fields.items():
|
||||
setattr(server_args, name, value)
|
||||
|
||||
|
||||
def declare_late_resolution(server_args: Any, source: str, **fields: Any) -> None:
|
||||
@@ -304,10 +292,10 @@ def declare_late_resolution(server_args: Any, source: str, **fields: Any) -> Non
|
||||
normalization and the auto-parser detection need the launcher's validation
|
||||
stage (and, for the parsers, a tokenizer / chat-template load). They still
|
||||
belong to the resolution pipeline — they decide what the process will run
|
||||
with — so they write the fields in place, before anything publishes the
|
||||
object. Writing in place is the point: every holder of that instance (the
|
||||
HTTP server, the multi-tokenizer workers it serializes for, the schedulers
|
||||
it forks) must see the resolved value.
|
||||
with — so their decision goes to the stash like any other, and the record
|
||||
keeps what the caller passed. Every holder of that instance reads the
|
||||
decision the same way the rest of the pipeline does: the bags it publishes,
|
||||
or ``resolution_result``, both of which survive the pickle to a child.
|
||||
|
||||
Refuses to touch the published instance: after publish the bags exist and a
|
||||
field write would desync them, which is what ``get_context().override`` is
|
||||
@@ -334,7 +322,6 @@ def declare_late_resolution(server_args: Any, source: str, **fields: Any) -> Non
|
||||
stash = []
|
||||
object.__setattr__(server_args, "_resolved_overrides", stash)
|
||||
stash.append((source, dict(fields)))
|
||||
_apply_fields(server_args, fields)
|
||||
|
||||
|
||||
def declare_direct_writes(
|
||||
@@ -348,7 +335,10 @@ def declare_direct_writes(
|
||||
Out-of-tree platform plugins are handed the record and set fields on it.
|
||||
Their implementations live outside this tree, so they cannot be converted
|
||||
by editing the resolver; and the raw snapshot is taken before the pipeline
|
||||
starts, so a plugin's default is neither declared nor raw.
|
||||
starts, so a plugin's default is neither declared nor raw. The write itself
|
||||
stays: this captures it into the stash so the projection and the bags carry
|
||||
it, but reverting the field would break the plugin's own reads of what it
|
||||
just set. It is the only field a record still carries from resolution.
|
||||
|
||||
Rebinding is what the diff sees, and rebinding is all it needs to see: a
|
||||
plugin that mutates a value in place reaches the projection anyway, because
|
||||
@@ -384,24 +374,12 @@ def declare_direct_writes(
|
||||
return result
|
||||
|
||||
|
||||
def materialize_declarations(server_args: Any) -> None:
|
||||
"""Apply the accumulated declarations onto ``server_args`` once, at the
|
||||
end of ``__post_init__`` (gate order: last writer wins). After this the
|
||||
fields carry the resolved configuration — every post-init reader, in any
|
||||
process, reads them directly; ``resolved_view`` remains an internal
|
||||
helper for mid-resolution code only."""
|
||||
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
|
||||
for field, value in declared.items():
|
||||
setattr(server_args, field, value)
|
||||
server_args._declarations_materialized = True
|
||||
|
||||
|
||||
def resolution_result(server_args: Any, field: str, default: Any = None) -> Any:
|
||||
"""What resolution decided for ``field``: the declaration if there is one,
|
||||
otherwise what the caller supplied.
|
||||
|
||||
This is what the config projection reads. Reading the field instead would
|
||||
work only for as long as declarations materialize onto the record -- and
|
||||
work whatever the caller passed onto the record -- and
|
||||
the point of declaring is that they will not, so the projection must not
|
||||
depend on it. A config that never ran the pipeline (a mock, a partial
|
||||
fixture) carries no raw snapshot; its fields are all it has.
|
||||
@@ -422,9 +400,8 @@ def resolution_projection(server_args: Any) -> Dict[str, Any]:
|
||||
|
||||
The whole-object shape of ``resolution_result``, for the exits that hand out
|
||||
the entire configuration (``/server_info``, the gRPC and engine readbacks).
|
||||
They used ``dataclasses.asdict``, which reads the fields -- correct only for
|
||||
as long as declarations materialize onto the record, and the point of
|
||||
declaring is that they will not. Field values only: the private resolution
|
||||
They used ``dataclasses.asdict``, which reads the fields -- the operator's
|
||||
input, not what resolution decided. Field values only: the private resolution
|
||||
bookkeeping and the ``model_config`` memo that a ``vars()`` dump carried into
|
||||
the readback are not configuration.
|
||||
"""
|
||||
@@ -453,10 +430,14 @@ def _plain(value: Any) -> Any:
|
||||
|
||||
|
||||
def resolved_view(server_args: Any) -> ResolvedView:
|
||||
"""Read-only view of the resolving configuration for mid-resolution code
|
||||
that is not a pass (``__post_init__`` handlers and hooks). Internal to
|
||||
the resolution pipeline: after ``materialize_declarations`` runs, the
|
||||
fields themselves carry the resolved values — read them directly."""
|
||||
"""Read-only view of the resolving configuration: the declarations
|
||||
overlaid on the fields, snapshotted per call.
|
||||
|
||||
For mid-resolution code that is not a pass (``__post_init__`` handlers and
|
||||
hooks), and for the record's own members that must answer with what
|
||||
resolution decided -- a declaration-only resolver (a model-specific
|
||||
override, a registry entry) never writes the field, so a field read there
|
||||
answers with the raw input."""
|
||||
return ResolvedView(server_args, overlay=_declaration_overlay(server_args))
|
||||
|
||||
|
||||
|
||||
@@ -251,7 +251,7 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
kwargs["log_level"] = "error"
|
||||
server_args = self.server_args_class(**kwargs)
|
||||
self.server_args = server_args
|
||||
logger.info(f"{server_args=}")
|
||||
logger.info(f"server_args={server_args.resolved_dict()}")
|
||||
|
||||
# Rust Server is not supported with the offline Engine API
|
||||
if envs.SGLANG_RUST_SERVER.get():
|
||||
@@ -1076,7 +1076,7 @@ class Engine(EngineScoreMixin, EngineBase):
|
||||
# Allocate ports for inter-process communications
|
||||
if port_args is None:
|
||||
port_args = PortArgs.init_new(server_args)
|
||||
logger.info(f"{server_args=}")
|
||||
logger.info(f"server_args={server_args.resolved_dict()}")
|
||||
|
||||
# Start the engine info bootstrap server if per-rank info is needed.
|
||||
engine_info_bootstrap_server = None
|
||||
|
||||
@@ -803,8 +803,8 @@ async def get_server_info():
|
||||
async def server_info():
|
||||
"""The startup configuration, plus live scheduler state.
|
||||
|
||||
The `ServerArgs` fields here are the record: what the launcher was given,
|
||||
with resolution written back into it. Fields the control plane changes
|
||||
The values here are the resolution result: what the launcher was given,
|
||||
with every decision resolution made applied over it. Fields the control plane changes
|
||||
after publication -- the model a weight update swapped in, its load format,
|
||||
an operator-set weight version -- are reported by `/model_info`, and the
|
||||
HiCache mirror by `GET /hicache/storage-backend`.
|
||||
|
||||
@@ -247,11 +247,10 @@ class MiniMaxSparseAttnBackend(AttentionBackend):
|
||||
Phase,
|
||||
check_cuda_graph_backend,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
|
||||
_sa = getattr(runner, "server_args", None)
|
||||
self.speculative_num_draft_tokens = getattr(
|
||||
_sa, "speculative_num_draft_tokens", None
|
||||
)
|
||||
spec = get_spec()
|
||||
self.speculative_num_draft_tokens = spec.speculative_num_draft_tokens
|
||||
_decode_cuda_graph = not check_cuda_graph_backend(
|
||||
Phase.DECODE, Backend.DISABLED
|
||||
)
|
||||
@@ -264,7 +263,7 @@ class MiniMaxSparseAttnBackend(AttentionBackend):
|
||||
if (
|
||||
self.use_msa
|
||||
and _decode_cuda_graph
|
||||
and getattr(_sa, "speculative_algorithm", None) is not None
|
||||
and spec.speculative_algorithm is not None
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"MiniMax-M3 MSA attention does not support speculative decoding under "
|
||||
|
||||
@@ -315,8 +315,8 @@ def initialize_moe_config():
|
||||
"""Seed the MoE runtime flags from the published configuration.
|
||||
|
||||
Reads the bags: `moe_a2a_backend` and its siblings are resolution's
|
||||
answers, and the record carries them only while declarations materialize
|
||||
onto it. Called once per process after publish
|
||||
answers, and the record carries the operator's input. Called once per
|
||||
process after publish
|
||||
(scheduler init, the benchmark work functions).
|
||||
"""
|
||||
exec_moe = get_exec().moe
|
||||
|
||||
@@ -39,7 +39,6 @@ from sglang.srt.runtime_context import (
|
||||
get_observability,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_server_args,
|
||||
get_serving,
|
||||
get_spec,
|
||||
)
|
||||
@@ -4412,7 +4411,7 @@ class Scheduler(
|
||||
# Resolved config (pristine server_args + post-publish overrides) so a
|
||||
# readback reflects values changed via /set_internal_state, not startup.
|
||||
ret = get_context().resolved_server_args_dict()
|
||||
ret["world_size"] = compute_world_size(get_server_args())
|
||||
ret["world_size"] = compute_world_size(get_parallel().config)
|
||||
ret["last_gen_throughput"] = self.metrics_reporter.last_gen_throughput
|
||||
draft_graph_memory_usage = (
|
||||
None if self.draft_worker is None else self.draft_worker.graph_memory_usage
|
||||
|
||||
@@ -1380,7 +1380,7 @@ class KVCacheConfigurator:
|
||||
"""
|
||||
full_pool_class = DSATokenToKVPool if is_dsa_model else MLATokenToKVPool
|
||||
common = {
|
||||
"page_size": self.server_args.page_size,
|
||||
"page_size": get_schedule().page_size,
|
||||
"device": self.device,
|
||||
"enable_memory_saver": False,
|
||||
}
|
||||
@@ -1401,7 +1401,7 @@ class KVCacheConfigurator:
|
||||
return SWAKVPool(
|
||||
size=full_max_total_num_tokens,
|
||||
size_swa=swa_max_total_num_tokens,
|
||||
page_size=self.server_args.page_size,
|
||||
page_size=get_schedule().page_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
head_num=0,
|
||||
head_dim=0,
|
||||
|
||||
@@ -702,12 +702,13 @@ def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str,
|
||||
|
||||
def resolve_auto_parsers(server_args) -> None:
|
||||
"""Resolve ``--reasoning-parser=auto`` / ``--tool-call-parser=auto`` from the
|
||||
chat template, in place, before anything publishes ``server_args``.
|
||||
chat template, before anything publishes ``server_args``.
|
||||
|
||||
Performs a lightweight tokenizer load, so it runs once in engine init. In
|
||||
place because everyone who holds this instance must see the resolved value:
|
||||
the schedulers it forks, the HTTP server, and the tokenizer workers it is
|
||||
serialized for.
|
||||
Performs a lightweight tokenizer load, so it runs once in engine init. The
|
||||
decision goes to this instance's declaration stash, so every holder of it
|
||||
carries it -- the schedulers it forks, the HTTP server, the tokenizer
|
||||
workers it is serialized for -- and each publishes bags projected from it.
|
||||
The fields stay what the operator passed.
|
||||
"""
|
||||
cfg = resolving_view(server_args)
|
||||
needs = tuple(
|
||||
|
||||
@@ -35,7 +35,7 @@ from sglang.srt.ray.scheduler_actor import SchedulerActor
|
||||
from sglang.srt.runtime_context import (
|
||||
get_parallel,
|
||||
)
|
||||
from sglang.srt.server_args import PortArgs, ServerArgs
|
||||
from sglang.srt.server_args import PortArgs, ServerArgs, compute_world_size
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -108,14 +108,10 @@ def _get_bundle_node_ip(placement_group: PlacementGroup, bundle_idx: int) -> str
|
||||
def _compute_world_size() -> int:
|
||||
"""Compute world_size (total number of scheduler actors/GPUs needed).
|
||||
|
||||
Normal: dp_size * tp_size * pp_size; DP attention: tp_size * pp_size.
|
||||
Reads the published parallel leaves: the driver is sizing the actors that
|
||||
will hold the process groups, so there is nothing live to ask.
|
||||
"""
|
||||
parallel = get_parallel().config
|
||||
if parallel.enable_dp_attention:
|
||||
return parallel.tp_size * parallel.pp_size
|
||||
return parallel.dp_size * parallel.tp_size * parallel.pp_size
|
||||
return compute_world_size(get_parallel().config)
|
||||
|
||||
|
||||
def _resolve_bundle_indices(pg: PlacementGroup, world_size: int) -> List[int]:
|
||||
|
||||
@@ -24,9 +24,10 @@ getters. The resolved parallel **configuration** is the same object's ``config``
|
||||
hop (``get_parallel().config.tp_size``), which reads the published ``parallel``
|
||||
bag: bare is the live group, ``config`` is what was configured.
|
||||
|
||||
``get_server_args()`` returns the process-wide ``ServerArgs``. This is the pristine / resolved-at-startup **read-only** record kept
|
||||
for debug and reproduction; business code reads resolved config from the
|
||||
namespace bags below, not from this object. The context owns the storage:
|
||||
``get_server_args()`` returns the process-wide ``ServerArgs``. This is the
|
||||
user's raw input, kept **read-only** for debug and reproduction; what
|
||||
resolution decided lives in the declarations (``resolution_result``) and, for
|
||||
business code, in the namespace bags below -- never on this object's fields. The context owns the storage:
|
||||
publishing goes through ``RuntimeContext.set_server_args`` (the legacy
|
||||
``set_global_server_args_for_scheduler`` / ``get_global_server_args`` are thin
|
||||
shims over this slot).
|
||||
@@ -440,8 +441,8 @@ class DpFlags(_FlagGroupBase):
|
||||
class Flags(_FlagGroupBase):
|
||||
"""Root of the runtime-flags tier.
|
||||
|
||||
Resolved configuration lives on ``server_args`` fields (materialized at
|
||||
the end of ``__post_init__``) — this tier only carries genuine runtime
|
||||
Resolved configuration lives in the config bags below (projected from the
|
||||
declarations at publish) — this tier only carries genuine runtime
|
||||
state whose value is not a function of the configuration alone, grouped
|
||||
by lifecycle (``capture``) or subsystem (``moe`` / ``dp``).
|
||||
"""
|
||||
@@ -700,8 +701,7 @@ def _build_config_bags(server_args: Any) -> dict:
|
||||
"""Snapshot the resolution result into the namespace bag tree, driven by
|
||||
the ``NS(...)`` metadata on the dataclass fields. Each leaf comes from
|
||||
``resolution_result`` -- the declaration if resolution made one, else what
|
||||
the caller supplied -- rather than from the field, which carries the same
|
||||
value only while declarations still materialize. Returns
|
||||
the caller supplied. Returns
|
||||
``{top_level_name: _ConfigBag}``, arbitrarily nested (``exec.moe.eplb.…``).
|
||||
Only dataclass fields carry ``NS`` markers, so derived properties/methods are
|
||||
naturally excluded (they stay on the bag). A name used as both a leaf and a
|
||||
@@ -783,7 +783,13 @@ class RuntimeContext:
|
||||
if stream is None:
|
||||
import torch
|
||||
|
||||
device = self._server_args.device if self._server_args else "cuda"
|
||||
from sglang.srt.arg_groups.overrides import resolution_result
|
||||
|
||||
device = (
|
||||
resolution_result(self._server_args, "device")
|
||||
if self._server_args
|
||||
else "cuda"
|
||||
)
|
||||
stream = torch.get_device_module(device).Stream()
|
||||
self.resources.streams[name] = stream
|
||||
return stream
|
||||
@@ -819,8 +825,8 @@ class RuntimeContext:
|
||||
Overwrite-allowed: a re-publish replaces the slot (test kits re-publish
|
||||
per test; production ordering discipline lives at the call-sites, e.g.
|
||||
the draft-worker guard in ``ModelRunner.__init__``). The published
|
||||
object already carries the resolved configuration (declarations
|
||||
materialize at the end of ``__post_init__``).
|
||||
object is the raw input; the resolution it carries is its declaration
|
||||
stash, which is what the bags are projected from.
|
||||
"""
|
||||
# Seed the capture tier for the new lifecycle (defaults for sentinel
|
||||
# and mock publishes, which carry no config).
|
||||
@@ -1078,10 +1084,11 @@ class _ServerArgsOverride:
|
||||
}
|
||||
if declared:
|
||||
declare_late_resolution(server_args, "override_server_args", **declared)
|
||||
_apply_fields(
|
||||
server_args,
|
||||
{name: value for name, value in self._fields.items() if name[0] == "_"},
|
||||
)
|
||||
# This hook stands in for a launch: the caller's values are both what
|
||||
# the operator passed and what resolution decided, so they go on the
|
||||
# record as well as into the stash. Production late resolution declares
|
||||
# only -- there the record stays the operator's input.
|
||||
_apply_fields(server_args, self._fields)
|
||||
ctx.set_server_args(server_args)
|
||||
self._installed = True
|
||||
return server_args
|
||||
|
||||
@@ -3709,7 +3709,7 @@ class ServerArgs:
|
||||
arrived by pickle and brought its declarations along, so the child has
|
||||
nothing left to derive and projects what the parent decided.
|
||||
"""
|
||||
if getattr(self, "_declarations_materialized", False):
|
||||
if getattr(self, "_resolution_finished", False):
|
||||
return
|
||||
if getattr(self, "_resolution_failed", False):
|
||||
raise RuntimeError(
|
||||
@@ -3721,23 +3721,22 @@ class ServerArgs:
|
||||
try:
|
||||
self._run_resolution_pipeline()
|
||||
except BaseException:
|
||||
# The handlers that ran already wrote to the record, and they are
|
||||
# not idempotent over their own output.
|
||||
# The handlers that ran already declared, and they are not
|
||||
# idempotent over their own output.
|
||||
object.__setattr__(self, "_resolution_failed", True)
|
||||
raise
|
||||
# Set here too, because the dummy/absent-model path returns before the
|
||||
# materialization that normally sets it: the gate is about whether the
|
||||
# handlers ran, not how far they got.
|
||||
self._declarations_materialized = True
|
||||
# end of the pipeline that normally sets it: the gate is about whether
|
||||
# the handlers ran, not how far they got.
|
||||
self._resolution_finished = True
|
||||
|
||||
def resolved_dict(self) -> Dict[str, Any]:
|
||||
"""This configuration as a plain dict of resolved field values.
|
||||
|
||||
What the whole-object readbacks report (`/server_info` and its gRPC and
|
||||
in-process twins). `dataclasses.asdict(self)` reads the fields, which
|
||||
carry resolution's result only while declarations materialize onto the
|
||||
record; this reads the declarations, so it keeps answering with what
|
||||
resolution decided once they stop. Nested dataclass fields are expanded
|
||||
carry the raw input; this reads the declarations, so it answers with what
|
||||
resolution decided. Nested dataclass fields are expanded
|
||||
the way `asdict` expands them; the private resolution bookkeeping and the
|
||||
`model_config` memo are not fields and do not appear.
|
||||
"""
|
||||
@@ -3750,12 +3749,11 @@ class ServerArgs:
|
||||
|
||||
`dataclasses.replace` builds a new instance, so the copy carries none of
|
||||
what makes a record resolved: no raw snapshot, no declarations, no
|
||||
materialization. The next publish therefore finds an unmaterialized
|
||||
record and runs the pipeline over values it already decided -- DP
|
||||
attention halves `chunked_prefill_size` a second time (8192 -> 4096 ->
|
||||
2048) and the schedule conservativeness is scaled again (0.3 -> 0.09).
|
||||
The Ray paths replace `dist_init_addr` on a resolved record, which is
|
||||
how they hit it.
|
||||
finished flag. The next publish therefore resolves it again, which
|
||||
drops every decision the stash held -- the late ones (the auto-detected
|
||||
parsers) and the direct ones alike -- and re-runs the device probes in
|
||||
whatever process opened the copy. The Ray paths replace
|
||||
`dist_init_addr` on a resolved record, which is how they reach this.
|
||||
|
||||
The change is appended to the stash rather than left on the field: the
|
||||
projection reads the raw snapshot plus the declarations, so a field the
|
||||
@@ -3770,7 +3768,7 @@ class ServerArgs:
|
||||
copy's deep structure in-process mutates the parent's too.
|
||||
"""
|
||||
replacement = dataclasses.replace(self, **changes)
|
||||
if not getattr(self, "_declarations_materialized", False):
|
||||
if not getattr(self, "_resolution_finished", False):
|
||||
# Not resolved yet: the copy goes through the gate itself.
|
||||
return replacement
|
||||
|
||||
@@ -3780,7 +3778,7 @@ class ServerArgs:
|
||||
# (the read-only guard refuses the write).
|
||||
field_names = {field.name for field in dataclasses.fields(self)}
|
||||
for name, value in vars(self).items():
|
||||
if name in field_names or name == "_declarations_materialized":
|
||||
if name in field_names or name == "_resolution_finished":
|
||||
continue
|
||||
if isinstance(value, (dict, list, set)):
|
||||
value = copy.copy(value)
|
||||
@@ -3791,7 +3789,7 @@ class ServerArgs:
|
||||
object.__setattr__(replacement, "_resolved_overrides", stash)
|
||||
if changes:
|
||||
stash.append((source, dict(changes)))
|
||||
object.__setattr__(replacement, "_declarations_materialized", True)
|
||||
object.__setattr__(replacement, "_resolution_finished", True)
|
||||
return replacement
|
||||
|
||||
def _declare(self, source: str, **fields: Any) -> None:
|
||||
@@ -4022,13 +4020,7 @@ class ServerArgs:
|
||||
# time; last declarations of the resolution, mirroring that order.
|
||||
self._handle_model_capability_adjustments()
|
||||
|
||||
# End of resolution: apply the accumulated declarations onto the
|
||||
# fields once (gate order). From here on server_args carries the
|
||||
# resolved configuration — post-init readers, in any process, read
|
||||
# the fields directly.
|
||||
from sglang.srt.arg_groups.overrides import materialize_declarations
|
||||
|
||||
materialize_declarations(self)
|
||||
self._resolution_finished = True
|
||||
|
||||
def _handle_return_hidden_states_mode(self):
|
||||
cfg = resolving_view(self)
|
||||
@@ -9794,22 +9786,22 @@ class ServerArgs:
|
||||
def _late_resolution(self, source: str, **fields) -> None:
|
||||
"""Resolve fields at the launcher's validation stage (pre-publish).
|
||||
|
||||
See ``arg_groups.overrides.declare_late_resolution``: in place, because
|
||||
every holder of this instance must see the resolved value, and refused
|
||||
outright once the config is published.
|
||||
See ``arg_groups.overrides.declare_late_resolution``: the decision goes
|
||||
to this instance's declaration stash, so every holder of it carries the
|
||||
decision and publishes bags that answer with it. Refused outright once
|
||||
the config is published.
|
||||
"""
|
||||
from sglang.srt.arg_groups.overrides import declare_late_resolution
|
||||
|
||||
declare_late_resolution(self, source, **fields)
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
# After materialization the fields are the resolved startup
|
||||
# configuration -- the pristine, READ-ONLY record that the config bags
|
||||
# were projected from. Resolved config changes go to the bags via
|
||||
# Once resolution has finished the record is the READ-ONLY raw input
|
||||
# the config bags were projected from. Resolved config changes go to the bags via
|
||||
# get_context().override(source, ...); a value one runner or worker
|
||||
# owns travels as a constructor argument to it.
|
||||
if (
|
||||
getattr(self, "_declarations_materialized", False)
|
||||
getattr(self, "_resolution_finished", False)
|
||||
and not getattr(self, "_internal_write", False)
|
||||
and name not in _CACHE_SLOTS
|
||||
and (not name.startswith("_") or name in _underscore_field_names())
|
||||
@@ -9832,7 +9824,13 @@ class ServerArgs:
|
||||
return attention_backends_of(resolved_view(self))
|
||||
|
||||
def get_attention_backends(self):
|
||||
return attention_backends_of(self)
|
||||
"""The (prefill, decode) pair resolution decided.
|
||||
|
||||
Reads through the declaration stash, not the fields: the model-specific
|
||||
overrides declare into the stash without writing the fields, so a field
|
||||
read answers with what the operator typed.
|
||||
"""
|
||||
return attention_backends_of(resolved_view(self))
|
||||
|
||||
def use_mla_backend(self):
|
||||
from sglang.srt.configs.model_config import AttentionArch
|
||||
@@ -9884,7 +9882,7 @@ class ServerArgs:
|
||||
# state needs steps + 1 draft-token slots. Revisit this if topk>1
|
||||
# is supported.
|
||||
result = max(candidate_steps) + 1
|
||||
if getattr(self, "_declarations_materialized", False):
|
||||
if getattr(self, "_resolution_finished", False):
|
||||
object.__setattr__(self, "_max_speculative_num_draft_tokens", result)
|
||||
return result
|
||||
|
||||
@@ -9913,7 +9911,7 @@ class ServerArgs:
|
||||
assert (
|
||||
max(chunk_size, page_size) % min(chunk_size, page_size) == 0
|
||||
), f"For SSM models, either chunk_size or page_size must be divisible by the other, got {chunk_size=}, {page_size=}"
|
||||
if not getattr(self, "_declarations_materialized", False):
|
||||
if not getattr(self, "_resolution_finished", False):
|
||||
return max(chunk_size, page_size)
|
||||
self._mamba_cache_chunk_size = max(chunk_size, page_size)
|
||||
return self._mamba_cache_chunk_size
|
||||
@@ -10574,8 +10572,8 @@ class ServerArgs:
|
||||
"endpoint_host": host,
|
||||
"endpoint_port_base": port,
|
||||
"topic": cfg.topic,
|
||||
"block_size": self.kv_event_block_size,
|
||||
"dp_size": self.dp_size,
|
||||
"block_size": resolved.kv_event_block_size,
|
||||
"dp_size": resolved.dp_size,
|
||||
}
|
||||
|
||||
def should_report_expert_balancedness(self) -> bool:
|
||||
@@ -10593,12 +10591,19 @@ class ServerArgs:
|
||||
return cfg.expert_balancedness_report_mode in ("prometheus", "both")
|
||||
|
||||
|
||||
def compute_world_size(server_args: ServerArgs) -> int:
|
||||
"""Return the total GPU count across all data-parallel replicas."""
|
||||
def compute_world_size(config) -> int:
|
||||
"""Return the total GPU count across all data-parallel replicas.
|
||||
|
||||
Takes the resolved topology -- the published `parallel` bag, or a view over
|
||||
the declarations. `enable_dp_attention` and `dp_size` are both resolution's
|
||||
answers (`_handle_dwdp` fills the pair, DeepSeek MLA context parallelism
|
||||
turns DP attention on), so a raw-record read would size the world from what
|
||||
the operator typed.
|
||||
"""
|
||||
return (
|
||||
(1 if server_args.enable_dp_attention else server_args.dp_size)
|
||||
* server_args.tp_size
|
||||
* server_args.pp_size
|
||||
(1 if config.enable_dp_attention else config.dp_size)
|
||||
* config.tp_size
|
||||
* config.pp_size
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user