config: ServerArgs holds the raw input (#36255)

This commit is contained in:
Cheng Wan
2026-08-26 05:14:05 -07:00
committed by GitHub
parent 27c36368b6
commit 413df1f8db
34 changed files with 1201 additions and 383 deletions
+1 -1
View File
@@ -1005,7 +1005,7 @@ def main(server_args, bench_args):
# The decode phase has to capture the batch sizes this run benchmarks, and
# the per-phase convenience knob loses to an explicit --cuda-graph-config
# JSON (resolution applies that last), so the size is merged into that JSON.
if getattr(server_args, "_declarations_materialized", False):
if getattr(server_args, "_resolution_finished", False):
# A record the caller already resolved: nothing will parse a raw dict
# again, so the declaration has to be the finished typed config.
merged = resolution_result(server_args, "cuda_graph_config")
+4 -4
View File
@@ -75,10 +75,10 @@ class Arg:
# When True, this field is skipped by add_cli_args_from_dataclass.
# Use for fields that have no CLI surface (e.g. injected via Python only).
no_cli: bool = False
# When True, this field may be written by config resolution (model
# overrides and post-process passes): it is part of the whitelist accepted
# by the declaration stash, and its resolved value materializes onto the
# field at the end of __post_init__.
# When True, config resolution (model overrides and post-process passes)
# may decide this field: the declaration stash accepts the name, and
# `resolution_result` and the config bags answer with the decision. The
# field keeps what the operator passed.
resolvable: bool = False
+41 -60
View File
@@ -14,9 +14,13 @@
"""Declarative model-override registry.
Model-identity adjustments to the server configuration are DECLARED here and
materialized onto ``server_args`` at the end of ``__post_init__`` (gate
order, last writer wins) — model code never mutates ``ServerArgs`` fields
imperatively.
appended to the record's declaration stash (gate order, last writer wins).
Nothing here writes back onto ``ServerArgs``: the record holds the user's raw
input, and a decision is read through ``resolution_result`` or the published
config bags — model code never mutates ``ServerArgs`` fields imperatively. The
one channel that still leaves a field changed is ``declare_direct_writes``,
which does not perform the write: it captures one an out-of-tree plugin already
made, and undoing it would surprise the plugin's own reads.
Two declaration forms, keyed on ``hf_config.architectures[0]``:
@@ -206,10 +210,8 @@ def register_post_process(fn: Callable[..., dict]) -> Callable[..., dict]:
def _declaration_overlay(server_args: Any) -> Dict[str, Any]:
"""What the declarations say so far, last writer wins.
Passes declare without touching the fields until
``materialize_declarations``, so a mid-resolution reader needs this to see
them; handlers and hooks write as they declare, and for those the overlay
repeats what the field already holds."""
Nothing writes the fields, so a mid-resolution reader needs this to see a
decision at all; the fields keep what the caller supplied."""
overlay: Dict[str, Any] = {}
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
overlay.update(declared)
@@ -222,9 +224,9 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
Evaluates the pass on the resolving state (a read-only view with the
accumulated declarations overlaid from the stash) and appends its
declaration to the stash. During ``__post_init__`` the fields stay
untouched — ``materialize_declarations`` applies the whole stash once at
the end of resolution; a pass invoked after materialization (a post-init
slot) writes through immediately.
untouched: the stash is what the config bags are projected from. A pass
invoked after resolution finished (a post-init slot) writes through
immediately, because there is no later projection to pick it up.
"""
declared = fn(ResolvedView(server_args, overlay=_declaration_overlay(server_args)))
if not isinstance(declared, dict):
@@ -244,7 +246,7 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
stash = server_args._resolved_overrides = []
stash.append(entry)
validate_declarations(server_args, [entry])
if getattr(server_args, "_declarations_materialized", False):
if getattr(server_args, "_resolution_finished", False):
_apply_fields(server_args, declared)
@@ -260,29 +262,17 @@ def _apply_fields(server_args: Any, fields: Dict[str, Any]) -> None:
def declare_resolution(server_args: Any, source: str, **fields: Any) -> None:
"""Record a resolution write in the declaration stash, and apply it now.
"""Record a resolution write in the declaration stash.
The stash is what the projection reads, so a resolver that only assigns
the field leaves that write invisible to it. The immediate write keeps the
resolver's successors seeing the value where they read the field directly.
The stash *is* the resolution result: the bags are projected from it,
`resolution_result` answers from it, and no field is written. A resolver
reading a field another resolver may have decided must read `resolving_view`
(or `ServerArgs._resolved()`), which
`test_resolution_reads_the_declarations` pins.
What it does change is which writer wins. A declaration is appended and
replayed last, so a resolver that declares a field a *deferred* writer (a
post-process pass, a registry entry) also decides now beats it, where its
bare assignment used to be overwritten by that writer's declaration. A
resolver that gates on such a field has to read the resolving view rather
than the raw field, or it decides from a value that is already stale.
For resolvers inside ``__post_init__``: the handlers on ``ServerArgs``
(through ``self._declare``) and the ``arg_groups`` hooks and hardware
defaults they call. Resolution that has to wait for the launcher stage
goes through ``declare_late_resolution`` instead.
Names arrive as keyword arguments, which accept anything; a misspelled one
would otherwise become a new attribute that nothing ever reads, so it is
rejected here. This is not the model-override whitelist: that one limits
which fields a *registry entry* may reach, while a resolver writing the
field it owns is the pipeline resolving by construction.
For resolvers inside ``__post_init__``; launcher-stage resolution goes
through ``declare_late_resolution``. A name that is not a field is rejected
here rather than becoming an attribute nothing reads.
"""
if dataclasses.is_dataclass(type(server_args)):
unknown = sorted(set(fields) - field_names(type(server_args)))
@@ -293,8 +283,6 @@ def declare_resolution(server_args: Any, source: str, **fields: Any) -> None:
stash = []
object.__setattr__(server_args, "_resolved_overrides", stash)
stash.append((source, dict(fields)))
for name, value in fields.items():
setattr(server_args, name, value)
def declare_late_resolution(server_args: Any, source: str, **fields: Any) -> None:
@@ -304,10 +292,10 @@ def declare_late_resolution(server_args: Any, source: str, **fields: Any) -> Non
normalization and the auto-parser detection need the launcher's validation
stage (and, for the parsers, a tokenizer / chat-template load). They still
belong to the resolution pipeline — they decide what the process will run
with — so they write the fields in place, before anything publishes the
object. Writing in place is the point: every holder of that instance (the
HTTP server, the multi-tokenizer workers it serializes for, the schedulers
it forks) must see the resolved value.
with — so their decision goes to the stash like any other, and the record
keeps what the caller passed. Every holder of that instance reads the
decision the same way the rest of the pipeline does: the bags it publishes,
or ``resolution_result``, both of which survive the pickle to a child.
Refuses to touch the published instance: after publish the bags exist and a
field write would desync them, which is what ``get_context().override`` is
@@ -334,7 +322,6 @@ def declare_late_resolution(server_args: Any, source: str, **fields: Any) -> Non
stash = []
object.__setattr__(server_args, "_resolved_overrides", stash)
stash.append((source, dict(fields)))
_apply_fields(server_args, fields)
def declare_direct_writes(
@@ -348,7 +335,10 @@ def declare_direct_writes(
Out-of-tree platform plugins are handed the record and set fields on it.
Their implementations live outside this tree, so they cannot be converted
by editing the resolver; and the raw snapshot is taken before the pipeline
starts, so a plugin's default is neither declared nor raw.
starts, so a plugin's default is neither declared nor raw. The write itself
stays: this captures it into the stash so the projection and the bags carry
it, but reverting the field would break the plugin's own reads of what it
just set. It is the only field a record still carries from resolution.
Rebinding is what the diff sees, and rebinding is all it needs to see: a
plugin that mutates a value in place reaches the projection anyway, because
@@ -384,24 +374,12 @@ def declare_direct_writes(
return result
def materialize_declarations(server_args: Any) -> None:
"""Apply the accumulated declarations onto ``server_args`` once, at the
end of ``__post_init__`` (gate order: last writer wins). After this the
fields carry the resolved configuration — every post-init reader, in any
process, reads them directly; ``resolved_view`` remains an internal
helper for mid-resolution code only."""
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
for field, value in declared.items():
setattr(server_args, field, value)
server_args._declarations_materialized = True
def resolution_result(server_args: Any, field: str, default: Any = None) -> Any:
"""What resolution decided for ``field``: the declaration if there is one,
otherwise what the caller supplied.
This is what the config projection reads. Reading the field instead would
work only for as long as declarations materialize onto the record -- and
work whatever the caller passed onto the record -- and
the point of declaring is that they will not, so the projection must not
depend on it. A config that never ran the pipeline (a mock, a partial
fixture) carries no raw snapshot; its fields are all it has.
@@ -422,9 +400,8 @@ def resolution_projection(server_args: Any) -> Dict[str, Any]:
The whole-object shape of ``resolution_result``, for the exits that hand out
the entire configuration (``/server_info``, the gRPC and engine readbacks).
They used ``dataclasses.asdict``, which reads the fields -- correct only for
as long as declarations materialize onto the record, and the point of
declaring is that they will not. Field values only: the private resolution
They used ``dataclasses.asdict``, which reads the fields -- the operator's
input, not what resolution decided. Field values only: the private resolution
bookkeeping and the ``model_config`` memo that a ``vars()`` dump carried into
the readback are not configuration.
"""
@@ -453,10 +430,14 @@ def _plain(value: Any) -> Any:
def resolved_view(server_args: Any) -> ResolvedView:
"""Read-only view of the resolving configuration for mid-resolution code
that is not a pass (``__post_init__`` handlers and hooks). Internal to
the resolution pipeline: after ``materialize_declarations`` runs, the
fields themselves carry the resolved values — read them directly."""
"""Read-only view of the resolving configuration: the declarations
overlaid on the fields, snapshotted per call.
For mid-resolution code that is not a pass (``__post_init__`` handlers and
hooks), and for the record's own members that must answer with what
resolution decided -- a declaration-only resolver (a model-specific
override, a registry entry) never writes the field, so a field read there
answers with the raw input."""
return ResolvedView(server_args, overlay=_declaration_overlay(server_args))
+2 -2
View File
@@ -251,7 +251,7 @@ class Engine(EngineScoreMixin, EngineBase):
kwargs["log_level"] = "error"
server_args = self.server_args_class(**kwargs)
self.server_args = server_args
logger.info(f"{server_args=}")
logger.info(f"server_args={server_args.resolved_dict()}")
# Rust Server is not supported with the offline Engine API
if envs.SGLANG_RUST_SERVER.get():
@@ -1076,7 +1076,7 @@ class Engine(EngineScoreMixin, EngineBase):
# Allocate ports for inter-process communications
if port_args is None:
port_args = PortArgs.init_new(server_args)
logger.info(f"{server_args=}")
logger.info(f"server_args={server_args.resolved_dict()}")
# Start the engine info bootstrap server if per-rank info is needed.
engine_info_bootstrap_server = None
+2 -2
View File
@@ -803,8 +803,8 @@ async def get_server_info():
async def server_info():
"""The startup configuration, plus live scheduler state.
The `ServerArgs` fields here are the record: what the launcher was given,
with resolution written back into it. Fields the control plane changes
The values here are the resolution result: what the launcher was given,
with every decision resolution made applied over it. Fields the control plane changes
after publication -- the model a weight update swapped in, its load format,
an operator-set weight version -- are reported by `/model_info`, and the
HiCache mirror by `GET /hicache/storage-backend`.
@@ -247,11 +247,10 @@ class MiniMaxSparseAttnBackend(AttentionBackend):
Phase,
check_cuda_graph_backend,
)
from sglang.srt.runtime_context import get_spec
_sa = getattr(runner, "server_args", None)
self.speculative_num_draft_tokens = getattr(
_sa, "speculative_num_draft_tokens", None
)
spec = get_spec()
self.speculative_num_draft_tokens = spec.speculative_num_draft_tokens
_decode_cuda_graph = not check_cuda_graph_backend(
Phase.DECODE, Backend.DISABLED
)
@@ -264,7 +263,7 @@ class MiniMaxSparseAttnBackend(AttentionBackend):
if (
self.use_msa
and _decode_cuda_graph
and getattr(_sa, "speculative_algorithm", None) is not None
and spec.speculative_algorithm is not None
):
raise NotImplementedError(
"MiniMax-M3 MSA attention does not support speculative decoding under "
+2 -2
View File
@@ -315,8 +315,8 @@ def initialize_moe_config():
"""Seed the MoE runtime flags from the published configuration.
Reads the bags: `moe_a2a_backend` and its siblings are resolution's
answers, and the record carries them only while declarations materialize
onto it. Called once per process after publish
answers, and the record carries the operator's input. Called once per
process after publish
(scheduler init, the benchmark work functions).
"""
exec_moe = get_exec().moe
+1 -2
View File
@@ -39,7 +39,6 @@ from sglang.srt.runtime_context import (
get_observability,
get_parallel,
get_schedule,
get_server_args,
get_serving,
get_spec,
)
@@ -4412,7 +4411,7 @@ class Scheduler(
# Resolved config (pristine server_args + post-publish overrides) so a
# readback reflects values changed via /set_internal_state, not startup.
ret = get_context().resolved_server_args_dict()
ret["world_size"] = compute_world_size(get_server_args())
ret["world_size"] = compute_world_size(get_parallel().config)
ret["last_gen_throughput"] = self.metrics_reporter.last_gen_throughput
draft_graph_memory_usage = (
None if self.draft_worker is None else self.draft_worker.graph_memory_usage
@@ -1380,7 +1380,7 @@ class KVCacheConfigurator:
"""
full_pool_class = DSATokenToKVPool if is_dsa_model else MLATokenToKVPool
common = {
"page_size": self.server_args.page_size,
"page_size": get_schedule().page_size,
"device": self.device,
"enable_memory_saver": False,
}
@@ -1401,7 +1401,7 @@ class KVCacheConfigurator:
return SWAKVPool(
size=full_max_total_num_tokens,
size_swa=swa_max_total_num_tokens,
page_size=self.server_args.page_size,
page_size=get_schedule().page_size,
dtype=self.kv_cache_dtype,
head_num=0,
head_dim=0,
@@ -702,12 +702,13 @@ def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str,
def resolve_auto_parsers(server_args) -> None:
"""Resolve ``--reasoning-parser=auto`` / ``--tool-call-parser=auto`` from the
chat template, in place, before anything publishes ``server_args``.
chat template, before anything publishes ``server_args``.
Performs a lightweight tokenizer load, so it runs once in engine init. In
place because everyone who holds this instance must see the resolved value:
the schedulers it forks, the HTTP server, and the tokenizer workers it is
serialized for.
Performs a lightweight tokenizer load, so it runs once in engine init. The
decision goes to this instance's declaration stash, so every holder of it
carries it -- the schedulers it forks, the HTTP server, the tokenizer
workers it is serialized for -- and each publishes bags projected from it.
The fields stay what the operator passed.
"""
cfg = resolving_view(server_args)
needs = tuple(
+2 -6
View File
@@ -35,7 +35,7 @@ from sglang.srt.ray.scheduler_actor import SchedulerActor
from sglang.srt.runtime_context import (
get_parallel,
)
from sglang.srt.server_args import PortArgs, ServerArgs
from sglang.srt.server_args import PortArgs, ServerArgs, compute_world_size
logger = logging.getLogger(__name__)
@@ -108,14 +108,10 @@ def _get_bundle_node_ip(placement_group: PlacementGroup, bundle_idx: int) -> str
def _compute_world_size() -> int:
"""Compute world_size (total number of scheduler actors/GPUs needed).
Normal: dp_size * tp_size * pp_size; DP attention: tp_size * pp_size.
Reads the published parallel leaves: the driver is sizing the actors that
will hold the process groups, so there is nothing live to ask.
"""
parallel = get_parallel().config
if parallel.enable_dp_attention:
return parallel.tp_size * parallel.pp_size
return parallel.dp_size * parallel.tp_size * parallel.pp_size
return compute_world_size(get_parallel().config)
def _resolve_bundle_indices(pg: PlacementGroup, world_size: int) -> List[int]:
+21 -14
View File
@@ -24,9 +24,10 @@ getters. The resolved parallel **configuration** is the same object's ``config``
hop (``get_parallel().config.tp_size``), which reads the published ``parallel``
bag: bare is the live group, ``config`` is what was configured.
``get_server_args()`` returns the process-wide ``ServerArgs``. This is the pristine / resolved-at-startup **read-only** record kept
for debug and reproduction; business code reads resolved config from the
namespace bags below, not from this object. The context owns the storage:
``get_server_args()`` returns the process-wide ``ServerArgs``. This is the
user's raw input, kept **read-only** for debug and reproduction; what
resolution decided lives in the declarations (``resolution_result``) and, for
business code, in the namespace bags below -- never on this object's fields. The context owns the storage:
publishing goes through ``RuntimeContext.set_server_args`` (the legacy
``set_global_server_args_for_scheduler`` / ``get_global_server_args`` are thin
shims over this slot).
@@ -440,8 +441,8 @@ class DpFlags(_FlagGroupBase):
class Flags(_FlagGroupBase):
"""Root of the runtime-flags tier.
Resolved configuration lives on ``server_args`` fields (materialized at
the end of ``__post_init__``) this tier only carries genuine runtime
Resolved configuration lives in the config bags below (projected from the
declarations at publish) this tier only carries genuine runtime
state whose value is not a function of the configuration alone, grouped
by lifecycle (``capture``) or subsystem (``moe`` / ``dp``).
"""
@@ -700,8 +701,7 @@ def _build_config_bags(server_args: Any) -> dict:
"""Snapshot the resolution result into the namespace bag tree, driven by
the ``NS(...)`` metadata on the dataclass fields. Each leaf comes from
``resolution_result`` -- the declaration if resolution made one, else what
the caller supplied -- rather than from the field, which carries the same
value only while declarations still materialize. Returns
the caller supplied. Returns
``{top_level_name: _ConfigBag}``, arbitrarily nested (``exec.moe.eplb.``).
Only dataclass fields carry ``NS`` markers, so derived properties/methods are
naturally excluded (they stay on the bag). A name used as both a leaf and a
@@ -783,7 +783,13 @@ class RuntimeContext:
if stream is None:
import torch
device = self._server_args.device if self._server_args else "cuda"
from sglang.srt.arg_groups.overrides import resolution_result
device = (
resolution_result(self._server_args, "device")
if self._server_args
else "cuda"
)
stream = torch.get_device_module(device).Stream()
self.resources.streams[name] = stream
return stream
@@ -819,8 +825,8 @@ class RuntimeContext:
Overwrite-allowed: a re-publish replaces the slot (test kits re-publish
per test; production ordering discipline lives at the call-sites, e.g.
the draft-worker guard in ``ModelRunner.__init__``). The published
object already carries the resolved configuration (declarations
materialize at the end of ``__post_init__``).
object is the raw input; the resolution it carries is its declaration
stash, which is what the bags are projected from.
"""
# Seed the capture tier for the new lifecycle (defaults for sentinel
# and mock publishes, which carry no config).
@@ -1078,10 +1084,11 @@ class _ServerArgsOverride:
}
if declared:
declare_late_resolution(server_args, "override_server_args", **declared)
_apply_fields(
server_args,
{name: value for name, value in self._fields.items() if name[0] == "_"},
)
# This hook stands in for a launch: the caller's values are both what
# the operator passed and what resolution decided, so they go on the
# record as well as into the stash. Production late resolution declares
# only -- there the record stays the operator's input.
_apply_fields(server_args, self._fields)
ctx.set_server_args(server_args)
self._installed = True
return server_args
+47 -42
View File
@@ -3709,7 +3709,7 @@ class ServerArgs:
arrived by pickle and brought its declarations along, so the child has
nothing left to derive and projects what the parent decided.
"""
if getattr(self, "_declarations_materialized", False):
if getattr(self, "_resolution_finished", False):
return
if getattr(self, "_resolution_failed", False):
raise RuntimeError(
@@ -3721,23 +3721,22 @@ class ServerArgs:
try:
self._run_resolution_pipeline()
except BaseException:
# The handlers that ran already wrote to the record, and they are
# not idempotent over their own output.
# The handlers that ran already declared, and they are not
# idempotent over their own output.
object.__setattr__(self, "_resolution_failed", True)
raise
# Set here too, because the dummy/absent-model path returns before the
# materialization that normally sets it: the gate is about whether the
# handlers ran, not how far they got.
self._declarations_materialized = True
# end of the pipeline that normally sets it: the gate is about whether
# the handlers ran, not how far they got.
self._resolution_finished = True
def resolved_dict(self) -> Dict[str, Any]:
"""This configuration as a plain dict of resolved field values.
What the whole-object readbacks report (`/server_info` and its gRPC and
in-process twins). `dataclasses.asdict(self)` reads the fields, which
carry resolution's result only while declarations materialize onto the
record; this reads the declarations, so it keeps answering with what
resolution decided once they stop. Nested dataclass fields are expanded
carry the raw input; this reads the declarations, so it answers with what
resolution decided. Nested dataclass fields are expanded
the way `asdict` expands them; the private resolution bookkeeping and the
`model_config` memo are not fields and do not appear.
"""
@@ -3750,12 +3749,11 @@ class ServerArgs:
`dataclasses.replace` builds a new instance, so the copy carries none of
what makes a record resolved: no raw snapshot, no declarations, no
materialization. The next publish therefore finds an unmaterialized
record and runs the pipeline over values it already decided -- DP
attention halves `chunked_prefill_size` a second time (8192 -> 4096 ->
2048) and the schedule conservativeness is scaled again (0.3 -> 0.09).
The Ray paths replace `dist_init_addr` on a resolved record, which is
how they hit it.
finished flag. The next publish therefore resolves it again, which
drops every decision the stash held -- the late ones (the auto-detected
parsers) and the direct ones alike -- and re-runs the device probes in
whatever process opened the copy. The Ray paths replace
`dist_init_addr` on a resolved record, which is how they reach this.
The change is appended to the stash rather than left on the field: the
projection reads the raw snapshot plus the declarations, so a field the
@@ -3770,7 +3768,7 @@ class ServerArgs:
copy's deep structure in-process mutates the parent's too.
"""
replacement = dataclasses.replace(self, **changes)
if not getattr(self, "_declarations_materialized", False):
if not getattr(self, "_resolution_finished", False):
# Not resolved yet: the copy goes through the gate itself.
return replacement
@@ -3780,7 +3778,7 @@ class ServerArgs:
# (the read-only guard refuses the write).
field_names = {field.name for field in dataclasses.fields(self)}
for name, value in vars(self).items():
if name in field_names or name == "_declarations_materialized":
if name in field_names or name == "_resolution_finished":
continue
if isinstance(value, (dict, list, set)):
value = copy.copy(value)
@@ -3791,7 +3789,7 @@ class ServerArgs:
object.__setattr__(replacement, "_resolved_overrides", stash)
if changes:
stash.append((source, dict(changes)))
object.__setattr__(replacement, "_declarations_materialized", True)
object.__setattr__(replacement, "_resolution_finished", True)
return replacement
def _declare(self, source: str, **fields: Any) -> None:
@@ -4022,13 +4020,7 @@ class ServerArgs:
# time; last declarations of the resolution, mirroring that order.
self._handle_model_capability_adjustments()
# End of resolution: apply the accumulated declarations onto the
# fields once (gate order). From here on server_args carries the
# resolved configuration — post-init readers, in any process, read
# the fields directly.
from sglang.srt.arg_groups.overrides import materialize_declarations
materialize_declarations(self)
self._resolution_finished = True
def _handle_return_hidden_states_mode(self):
cfg = resolving_view(self)
@@ -9794,22 +9786,22 @@ class ServerArgs:
def _late_resolution(self, source: str, **fields) -> None:
"""Resolve fields at the launcher's validation stage (pre-publish).
See ``arg_groups.overrides.declare_late_resolution``: in place, because
every holder of this instance must see the resolved value, and refused
outright once the config is published.
See ``arg_groups.overrides.declare_late_resolution``: the decision goes
to this instance's declaration stash, so every holder of it carries the
decision and publishes bags that answer with it. Refused outright once
the config is published.
"""
from sglang.srt.arg_groups.overrides import declare_late_resolution
declare_late_resolution(self, source, **fields)
def __setattr__(self, name, value):
# After materialization the fields are the resolved startup
# configuration -- the pristine, READ-ONLY record that the config bags
# were projected from. Resolved config changes go to the bags via
# Once resolution has finished the record is the READ-ONLY raw input
# the config bags were projected from. Resolved config changes go to the bags via
# get_context().override(source, ...); a value one runner or worker
# owns travels as a constructor argument to it.
if (
getattr(self, "_declarations_materialized", False)
getattr(self, "_resolution_finished", False)
and not getattr(self, "_internal_write", False)
and name not in _CACHE_SLOTS
and (not name.startswith("_") or name in _underscore_field_names())
@@ -9832,7 +9824,13 @@ class ServerArgs:
return attention_backends_of(resolved_view(self))
def get_attention_backends(self):
return attention_backends_of(self)
"""The (prefill, decode) pair resolution decided.
Reads through the declaration stash, not the fields: the model-specific
overrides declare into the stash without writing the fields, so a field
read answers with what the operator typed.
"""
return attention_backends_of(resolved_view(self))
def use_mla_backend(self):
from sglang.srt.configs.model_config import AttentionArch
@@ -9884,7 +9882,7 @@ class ServerArgs:
# state needs steps + 1 draft-token slots. Revisit this if topk>1
# is supported.
result = max(candidate_steps) + 1
if getattr(self, "_declarations_materialized", False):
if getattr(self, "_resolution_finished", False):
object.__setattr__(self, "_max_speculative_num_draft_tokens", result)
return result
@@ -9913,7 +9911,7 @@ class ServerArgs:
assert (
max(chunk_size, page_size) % min(chunk_size, page_size) == 0
), f"For SSM models, either chunk_size or page_size must be divisible by the other, got {chunk_size=}, {page_size=}"
if not getattr(self, "_declarations_materialized", False):
if not getattr(self, "_resolution_finished", False):
return max(chunk_size, page_size)
self._mamba_cache_chunk_size = max(chunk_size, page_size)
return self._mamba_cache_chunk_size
@@ -10574,8 +10572,8 @@ class ServerArgs:
"endpoint_host": host,
"endpoint_port_base": port,
"topic": cfg.topic,
"block_size": self.kv_event_block_size,
"dp_size": self.dp_size,
"block_size": resolved.kv_event_block_size,
"dp_size": resolved.dp_size,
}
def should_report_expert_balancedness(self) -> bool:
@@ -10593,12 +10591,19 @@ class ServerArgs:
return cfg.expert_balancedness_report_mode in ("prometheus", "both")
def compute_world_size(server_args: ServerArgs) -> int:
"""Return the total GPU count across all data-parallel replicas."""
def compute_world_size(config) -> int:
"""Return the total GPU count across all data-parallel replicas.
Takes the resolved topology -- the published `parallel` bag, or a view over
the declarations. `enable_dp_attention` and `dp_size` are both resolution's
answers (`_handle_dwdp` fills the pair, DeepSeek MLA context parallelism
turns DP attention on), so a raw-record read would size the world from what
the operator typed.
"""
return (
(1 if server_args.enable_dp_attention else server_args.dp_size)
* server_args.tp_size
* server_args.pp_size
(1 if config.enable_dp_attention else config.dp_size)
* config.tp_size
* config.pp_size
)