config: retire ServerArgs.override in favour of derive()

`ServerArgs.override(source, **fields)` was the last way to change a resolved
`ServerArgs` in place. Every remaining call-site was one of two things, and
neither wanted an in-place write:

- **A config for someone else.** A draft worker's context length, an encode
  worker's device, the compile script's watchdog, the client's port pick, a test
  fixture's backends. These already deepcopied first — the write was on the copy.
- **A launcher-stage resolution.** `resolve_auto_parsers` detected the chat
  template's parsers and wrote them back, to be inherited by the schedulers it
  spawns.

Both are "one config becomes another", so `derive(source, **fields)` returns the
variant and leaves the receiver — and any bags projected from it — untouched. It
deliberately is not `dataclasses.replace`: resolution does not re-run, because
the values being set are decided after it, from inputs it never had. Provenance
and the resolvable-field stash work as before, on the copy.

`resolve_auto_parsers` now computes the parsers and returns the config to launch
with; the detection helpers stop taking a config to mutate. `HiMambaRadixCache`
re-applied a HiCache layout normalization `__post_init__` already performs (the
same duplicate removed from `UnifiedRadixCache` in ebb1c88d23) and just goes.

With no in-place mutation left, `ServerArgs.__setattr__` raising after
resolution *is* the guarantee, so the textual writer ratchet retires and
`test_server_args_derive.py` pins the contract instead: the receiver survives
deriving, the published instance still refuses assignment, and deriving does not
publish. `SGLANG_STRICT_CONFIG_MUTATION` was already unused — the guard has been
unconditional since the mutation sweep — and goes with it.

The detection tests drop their `SimpleNamespace` stand-in for a real
`ServerArgs`; the test kit and the MLA chunk-metadata fixture publish a derived
variant instead of writing the runner's published config.
This commit is contained in:
Cheng Wan
2026-08-05 19:30:53 -07:00
committed by GitHub
parent d33ab39ebc
commit 99cfc90658
22 changed files with 411 additions and 279 deletions
+10 -15
View File
@@ -169,21 +169,18 @@ def launch_server_process_and_send_one_request(
)
def refine_server_args(server_args: ServerArgs, compile_args: CompileArgs):
# Disable cuda graph and torch compile to save time. Writes after
# ServerArgs.__post_init__ don't propagate to cuda_graph_config via the
# legacy disable_cuda_graph field, so flip both phases directly.
def compile_server_args(args, compile_args: CompileArgs) -> ServerArgs:
"""The config this script serves with: no cuda graph, no torch compile, and a
watchdog that outlives the compilation."""
args.enable_torch_compile = False
# Watchdog timeout follows compile_args.timeout because compilation takes long.
args.watchdog_timeout = compile_args.timeout
args.warmups = "compile-deep-gemm"
server_args = ServerArgs.from_cli_args(args)
server_args.cuda_graph_config[Phase.DECODE].backend = Backend.DISABLED
server_args.cuda_graph_config[Phase.PREFILL].backend = Backend.DISABLED
print(f"Disable CUDA Graph and Torch Compile to save time...")
# Watchdog timeout follows compile_args.timeout because compilation takes long.
server_args.override(
"compile_deep_gemm.refine_server_args",
enable_torch_compile=False,
watchdog_timeout=compile_args.timeout,
warmups="compile-deep-gemm",
)
return server_args
def run_compile(server_args: ServerArgs, compile_args: CompileArgs):
@@ -216,9 +213,7 @@ if __name__ == "__main__":
ServerArgs.add_cli_args(parser)
CompileArgs.add_cli_args(parser)
args = parser.parse_args()
server_args = ServerArgs.from_cli_args(args)
compile_args = CompileArgs.from_cli_args(args)
refine_server_args(server_args, compile_args)
server_args = compile_server_args(args, compile_args)
run_compile(server_args, compile_args)
@@ -384,13 +384,15 @@ class Runtime:
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils.network import is_port_available
self.server_args = ServerArgs(*args, log_level=log_level, **kwargs)
# Pre-allocate ports
for port in range(self.server_args.port, 40000):
# Pre-allocate a port before building the config, so the config is born
# with the port this runtime will serve on.
requested_port = kwargs.pop(
"port", ServerArgs.__dataclass_fields__["port"].default
)
for port in range(requested_port, 40000):
if is_port_available(port):
break
self.server_args.override("runtime_endpoint.port_alloc", port=port)
self.server_args = ServerArgs(*args, log_level=log_level, port=port, **kwargs)
self.url = self.server_args.url()
self.generate_url = self.url + "/generate"
+37 -2
View File
@@ -208,12 +208,47 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
def _apply_fields(server_args: Any, fields: Dict[str, Any]) -> None:
"""Write fields on behalf of the pipeline (bypasses the strict bare-
assignment guard that protects post-resolution mutation)."""
object.__setattr__(server_args, "_in_override", True)
object.__setattr__(server_args, "_internal_write", True)
try:
for field, value in fields.items():
setattr(server_args, field, value)
finally:
object.__setattr__(server_args, "_in_override", False)
object.__setattr__(server_args, "_internal_write", False)
def declare_late_resolution(server_args: Any, source: str, **fields: Any) -> None:
"""Resolve fields on a config that is **not published yet**.
A few resolution rules cannot run inside ``__post_init__``: LoRA
normalization and the auto-parser detection need the launcher's validation
stage (and, for the parsers, a tokenizer / chat-template load). They still
belong to the resolution pipeline — they decide what the process will run
with — so they write the fields in place, before anything publishes the
object. Writing in place is the point: every holder of that instance (the
HTTP server, the multi-tokenizer workers it serializes for, the schedulers
it forks) must see the resolved value.
Refuses to touch the published instance: after publish the bags exist and a
field write would desync them, which is what ``get_context().override`` is
for.
"""
from sglang.srt.runtime_context import get_context
try:
published = get_context().server_args
except ValueError:
published = None
if published is server_args:
raise ValueError(
f"declare_late_resolution({source!r}) called on the published config; "
"post-publish changes go to the bags via get_context().override(...)"
)
log = getattr(server_args, "_runtime_mutations", None)
if log is None:
log = []
object.__setattr__(server_args, "_runtime_mutations", log)
log.append((source, dict(fields)))
_apply_fields(server_args, fields)
def materialize_declarations(server_args: Any) -> None:
@@ -1,7 +1,6 @@
import asyncio
import concurrent.futures
import contextlib
import copy
import ctypes
import functools
import logging
@@ -3520,10 +3519,7 @@ async def run_dp_worker(
# gpu_id is the device chosen by maybe_reindex_device_id in the parent:
# 0 when CVD is pinned to one GPU, else the absolute id. rank=0, so
# MMEncoder runs set_device(base_gpu_id).
args = copy.deepcopy(server_args)
# The copy is already resolved (read-only); route the per-worker
# specialization through the audited mutation entry.
args.override("encode_server.dp_worker", base_gpu_id=gpu_id, tp_size=1)
args = server_args.derive("encode_server.dp_worker", base_gpu_id=gpu_id, tp_size=1)
enc = MMEncoder(args, dist_init_method=f"tcp://127.0.0.1:{get_free_port()}", rank=0)
global encoder_metrics_collector
-4
View File
@@ -247,10 +247,6 @@ class DsparkFoldedSampling(IntEnum):
class Envs:
# Raise on bare server_args field assignments after resolution; mutation
# must go through ServerArgs.override() (enabled by the test harness).
SGLANG_STRICT_CONFIG_MUTATION = EnvBool(False)
# Per-role config-namespace bookkeeping: off / record / enforce (value is
# validated fail-loud in runtime_context, which resolves it once at import
# so the read stays dynamo-prunable).
@@ -99,16 +99,6 @@ class HiMambaRadixCache(MambaRadixCache):
def __init__(self, params: CacheInitParams, server_args: ServerArgs):
self._enable_metrics_flag = params.enable_metrics
if server_args.hicache_io_backend == "direct":
if server_args.hicache_mem_layout == "page_first":
server_args.override(
"hicache.mem_layout_force", hicache_mem_layout="page_first_direct"
)
logger.warning(
"Page first layout is not supported with direct IO backend, "
"switching to page first direct layout"
)
self.page_size = params.page_size
self.hybrid_kv_cache = params.token_to_kv_pool_allocator.get_kvcache()
if not isinstance(self.hybrid_kv_cache, HybridLinearKVPool):
+48 -54
View File
@@ -22,13 +22,15 @@ import logging
import os
import re
from dataclasses import dataclass
from typing import Callable, Optional, Tuple
from typing import Callable, Dict, Optional, Tuple
import jinja2
import jinja2.ext
import jinja2.nodes
import jinja2.sandbox
from sglang.srt.arg_groups.overrides import declare_late_resolution
logger = logging.getLogger(__name__)
@@ -625,26 +627,21 @@ def detect_inline_system_support(chat_template: Optional[str]) -> bool:
return False
def _resolve_auto_parser(
server_args,
def _detect_auto_parser(
attr: str,
ctx: TemplateDetectionContext,
rules: Tuple[DetectionRule, ...],
label: str,
) -> None:
"""Resolve a single auto parser, updating server_args in place."""
) -> Optional[str]:
"""The parser one auto field resolves to (``None`` disables it)."""
detected = match_rules(ctx, rules, label)
if detected:
server_args.override(source="template-detection", **{attr: detected})
logger.info(
f"Auto-detected --{attr.replace('_', '-')} as '{detected}' from chat template"
)
else:
logger.warning(
f"--{attr.replace('_', '-')}=auto specified but could not detect "
f"{label} from chat template. Disabling {label}."
)
server_args.override(source="template-detection", **{attr: None})
return detected
_log_undetected_parser(attr, label)
return None
def _load_explicit_jinja_template(chat_template_arg: Optional[str]) -> Optional[str]:
@@ -658,15 +655,15 @@ def _load_explicit_jinja_template(chat_template_arg: Optional[str]) -> Optional[
return f.read().replace("\\n", "\n")
def _disable_auto_parser(server_args, attr: str, label: str) -> None:
def _log_undetected_parser(attr: str, label: str) -> None:
logger.warning(
f"--{attr.replace('_', '-')}=auto specified but could not detect "
f"{label} from chat template. Disabling {label}."
)
server_args.override(source="template-detection", **{attr: None})
def _resolve_architecture_auto_parsers(server_args) -> None:
def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str, str]:
"""The parsers the model architecture implies, for the fields still on auto."""
from sglang.srt.utils.hf_transformers_utils import get_config
config = get_config(
@@ -686,30 +683,37 @@ def _resolve_architecture_auto_parsers(server_args) -> None:
elif "DeepseekV3" in arch:
reasoning_parser, tool_call_parser = "deepseek-v3", "deepseekv32"
else:
return
return {}
resolved = {}
for attr, detected in (
("reasoning_parser", reasoning_parser),
("tool_call_parser", tool_call_parser),
):
if getattr(server_args, attr) == "auto":
server_args.override(source="template-detection", **{attr: detected})
if attr in needs:
resolved[attr] = detected
logger.info(
f"Auto-detected --{attr.replace('_', '-')} as '{detected}' "
f"from model architecture '{arch}'"
)
return resolved
def resolve_auto_parsers(server_args) -> None:
"""Resolve --reasoning-parser=auto and --tool-call-parser=auto before scheduler.
"""Resolve ``--reasoning-parser=auto`` / ``--tool-call-parser=auto`` from the
chat template, in place, before anything publishes ``server_args``.
This performs a lightweight tokenizer load to detect parsers from the chat
template. Called early in engine init before scheduler subprocesses are spawned.
Performs a lightweight tokenizer load, so it runs once in engine init. In
place because everyone who holds this instance must see the resolved value:
the schedulers it forks, the HTTP server, and the tokenizer workers it is
serialized for.
"""
needs_reasoning = server_args.reasoning_parser == "auto"
needs_tool_call = server_args.tool_call_parser == "auto"
if not needs_reasoning and not needs_tool_call:
needs = tuple(
attr
for attr in ("reasoning_parser", "tool_call_parser")
if getattr(server_args, attr) == "auto"
)
if not needs:
return
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
@@ -741,6 +745,8 @@ def resolve_auto_parsers(server_args) -> None:
ctx = build_detection_context(
template, tokenizer, reasoning_config, force_reasoning
)
detected: Dict[str, Optional[str]] = {}
if ctx is None:
if has_explicit_template_without_detection:
logger.warning(
@@ -750,38 +756,26 @@ def resolve_auto_parsers(server_args) -> None:
)
else:
try:
_resolve_architecture_auto_parsers(server_args)
detected.update(_architecture_auto_parsers(server_args, needs))
except Exception as e:
logger.warning(
"Failed to load model config for architecture-based auto-detection: %s",
e,
)
if needs_reasoning:
if server_args.reasoning_parser == "auto":
_disable_auto_parser(
server_args, "reasoning_parser", "reasoning parser"
)
if needs_tool_call:
if server_args.tool_call_parser == "auto":
_disable_auto_parser(
server_args, "tool_call_parser", "tool-call parser"
)
return
for attr, label in (
("reasoning_parser", "reasoning parser"),
("tool_call_parser", "tool-call parser"),
):
if attr in needs and attr not in detected:
_log_undetected_parser(attr, label)
detected[attr] = None
else:
for attr, rules, label in (
("reasoning_parser", REASONING_PARSER_RULES, "reasoning parser"),
("tool_call_parser", TOOL_CALL_PARSER_RULES, "tool-call parser"),
):
if attr in needs:
detected[attr] = _detect_auto_parser(attr, ctx, rules, label)
if needs_reasoning:
_resolve_auto_parser(
server_args,
"reasoning_parser",
ctx,
REASONING_PARSER_RULES,
"reasoning parser",
)
if needs_tool_call:
_resolve_auto_parser(
server_args,
"tool_call_parser",
ctx,
TOOL_CALL_PARSER_RULES,
"tool-call parser",
)
if detected:
declare_late_resolution(server_args, "template-detection", **detected)
+2 -3
View File
@@ -1011,8 +1011,7 @@ class _ServerArgsOverride:
def install(self) -> ServerArgs:
"""Publish a fresh dummy-boundary ``ServerArgs`` carrying the
overrides (written through ``ServerArgs.override`` for provenance);
returns the published instance."""
overrides; returns the published instance."""
from sglang.srt.server_args import ServerArgs
assert not self._installed, "override_server_args already installed"
@@ -1025,7 +1024,7 @@ class _ServerArgsOverride:
self._prev_capture = ctx.flags.capture.enable_torch_compile
server_args = ServerArgs(model_path="dummy")
if self._fields:
server_args.override(source="test-override", **self._fields)
server_args = server_args.derive("test-override", **self._fields)
# The dummy boundary skips materialization, which would leave the
# strict mutation guard unarmed on the published object — mark it
# materialized so bare post-publish writes raise like they do on a
+50 -30
View File
@@ -16,6 +16,7 @@
from __future__ import annotations
import argparse
import copy
import dataclasses
import glob
import importlib
@@ -8547,56 +8548,73 @@ class ServerArgs:
return resolved_view(self)
def override(self, source: str, **fields) -> None:
"""The single post-resolution mutation point.
def _late_resolution(self, source: str, **fields) -> None:
"""Resolve fields at the launcher's validation stage (pre-publish).
After ``__post_init__`` the configuration is resolved; the audited
runtime adjustments (load-resolved values, control-plane
reconfiguration, deployment wiring) go through here instead of
assigning fields. Whitelisted resolvable fields also join the
declaration stash, so a republish resolves the same values;
everything is recorded with its ``source`` for provenance.
See ``arg_groups.overrides.declare_late_resolution``: in place, because
every holder of this instance must see the resolved value, and refused
outright once the config is published.
"""
from sglang.srt.arg_groups.overrides import declare_late_resolution
declare_late_resolution(self, source, **fields)
def derive(self, source: str, **fields) -> ServerArgs:
"""A copy carrying variant values: a draft worker's context length, an
encode worker's device, a launcher's late port pick.
The receiver is untouched, so a config already published from it -- and
the namespace bags projected out of it -- stay true; the variant is a
second config, to be published in its own right or handed to whoever
owns it. Resolution does not re-run: the values being set are decided
*after* it, from inputs resolution never had, and re-resolving a
resolved config re-derives conditional decisions from the wrong ones.
Whitelisted resolvable fields also join the copy's declaration stash so
a later re-resolution keeps them; ``source`` is recorded for provenance.
"""
from sglang.srt.arg_groups.arg_utils import resolvable_fields
whitelist = resolvable_fields(type(self))
variant = copy.deepcopy(self)
whitelist = resolvable_fields(type(variant))
declared = {k: v for k, v in fields.items() if k in whitelist}
rest = {k: v for k, v in fields.items() if k not in whitelist}
if declared:
stash = getattr(self, "_resolved_overrides", None)
stash = getattr(variant, "_resolved_overrides", None)
if stash is None:
stash = []
object.__setattr__(self, "_resolved_overrides", stash)
object.__setattr__(variant, "_resolved_overrides", stash)
stash.append((source, dict(declared)))
if rest:
log = getattr(self, "_runtime_mutations", None)
log = getattr(variant, "_runtime_mutations", None)
if log is None:
log = []
object.__setattr__(self, "_runtime_mutations", log)
object.__setattr__(variant, "_runtime_mutations", log)
log.append((source, dict(rest)))
object.__setattr__(self, "_in_override", True)
object.__setattr__(variant, "_internal_write", True)
try:
for field, value in fields.items():
setattr(self, field, value)
setattr(variant, field, value)
finally:
object.__setattr__(self, "_in_override", False)
object.__setattr__(variant, "_internal_write", False)
return variant
def __setattr__(self, name, value):
# after materialization the fields are the resolved startup
# configuration -- the pristine, READ-ONLY record. A bare assignment
# outside ServerArgs.override() (and the resolution pipeline, which runs
# before materialization) always raises; resolved config is mutated on
# the context bags via get_context().override(...), not here. (Formerly
# gated on SGLANG_STRICT_CONFIG_MUTATION; now unconditional.)
# After materialization the fields are the resolved startup
# configuration -- the pristine, READ-ONLY record that the config bags
# were projected from. Resolved config changes go to the bags via
# get_context().override(source, ...); a config that differs per runner
# or per worker is a separate object, built with derive().
if (
not name.startswith("_")
and getattr(self, "_declarations_materialized", False)
and not getattr(self, "_in_override", False)
and not getattr(self, "_internal_write", False)
):
raise AttributeError(
f"server_args.{name} assigned after resolution; server_args is "
"read-only -- use get_context().override(source, ...) to change "
"resolved config."
"resolved config, or server_args.derive(source, ...) to build a "
"variant for one runner."
)
object.__setattr__(self, name, value)
@@ -8914,7 +8932,7 @@ class ServerArgs:
# Enable LoRA if any LoRA paths are provided for backward compatibility.
if self.lora_paths:
if self.enable_lora is None:
self.override("check_lora_server_args", enable_lora=True)
self._late_resolution("check_lora_server_args", enable_lora=True)
logger.warning(
"--enable-lora is set to True because --lora-paths is provided."
)
@@ -8925,7 +8943,7 @@ class ServerArgs:
if self.enable_lora:
if self.enable_lora_overlap_loading is None:
self.override(
self._late_resolution(
"check_lora_server_args", enable_lora_overlap_loading=False
)
@@ -8984,9 +9002,11 @@ class ServerArgs:
"Expected a string or a dictionary."
)
parsed_lora_paths.append(lora_ref)
self.override("check_lora_server_args", lora_paths=parsed_lora_paths)
self._late_resolution(
"check_lora_server_args", lora_paths=parsed_lora_paths
)
elif isinstance(self.lora_paths, dict):
self.override(
self._late_resolution(
"check_lora_server_args",
lora_paths=[
LoRARef(
@@ -8999,7 +9019,7 @@ class ServerArgs:
],
)
elif self.lora_paths is None:
self.override("check_lora_server_args", lora_paths=[])
self._late_resolution("check_lora_server_args", lora_paths=[])
else:
raise ValueError(
f"Invalid type for --lora-paths: {type(self.lora_paths)}. "
@@ -9009,7 +9029,7 @@ class ServerArgs:
# Normalize target modules to a set; keep {"all"} as a sentinel
# that gets resolved model-awarely in lora_manager.init_lora_shapes().
if self.lora_target_modules:
self.override(
self._late_resolution(
"check_lora_server_args",
lora_target_modules=set(self.lora_target_modules),
)
@@ -1,7 +1,6 @@
from __future__ import annotations
import logging
from copy import deepcopy
from typing import TYPE_CHECKING, Optional
import msgspec
@@ -43,11 +42,11 @@ class DraftWorkerBundle(msgspec.Struct, frozen=True):
def _resolve_draft_attention_backend_fallback(
*, draft_server_args: ServerArgs, algo_label: str
*, server_args: ServerArgs, algo_label: str
) -> str:
draft_backend = draft_server_args.speculative_draft_attention_backend
draft_backend = server_args.speculative_draft_attention_backend
if draft_backend is None:
draft_backend, _ = draft_server_args.get_attention_backends()
draft_backend, _ = server_args.get_attention_backends()
if draft_backend is None:
return "triton" if torch.version.hip else "flashinfer"
if draft_backend not in _SUPPORTED_DRAFT_BACKENDS:
@@ -111,8 +110,7 @@ def draft_server_args_copy(server_args: ServerArgs, target_model_config) -> Serv
for _source, fields in get_context().overrides_log():
resolved.update(fields)
draft_server_args = deepcopy(server_args)
draft_server_args.override(
return server_args.derive(
"draft_worker.copy",
**{
**resolved,
@@ -120,7 +118,6 @@ def draft_server_args_copy(server_args: ServerArgs, target_model_config) -> Serv
**_draft_load_format_fields(),
},
)
return draft_server_args
def build_draft_tp_worker(
@@ -133,18 +130,15 @@ def build_draft_tp_worker(
algo_label: str,
attention_backend_override: Optional[str] = None,
) -> DraftWorkerBundle:
draft_server_args = deepcopy(server_args)
# An override names a draft-specific backend the caller has already
# validated (e.g. a self-drafting architecture); it skips the generic
# supported-backend fallback below.
draft_backend = attention_backend_override or (
_resolve_draft_attention_backend_fallback(
draft_server_args=draft_server_args, algo_label=algo_label
server_args=server_args, algo_label=algo_label
)
)
# Post-resolution ServerArgs rejects bare assignment; route the draft-copy
# adjustments through the audited mutation point.
draft_server_args.override(
draft_server_args = server_args.derive(
"draft_worker.build",
**draft_server_args_overrides(target_model_config, draft_backend),
)
@@ -295,7 +295,6 @@ def _configure_runner_for_eagle_draft(
*,
speculative_attention_mode: str = "decode",
) -> None:
server_args = runner.server_args
updates = {
"attention_backend": case.backend,
"cuda_graph_config": CudaGraphConfig(
@@ -326,11 +325,12 @@ def _configure_runner_for_eagle_draft(
"torch_compile_max_bs": 0,
"use_mla_backend": runner.use_mla_backend,
}
server_args.override(source="attention-unittest-eagle-draft", **updates)
# Re-publish so the bags pick up the overrides.
from sglang.srt.runtime_context import get_context
get_context().set_server_args(server_args)
runner.server_args = runner.server_args.derive(
"attention-unittest-eagle-draft", **updates
)
get_context().set_server_args(runner.server_args)
runner.spec_algorithm = SpeculativeAlgorithm.EAGLE
runner.is_draft_worker = True
@@ -391,11 +391,11 @@ def _build_frozen_kv_mtp_fixture(
runner_batch_size=settings.capture_batch_size,
)
_configure_runner_for_eagle_draft(fixture.runner, case, settings)
fixture.runner.server_args.override(
"attention_unittest.frozen_kv_draft", speculative_algorithm="FROZEN_KV_MTP"
)
from sglang.srt.runtime_context import get_context
fixture.runner.server_args = fixture.runner.server_args.derive(
"attention_unittest.frozen_kv_draft", speculative_algorithm="FROZEN_KV_MTP"
)
get_context().set_server_args(fixture.runner.server_args)
fixture.runner.spec_algorithm = SpeculativeAlgorithm.FROZEN_KV_MTP
fixture.runner.draft_attn_backend = fixture.backend
-4
View File
@@ -9,10 +9,6 @@ import inspect
import json
import logging
import os
# Registered tests run with the strict config-mutation guard: bare
# server_args assignments after resolution raise (use ServerArgs.override).
os.environ.setdefault("SGLANG_STRICT_CONFIG_MUTATION", "1")
import random
import re
import shlex