config: retire ServerArgs.override in favour of derive()
`ServerArgs.override(source, **fields)` was the last way to change a resolved
`ServerArgs` in place. Every remaining call-site was one of two things, and
neither wanted an in-place write:
- **A config for someone else.** A draft worker's context length, an encode
worker's device, the compile script's watchdog, the client's port pick, a test
fixture's backends. These already deepcopied first — the write was on the copy.
- **A launcher-stage resolution.** `resolve_auto_parsers` detected the chat
template's parsers and wrote them back, to be inherited by the schedulers it
spawns.
Both are "one config becomes another", so `derive(source, **fields)` returns the
variant and leaves the receiver — and any bags projected from it — untouched. It
deliberately is not `dataclasses.replace`: resolution does not re-run, because
the values being set are decided after it, from inputs it never had. Provenance
and the resolvable-field stash work as before, on the copy.
`resolve_auto_parsers` now computes the parsers and returns the config to launch
with; the detection helpers stop taking a config to mutate. `HiMambaRadixCache`
re-applied a HiCache layout normalization `__post_init__` already performs (the
same duplicate removed from `UnifiedRadixCache` in ebb1c88d23) and just goes.
With no in-place mutation left, `ServerArgs.__setattr__` raising after
resolution *is* the guarantee, so the textual writer ratchet retires and
`test_server_args_derive.py` pins the contract instead: the receiver survives
deriving, the published instance still refuses assignment, and deriving does not
publish. `SGLANG_STRICT_CONFIG_MUTATION` was already unused — the guard has been
unconditional since the mutation sweep — and goes with it.
The detection tests drop their `SimpleNamespace` stand-in for a real
`ServerArgs`; the test kit and the MLA chunk-metadata fixture publish a derived
variant instead of writing the runner's published config.
This commit is contained in:
@@ -169,21 +169,18 @@ def launch_server_process_and_send_one_request(
|
||||
)
|
||||
|
||||
|
||||
def refine_server_args(server_args: ServerArgs, compile_args: CompileArgs):
|
||||
# Disable cuda graph and torch compile to save time. Writes after
|
||||
# ServerArgs.__post_init__ don't propagate to cuda_graph_config via the
|
||||
# legacy disable_cuda_graph field, so flip both phases directly.
|
||||
def compile_server_args(args, compile_args: CompileArgs) -> ServerArgs:
|
||||
"""The config this script serves with: no cuda graph, no torch compile, and a
|
||||
watchdog that outlives the compilation."""
|
||||
args.enable_torch_compile = False
|
||||
# Watchdog timeout follows compile_args.timeout because compilation takes long.
|
||||
args.watchdog_timeout = compile_args.timeout
|
||||
args.warmups = "compile-deep-gemm"
|
||||
server_args = ServerArgs.from_cli_args(args)
|
||||
server_args.cuda_graph_config[Phase.DECODE].backend = Backend.DISABLED
|
||||
server_args.cuda_graph_config[Phase.PREFILL].backend = Backend.DISABLED
|
||||
print(f"Disable CUDA Graph and Torch Compile to save time...")
|
||||
|
||||
# Watchdog timeout follows compile_args.timeout because compilation takes long.
|
||||
server_args.override(
|
||||
"compile_deep_gemm.refine_server_args",
|
||||
enable_torch_compile=False,
|
||||
watchdog_timeout=compile_args.timeout,
|
||||
warmups="compile-deep-gemm",
|
||||
)
|
||||
return server_args
|
||||
|
||||
|
||||
def run_compile(server_args: ServerArgs, compile_args: CompileArgs):
|
||||
@@ -216,9 +213,7 @@ if __name__ == "__main__":
|
||||
ServerArgs.add_cli_args(parser)
|
||||
CompileArgs.add_cli_args(parser)
|
||||
args = parser.parse_args()
|
||||
server_args = ServerArgs.from_cli_args(args)
|
||||
compile_args = CompileArgs.from_cli_args(args)
|
||||
|
||||
refine_server_args(server_args, compile_args)
|
||||
server_args = compile_server_args(args, compile_args)
|
||||
|
||||
run_compile(server_args, compile_args)
|
||||
|
||||
@@ -384,13 +384,15 @@ class Runtime:
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.utils.network import is_port_available
|
||||
|
||||
self.server_args = ServerArgs(*args, log_level=log_level, **kwargs)
|
||||
|
||||
# Pre-allocate ports
|
||||
for port in range(self.server_args.port, 40000):
|
||||
# Pre-allocate a port before building the config, so the config is born
|
||||
# with the port this runtime will serve on.
|
||||
requested_port = kwargs.pop(
|
||||
"port", ServerArgs.__dataclass_fields__["port"].default
|
||||
)
|
||||
for port in range(requested_port, 40000):
|
||||
if is_port_available(port):
|
||||
break
|
||||
self.server_args.override("runtime_endpoint.port_alloc", port=port)
|
||||
self.server_args = ServerArgs(*args, log_level=log_level, port=port, **kwargs)
|
||||
|
||||
self.url = self.server_args.url()
|
||||
self.generate_url = self.url + "/generate"
|
||||
|
||||
@@ -208,12 +208,47 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
def _apply_fields(server_args: Any, fields: Dict[str, Any]) -> None:
|
||||
"""Write fields on behalf of the pipeline (bypasses the strict bare-
|
||||
assignment guard that protects post-resolution mutation)."""
|
||||
object.__setattr__(server_args, "_in_override", True)
|
||||
object.__setattr__(server_args, "_internal_write", True)
|
||||
try:
|
||||
for field, value in fields.items():
|
||||
setattr(server_args, field, value)
|
||||
finally:
|
||||
object.__setattr__(server_args, "_in_override", False)
|
||||
object.__setattr__(server_args, "_internal_write", False)
|
||||
|
||||
|
||||
def declare_late_resolution(server_args: Any, source: str, **fields: Any) -> None:
|
||||
"""Resolve fields on a config that is **not published yet**.
|
||||
|
||||
A few resolution rules cannot run inside ``__post_init__``: LoRA
|
||||
normalization and the auto-parser detection need the launcher's validation
|
||||
stage (and, for the parsers, a tokenizer / chat-template load). They still
|
||||
belong to the resolution pipeline — they decide what the process will run
|
||||
with — so they write the fields in place, before anything publishes the
|
||||
object. Writing in place is the point: every holder of that instance (the
|
||||
HTTP server, the multi-tokenizer workers it serializes for, the schedulers
|
||||
it forks) must see the resolved value.
|
||||
|
||||
Refuses to touch the published instance: after publish the bags exist and a
|
||||
field write would desync them, which is what ``get_context().override`` is
|
||||
for.
|
||||
"""
|
||||
from sglang.srt.runtime_context import get_context
|
||||
|
||||
try:
|
||||
published = get_context().server_args
|
||||
except ValueError:
|
||||
published = None
|
||||
if published is server_args:
|
||||
raise ValueError(
|
||||
f"declare_late_resolution({source!r}) called on the published config; "
|
||||
"post-publish changes go to the bags via get_context().override(...)"
|
||||
)
|
||||
log = getattr(server_args, "_runtime_mutations", None)
|
||||
if log is None:
|
||||
log = []
|
||||
object.__setattr__(server_args, "_runtime_mutations", log)
|
||||
log.append((source, dict(fields)))
|
||||
_apply_fields(server_args, fields)
|
||||
|
||||
|
||||
def materialize_declarations(server_args: Any) -> None:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import contextlib
|
||||
import copy
|
||||
import ctypes
|
||||
import functools
|
||||
import logging
|
||||
@@ -3520,10 +3519,7 @@ async def run_dp_worker(
|
||||
# gpu_id is the device chosen by maybe_reindex_device_id in the parent:
|
||||
# 0 when CVD is pinned to one GPU, else the absolute id. rank=0, so
|
||||
# MMEncoder runs set_device(base_gpu_id).
|
||||
args = copy.deepcopy(server_args)
|
||||
# The copy is already resolved (read-only); route the per-worker
|
||||
# specialization through the audited mutation entry.
|
||||
args.override("encode_server.dp_worker", base_gpu_id=gpu_id, tp_size=1)
|
||||
args = server_args.derive("encode_server.dp_worker", base_gpu_id=gpu_id, tp_size=1)
|
||||
enc = MMEncoder(args, dist_init_method=f"tcp://127.0.0.1:{get_free_port()}", rank=0)
|
||||
|
||||
global encoder_metrics_collector
|
||||
|
||||
@@ -247,10 +247,6 @@ class DsparkFoldedSampling(IntEnum):
|
||||
|
||||
class Envs:
|
||||
|
||||
# Raise on bare server_args field assignments after resolution; mutation
|
||||
# must go through ServerArgs.override() (enabled by the test harness).
|
||||
SGLANG_STRICT_CONFIG_MUTATION = EnvBool(False)
|
||||
|
||||
# Per-role config-namespace bookkeeping: off / record / enforce (value is
|
||||
# validated fail-loud in runtime_context, which resolves it once at import
|
||||
# so the read stays dynamo-prunable).
|
||||
|
||||
@@ -99,16 +99,6 @@ class HiMambaRadixCache(MambaRadixCache):
|
||||
|
||||
def __init__(self, params: CacheInitParams, server_args: ServerArgs):
|
||||
self._enable_metrics_flag = params.enable_metrics
|
||||
if server_args.hicache_io_backend == "direct":
|
||||
if server_args.hicache_mem_layout == "page_first":
|
||||
server_args.override(
|
||||
"hicache.mem_layout_force", hicache_mem_layout="page_first_direct"
|
||||
)
|
||||
logger.warning(
|
||||
"Page first layout is not supported with direct IO backend, "
|
||||
"switching to page first direct layout"
|
||||
)
|
||||
|
||||
self.page_size = params.page_size
|
||||
self.hybrid_kv_cache = params.token_to_kv_pool_allocator.get_kvcache()
|
||||
if not isinstance(self.hybrid_kv_cache, HybridLinearKVPool):
|
||||
|
||||
@@ -22,13 +22,15 @@ import logging
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional, Tuple
|
||||
from typing import Callable, Dict, Optional, Tuple
|
||||
|
||||
import jinja2
|
||||
import jinja2.ext
|
||||
import jinja2.nodes
|
||||
import jinja2.sandbox
|
||||
|
||||
from sglang.srt.arg_groups.overrides import declare_late_resolution
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -625,26 +627,21 @@ def detect_inline_system_support(chat_template: Optional[str]) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_auto_parser(
|
||||
server_args,
|
||||
def _detect_auto_parser(
|
||||
attr: str,
|
||||
ctx: TemplateDetectionContext,
|
||||
rules: Tuple[DetectionRule, ...],
|
||||
label: str,
|
||||
) -> None:
|
||||
"""Resolve a single auto parser, updating server_args in place."""
|
||||
) -> Optional[str]:
|
||||
"""The parser one auto field resolves to (``None`` disables it)."""
|
||||
detected = match_rules(ctx, rules, label)
|
||||
if detected:
|
||||
server_args.override(source="template-detection", **{attr: detected})
|
||||
logger.info(
|
||||
f"Auto-detected --{attr.replace('_', '-')} as '{detected}' from chat template"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"--{attr.replace('_', '-')}=auto specified but could not detect "
|
||||
f"{label} from chat template. Disabling {label}."
|
||||
)
|
||||
server_args.override(source="template-detection", **{attr: None})
|
||||
return detected
|
||||
_log_undetected_parser(attr, label)
|
||||
return None
|
||||
|
||||
|
||||
def _load_explicit_jinja_template(chat_template_arg: Optional[str]) -> Optional[str]:
|
||||
@@ -658,15 +655,15 @@ def _load_explicit_jinja_template(chat_template_arg: Optional[str]) -> Optional[
|
||||
return f.read().replace("\\n", "\n")
|
||||
|
||||
|
||||
def _disable_auto_parser(server_args, attr: str, label: str) -> None:
|
||||
def _log_undetected_parser(attr: str, label: str) -> None:
|
||||
logger.warning(
|
||||
f"--{attr.replace('_', '-')}=auto specified but could not detect "
|
||||
f"{label} from chat template. Disabling {label}."
|
||||
)
|
||||
server_args.override(source="template-detection", **{attr: None})
|
||||
|
||||
|
||||
def _resolve_architecture_auto_parsers(server_args) -> None:
|
||||
def _architecture_auto_parsers(server_args, needs: Tuple[str, ...]) -> Dict[str, str]:
|
||||
"""The parsers the model architecture implies, for the fields still on auto."""
|
||||
from sglang.srt.utils.hf_transformers_utils import get_config
|
||||
|
||||
config = get_config(
|
||||
@@ -686,30 +683,37 @@ def _resolve_architecture_auto_parsers(server_args) -> None:
|
||||
elif "DeepseekV3" in arch:
|
||||
reasoning_parser, tool_call_parser = "deepseek-v3", "deepseekv32"
|
||||
else:
|
||||
return
|
||||
return {}
|
||||
|
||||
resolved = {}
|
||||
for attr, detected in (
|
||||
("reasoning_parser", reasoning_parser),
|
||||
("tool_call_parser", tool_call_parser),
|
||||
):
|
||||
if getattr(server_args, attr) == "auto":
|
||||
server_args.override(source="template-detection", **{attr: detected})
|
||||
if attr in needs:
|
||||
resolved[attr] = detected
|
||||
logger.info(
|
||||
f"Auto-detected --{attr.replace('_', '-')} as '{detected}' "
|
||||
f"from model architecture '{arch}'"
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def resolve_auto_parsers(server_args) -> None:
|
||||
"""Resolve --reasoning-parser=auto and --tool-call-parser=auto before scheduler.
|
||||
"""Resolve ``--reasoning-parser=auto`` / ``--tool-call-parser=auto`` from the
|
||||
chat template, in place, before anything publishes ``server_args``.
|
||||
|
||||
This performs a lightweight tokenizer load to detect parsers from the chat
|
||||
template. Called early in engine init before scheduler subprocesses are spawned.
|
||||
Performs a lightweight tokenizer load, so it runs once in engine init. In
|
||||
place because everyone who holds this instance must see the resolved value:
|
||||
the schedulers it forks, the HTTP server, and the tokenizer workers it is
|
||||
serialized for.
|
||||
"""
|
||||
needs_reasoning = server_args.reasoning_parser == "auto"
|
||||
needs_tool_call = server_args.tool_call_parser == "auto"
|
||||
|
||||
if not needs_reasoning and not needs_tool_call:
|
||||
needs = tuple(
|
||||
attr
|
||||
for attr in ("reasoning_parser", "tool_call_parser")
|
||||
if getattr(server_args, attr) == "auto"
|
||||
)
|
||||
if not needs:
|
||||
return
|
||||
|
||||
from sglang.srt.utils.hf_transformers_utils import get_tokenizer
|
||||
@@ -741,6 +745,8 @@ def resolve_auto_parsers(server_args) -> None:
|
||||
ctx = build_detection_context(
|
||||
template, tokenizer, reasoning_config, force_reasoning
|
||||
)
|
||||
|
||||
detected: Dict[str, Optional[str]] = {}
|
||||
if ctx is None:
|
||||
if has_explicit_template_without_detection:
|
||||
logger.warning(
|
||||
@@ -750,38 +756,26 @@ def resolve_auto_parsers(server_args) -> None:
|
||||
)
|
||||
else:
|
||||
try:
|
||||
_resolve_architecture_auto_parsers(server_args)
|
||||
detected.update(_architecture_auto_parsers(server_args, needs))
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to load model config for architecture-based auto-detection: %s",
|
||||
e,
|
||||
)
|
||||
if needs_reasoning:
|
||||
if server_args.reasoning_parser == "auto":
|
||||
_disable_auto_parser(
|
||||
server_args, "reasoning_parser", "reasoning parser"
|
||||
)
|
||||
if needs_tool_call:
|
||||
if server_args.tool_call_parser == "auto":
|
||||
_disable_auto_parser(
|
||||
server_args, "tool_call_parser", "tool-call parser"
|
||||
)
|
||||
return
|
||||
for attr, label in (
|
||||
("reasoning_parser", "reasoning parser"),
|
||||
("tool_call_parser", "tool-call parser"),
|
||||
):
|
||||
if attr in needs and attr not in detected:
|
||||
_log_undetected_parser(attr, label)
|
||||
detected[attr] = None
|
||||
else:
|
||||
for attr, rules, label in (
|
||||
("reasoning_parser", REASONING_PARSER_RULES, "reasoning parser"),
|
||||
("tool_call_parser", TOOL_CALL_PARSER_RULES, "tool-call parser"),
|
||||
):
|
||||
if attr in needs:
|
||||
detected[attr] = _detect_auto_parser(attr, ctx, rules, label)
|
||||
|
||||
if needs_reasoning:
|
||||
_resolve_auto_parser(
|
||||
server_args,
|
||||
"reasoning_parser",
|
||||
ctx,
|
||||
REASONING_PARSER_RULES,
|
||||
"reasoning parser",
|
||||
)
|
||||
|
||||
if needs_tool_call:
|
||||
_resolve_auto_parser(
|
||||
server_args,
|
||||
"tool_call_parser",
|
||||
ctx,
|
||||
TOOL_CALL_PARSER_RULES,
|
||||
"tool-call parser",
|
||||
)
|
||||
if detected:
|
||||
declare_late_resolution(server_args, "template-detection", **detected)
|
||||
|
||||
@@ -1011,8 +1011,7 @@ class _ServerArgsOverride:
|
||||
|
||||
def install(self) -> ServerArgs:
|
||||
"""Publish a fresh dummy-boundary ``ServerArgs`` carrying the
|
||||
overrides (written through ``ServerArgs.override`` for provenance);
|
||||
returns the published instance."""
|
||||
overrides; returns the published instance."""
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
assert not self._installed, "override_server_args already installed"
|
||||
@@ -1025,7 +1024,7 @@ class _ServerArgsOverride:
|
||||
self._prev_capture = ctx.flags.capture.enable_torch_compile
|
||||
server_args = ServerArgs(model_path="dummy")
|
||||
if self._fields:
|
||||
server_args.override(source="test-override", **self._fields)
|
||||
server_args = server_args.derive("test-override", **self._fields)
|
||||
# The dummy boundary skips materialization, which would leave the
|
||||
# strict mutation guard unarmed on the published object — mark it
|
||||
# materialized so bare post-publish writes raise like they do on a
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import dataclasses
|
||||
import glob
|
||||
import importlib
|
||||
@@ -8547,56 +8548,73 @@ class ServerArgs:
|
||||
|
||||
return resolved_view(self)
|
||||
|
||||
def override(self, source: str, **fields) -> None:
|
||||
"""The single post-resolution mutation point.
|
||||
def _late_resolution(self, source: str, **fields) -> None:
|
||||
"""Resolve fields at the launcher's validation stage (pre-publish).
|
||||
|
||||
After ``__post_init__`` the configuration is resolved; the audited
|
||||
runtime adjustments (load-resolved values, control-plane
|
||||
reconfiguration, deployment wiring) go through here instead of
|
||||
assigning fields. Whitelisted resolvable fields also join the
|
||||
declaration stash, so a republish resolves the same values;
|
||||
everything is recorded with its ``source`` for provenance.
|
||||
See ``arg_groups.overrides.declare_late_resolution``: in place, because
|
||||
every holder of this instance must see the resolved value, and refused
|
||||
outright once the config is published.
|
||||
"""
|
||||
from sglang.srt.arg_groups.overrides import declare_late_resolution
|
||||
|
||||
declare_late_resolution(self, source, **fields)
|
||||
|
||||
def derive(self, source: str, **fields) -> ServerArgs:
|
||||
"""A copy carrying variant values: a draft worker's context length, an
|
||||
encode worker's device, a launcher's late port pick.
|
||||
|
||||
The receiver is untouched, so a config already published from it -- and
|
||||
the namespace bags projected out of it -- stay true; the variant is a
|
||||
second config, to be published in its own right or handed to whoever
|
||||
owns it. Resolution does not re-run: the values being set are decided
|
||||
*after* it, from inputs resolution never had, and re-resolving a
|
||||
resolved config re-derives conditional decisions from the wrong ones.
|
||||
|
||||
Whitelisted resolvable fields also join the copy's declaration stash so
|
||||
a later re-resolution keeps them; ``source`` is recorded for provenance.
|
||||
"""
|
||||
from sglang.srt.arg_groups.arg_utils import resolvable_fields
|
||||
|
||||
whitelist = resolvable_fields(type(self))
|
||||
variant = copy.deepcopy(self)
|
||||
whitelist = resolvable_fields(type(variant))
|
||||
declared = {k: v for k, v in fields.items() if k in whitelist}
|
||||
rest = {k: v for k, v in fields.items() if k not in whitelist}
|
||||
if declared:
|
||||
stash = getattr(self, "_resolved_overrides", None)
|
||||
stash = getattr(variant, "_resolved_overrides", None)
|
||||
if stash is None:
|
||||
stash = []
|
||||
object.__setattr__(self, "_resolved_overrides", stash)
|
||||
object.__setattr__(variant, "_resolved_overrides", stash)
|
||||
stash.append((source, dict(declared)))
|
||||
if rest:
|
||||
log = getattr(self, "_runtime_mutations", None)
|
||||
log = getattr(variant, "_runtime_mutations", None)
|
||||
if log is None:
|
||||
log = []
|
||||
object.__setattr__(self, "_runtime_mutations", log)
|
||||
object.__setattr__(variant, "_runtime_mutations", log)
|
||||
log.append((source, dict(rest)))
|
||||
object.__setattr__(self, "_in_override", True)
|
||||
object.__setattr__(variant, "_internal_write", True)
|
||||
try:
|
||||
for field, value in fields.items():
|
||||
setattr(self, field, value)
|
||||
setattr(variant, field, value)
|
||||
finally:
|
||||
object.__setattr__(self, "_in_override", False)
|
||||
object.__setattr__(variant, "_internal_write", False)
|
||||
return variant
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
# after materialization the fields are the resolved startup
|
||||
# configuration -- the pristine, READ-ONLY record. A bare assignment
|
||||
# outside ServerArgs.override() (and the resolution pipeline, which runs
|
||||
# before materialization) always raises; resolved config is mutated on
|
||||
# the context bags via get_context().override(...), not here. (Formerly
|
||||
# gated on SGLANG_STRICT_CONFIG_MUTATION; now unconditional.)
|
||||
# After materialization the fields are the resolved startup
|
||||
# configuration -- the pristine, READ-ONLY record that the config bags
|
||||
# were projected from. Resolved config changes go to the bags via
|
||||
# get_context().override(source, ...); a config that differs per runner
|
||||
# or per worker is a separate object, built with derive().
|
||||
if (
|
||||
not name.startswith("_")
|
||||
and getattr(self, "_declarations_materialized", False)
|
||||
and not getattr(self, "_in_override", False)
|
||||
and not getattr(self, "_internal_write", False)
|
||||
):
|
||||
raise AttributeError(
|
||||
f"server_args.{name} assigned after resolution; server_args is "
|
||||
"read-only -- use get_context().override(source, ...) to change "
|
||||
"resolved config."
|
||||
"resolved config, or server_args.derive(source, ...) to build a "
|
||||
"variant for one runner."
|
||||
)
|
||||
object.__setattr__(self, name, value)
|
||||
|
||||
@@ -8914,7 +8932,7 @@ class ServerArgs:
|
||||
# Enable LoRA if any LoRA paths are provided for backward compatibility.
|
||||
if self.lora_paths:
|
||||
if self.enable_lora is None:
|
||||
self.override("check_lora_server_args", enable_lora=True)
|
||||
self._late_resolution("check_lora_server_args", enable_lora=True)
|
||||
logger.warning(
|
||||
"--enable-lora is set to True because --lora-paths is provided."
|
||||
)
|
||||
@@ -8925,7 +8943,7 @@ class ServerArgs:
|
||||
|
||||
if self.enable_lora:
|
||||
if self.enable_lora_overlap_loading is None:
|
||||
self.override(
|
||||
self._late_resolution(
|
||||
"check_lora_server_args", enable_lora_overlap_loading=False
|
||||
)
|
||||
|
||||
@@ -8984,9 +9002,11 @@ class ServerArgs:
|
||||
"Expected a string or a dictionary."
|
||||
)
|
||||
parsed_lora_paths.append(lora_ref)
|
||||
self.override("check_lora_server_args", lora_paths=parsed_lora_paths)
|
||||
self._late_resolution(
|
||||
"check_lora_server_args", lora_paths=parsed_lora_paths
|
||||
)
|
||||
elif isinstance(self.lora_paths, dict):
|
||||
self.override(
|
||||
self._late_resolution(
|
||||
"check_lora_server_args",
|
||||
lora_paths=[
|
||||
LoRARef(
|
||||
@@ -8999,7 +9019,7 @@ class ServerArgs:
|
||||
],
|
||||
)
|
||||
elif self.lora_paths is None:
|
||||
self.override("check_lora_server_args", lora_paths=[])
|
||||
self._late_resolution("check_lora_server_args", lora_paths=[])
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid type for --lora-paths: {type(self.lora_paths)}. "
|
||||
@@ -9009,7 +9029,7 @@ class ServerArgs:
|
||||
# Normalize target modules to a set; keep {"all"} as a sentinel
|
||||
# that gets resolved model-awarely in lora_manager.init_lora_shapes().
|
||||
if self.lora_target_modules:
|
||||
self.override(
|
||||
self._late_resolution(
|
||||
"check_lora_server_args",
|
||||
lora_target_modules=set(self.lora_target_modules),
|
||||
)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from copy import deepcopy
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import msgspec
|
||||
@@ -43,11 +42,11 @@ class DraftWorkerBundle(msgspec.Struct, frozen=True):
|
||||
|
||||
|
||||
def _resolve_draft_attention_backend_fallback(
|
||||
*, draft_server_args: ServerArgs, algo_label: str
|
||||
*, server_args: ServerArgs, algo_label: str
|
||||
) -> str:
|
||||
draft_backend = draft_server_args.speculative_draft_attention_backend
|
||||
draft_backend = server_args.speculative_draft_attention_backend
|
||||
if draft_backend is None:
|
||||
draft_backend, _ = draft_server_args.get_attention_backends()
|
||||
draft_backend, _ = server_args.get_attention_backends()
|
||||
if draft_backend is None:
|
||||
return "triton" if torch.version.hip else "flashinfer"
|
||||
if draft_backend not in _SUPPORTED_DRAFT_BACKENDS:
|
||||
@@ -111,8 +110,7 @@ def draft_server_args_copy(server_args: ServerArgs, target_model_config) -> Serv
|
||||
for _source, fields in get_context().overrides_log():
|
||||
resolved.update(fields)
|
||||
|
||||
draft_server_args = deepcopy(server_args)
|
||||
draft_server_args.override(
|
||||
return server_args.derive(
|
||||
"draft_worker.copy",
|
||||
**{
|
||||
**resolved,
|
||||
@@ -120,7 +118,6 @@ def draft_server_args_copy(server_args: ServerArgs, target_model_config) -> Serv
|
||||
**_draft_load_format_fields(),
|
||||
},
|
||||
)
|
||||
return draft_server_args
|
||||
|
||||
|
||||
def build_draft_tp_worker(
|
||||
@@ -133,18 +130,15 @@ def build_draft_tp_worker(
|
||||
algo_label: str,
|
||||
attention_backend_override: Optional[str] = None,
|
||||
) -> DraftWorkerBundle:
|
||||
draft_server_args = deepcopy(server_args)
|
||||
# An override names a draft-specific backend the caller has already
|
||||
# validated (e.g. a self-drafting architecture); it skips the generic
|
||||
# supported-backend fallback below.
|
||||
draft_backend = attention_backend_override or (
|
||||
_resolve_draft_attention_backend_fallback(
|
||||
draft_server_args=draft_server_args, algo_label=algo_label
|
||||
server_args=server_args, algo_label=algo_label
|
||||
)
|
||||
)
|
||||
# Post-resolution ServerArgs rejects bare assignment; route the draft-copy
|
||||
# adjustments through the audited mutation point.
|
||||
draft_server_args.override(
|
||||
draft_server_args = server_args.derive(
|
||||
"draft_worker.build",
|
||||
**draft_server_args_overrides(target_model_config, draft_backend),
|
||||
)
|
||||
|
||||
@@ -295,7 +295,6 @@ def _configure_runner_for_eagle_draft(
|
||||
*,
|
||||
speculative_attention_mode: str = "decode",
|
||||
) -> None:
|
||||
server_args = runner.server_args
|
||||
updates = {
|
||||
"attention_backend": case.backend,
|
||||
"cuda_graph_config": CudaGraphConfig(
|
||||
@@ -326,11 +325,12 @@ def _configure_runner_for_eagle_draft(
|
||||
"torch_compile_max_bs": 0,
|
||||
"use_mla_backend": runner.use_mla_backend,
|
||||
}
|
||||
server_args.override(source="attention-unittest-eagle-draft", **updates)
|
||||
# Re-publish so the bags pick up the overrides.
|
||||
from sglang.srt.runtime_context import get_context
|
||||
|
||||
get_context().set_server_args(server_args)
|
||||
runner.server_args = runner.server_args.derive(
|
||||
"attention-unittest-eagle-draft", **updates
|
||||
)
|
||||
get_context().set_server_args(runner.server_args)
|
||||
|
||||
runner.spec_algorithm = SpeculativeAlgorithm.EAGLE
|
||||
runner.is_draft_worker = True
|
||||
@@ -391,11 +391,11 @@ def _build_frozen_kv_mtp_fixture(
|
||||
runner_batch_size=settings.capture_batch_size,
|
||||
)
|
||||
_configure_runner_for_eagle_draft(fixture.runner, case, settings)
|
||||
fixture.runner.server_args.override(
|
||||
"attention_unittest.frozen_kv_draft", speculative_algorithm="FROZEN_KV_MTP"
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context
|
||||
|
||||
fixture.runner.server_args = fixture.runner.server_args.derive(
|
||||
"attention_unittest.frozen_kv_draft", speculative_algorithm="FROZEN_KV_MTP"
|
||||
)
|
||||
get_context().set_server_args(fixture.runner.server_args)
|
||||
fixture.runner.spec_algorithm = SpeculativeAlgorithm.FROZEN_KV_MTP
|
||||
fixture.runner.draft_attn_backend = fixture.backend
|
||||
|
||||
@@ -9,10 +9,6 @@ import inspect
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
# Registered tests run with the strict config-mutation guard: bare
|
||||
# server_args assignments after resolution raise (use ServerArgs.override).
|
||||
os.environ.setdefault("SGLANG_STRICT_CONFIG_MUTATION", "1")
|
||||
import random
|
||||
import re
|
||||
import shlex
|
||||
|
||||
Reference in New Issue
Block a user