[refactor] Resolve config declarations onto server_args at the end of __post_init__ (#30297)

This commit is contained in:
Cheng Wan
2026-07-06 18:04:44 -07:00
committed by GitHub
parent cf4edda956
commit c861896721
11 changed files with 722 additions and 393 deletions
+22 -7
View File
@@ -43,21 +43,30 @@ def _hisparse_allowed_backends(kv_cache_dtype: str) -> set[str]:
def validate_hisparse_dsa_backend(
server_args: ServerArgs, attr: str, label: str
) -> None:
backend = getattr(server_args, attr)
allowed_backends = _hisparse_allowed_backends(server_args.kv_cache_dtype)
from sglang.srt.arg_groups.overrides import resolved_view
# Invoked after the DSA kv-cache-dtype / split-backend declarations:
# read the resolving state through the view.
view = resolved_view(server_args)
backend = getattr(view, attr)
kv_cache_dtype = view.kv_cache_dtype
allowed_backends = _hisparse_allowed_backends(kv_cache_dtype)
if backend is not None and backend not in allowed_backends:
raise ValueError(
f"HiSparse supports DSA {label} backend(s) {sorted(allowed_backends)} "
f"on this platform with --kv-cache-dtype={server_args.kv_cache_dtype}, "
f"on this platform with --kv-cache-dtype={kv_cache_dtype}, "
f"but got --dsa-{label}-backend={backend}. "
f"Please use --dsa-{label}-backend="
f"{_hisparse_default_backend(server_args.kv_cache_dtype)} "
f"{_hisparse_default_backend(kv_cache_dtype)} "
"or omit it."
)
def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
if server_args.kv_cache_dtype in HISPARSE_KV_CACHE_DTYPES:
from sglang.srt.arg_groups.overrides import resolved_view
kv_cache_dtype = resolved_view(server_args).kv_cache_dtype
if kv_cache_dtype in HISPARSE_KV_CACHE_DTYPES:
return
choices = " or ".join(
@@ -65,7 +74,7 @@ def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
)
raise ValueError(
f"HiSparse requires one of {HISPARSE_KV_CACHE_DTYPES} KV cache dtypes, "
f"but got --kv-cache-dtype={server_args.kv_cache_dtype}. Please use {choices}."
f"but got --kv-cache-dtype={kv_cache_dtype}. Please use {choices}."
)
@@ -112,7 +121,13 @@ def validate_hisparse(server_args: ServerArgs) -> None:
)
return
if server_args.kv_cache_dtype not in ("bfloat16", "auto", "fp8_e4m3"):
from sglang.srt.arg_groups.overrides import resolved_view
if resolved_view(server_args).kv_cache_dtype not in (
"bfloat16",
"auto",
"fp8_e4m3",
):
validate_hisparse_kv_cache_dtype(server_args)
for attr, label in [
+218 -57
View File
@@ -117,14 +117,10 @@ def _invoke_provider(
class ResolvedView:
"""Read-only view of the resolving configuration handed to post-process
passes.
During the dual-apply transition the view forwards every read to the live
``server_args`` — the pristine input plus the declarations replayed so far
plus any residual imperative writes — which is exactly the state the
legacy handler at the same slot observed. In the end state (dual-apply
retired) the same type overlays the accumulated declarations on the
pristine object. Writes are rejected: passes return declarations.
passes: the accumulated declarations overlaid on the pristine
``server_args`` (residual imperative writes of non-resolved fields show
through the fallthrough) — exactly the state the legacy handler at the
same slot observed. Writes are rejected: passes return declarations.
"""
__slots__ = ("_server_args", "_overlay")
@@ -162,14 +158,27 @@ def register_post_process(fn: Callable[..., dict]) -> Callable[..., dict]:
return fn
def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
"""Transition-period invocation of one pass at its legacy handler slot.
def _declaration_overlay(server_args: Any) -> Dict[str, Any]:
"""Accumulated declared values: declarations never mutate
``server_args``, so mid-resolution readers overlay them from the
declaration stash (last writer wins, like the gate)."""
overlay: Dict[str, Any] = {}
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
overlay.update(declared)
return overlay
Evaluates the pass on the live state (through a read-only view), appends
its declaration to the declaration stash, and dual-applies it in place —
byte-identical to the imperative handler write this replaces.
def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
"""Invoke one pass at its legacy handler slot.
Evaluates the pass on the resolving state (a read-only view with the
accumulated declarations overlaid from the stash) and appends its
declaration to the stash. During ``__post_init__`` the fields stay
untouched — ``materialize_declarations`` applies the whole stash once at
the end of resolution; a pass invoked after materialization (a post-init
slot) writes through immediately.
"""
declared = fn(ResolvedView(server_args))
declared = fn(ResolvedView(server_args, overlay=_declaration_overlay(server_args)))
if not isinstance(declared, dict):
raise TypeError(
f"post-process pass {fn.__qualname__} must return a dict, "
@@ -186,19 +195,70 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
# must sit at or after it in __post_init__ order.
stash = server_args._resolved_overrides = []
stash.append(entry)
apply_declarations_to_server_args(server_args, [entry])
validate_declarations(server_args, [entry])
if getattr(server_args, "_declarations_materialized", False):
for field, value in declared.items():
setattr(server_args, field, value)
def materialize_declarations(server_args: Any) -> None:
"""Apply the accumulated declarations onto ``server_args`` once, at the
end of ``__post_init__`` (gate order: last writer wins). After this the
fields carry the resolved configuration — every post-init reader, in any
process, reads them directly; ``resolved_view`` remains an internal
helper for mid-resolution code only."""
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
for field, value in declared.items():
setattr(server_args, field, value)
server_args._declarations_materialized = True
def resolved_view(server_args: Any) -> ResolvedView:
"""Read-only view of the resolving configuration for mid-resolution code
that is not a pass (``__post_init__`` handlers and hooks). Internal to
the resolution pipeline: after ``materialize_declarations`` runs, the
fields themselves carry the resolved values — read them directly."""
return ResolvedView(server_args, overlay=_declaration_overlay(server_args))
def attention_backends_of(cfg: Any) -> tuple:
"""(prefill, decode) attention backends of a config-shaped object (a
ResolvedView mid-resolution, or pristine server_args at dispatch time):
split fields fall back to the base backend."""
prefill = (
cfg.prefill_attention_backend
if cfg.prefill_attention_backend
else cfg.attention_backend
)
decode = (
cfg.decode_attention_backend
if cfg.decode_attention_backend
else cfg.attention_backend
)
return prefill, decode
def mamba_extra_buffer_of(cfg: Any) -> bool:
"""Mid-resolution equivalent of runtime_context.mamba_extra_buffer_enabled:
reads the (possibly overlaid) strategy from a config-shaped object."""
return cfg.disable_radix_cache is False and cfg.mamba_radix_cache_strategy in (
"extra_buffer",
"extra_buffer_lazy",
)
def declare_load_time_override(source: str, declared: Dict[str, Any]) -> None:
"""Transition helper for load-time resolved fields (model-file config
overrides, weight-resolved dtypes): dual-apply the declaration onto the
published ``server_args`` — byte-identical to the imperative write this
replaces — and record it into the flags tier through the runtime gate."""
"""Declare a load-time resolved field (model-file config overrides,
weight-resolved dtypes): apply it onto the published ``server_args`` —
resolution has already materialized, so post-init declarations write
through — and record it into the flags tier through the runtime gate."""
from sglang.srt.runtime_context import get_context
ctx = get_context()
entry = (source, dict(declared))
apply_declarations_to_server_args(ctx.server_args, [entry])
validate_declarations(ctx.server_args, [entry])
for field, value in declared.items():
setattr(ctx.server_args, field, value)
ctx.record_runtime_overrides([entry])
@@ -741,8 +801,11 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict:
use_mla_backend=server_args.use_mla_backend(),
model_config=server_args.get_model_config(),
)
# The mamba radix-cache pass runs before this dispatch: read the
# declared strategy through the view (the legacy branch observed the
# already-written field here).
if default_attn_backend == "trtllm_mha" and not (
not server_args.enable_mamba_extra_buffer()
not mamba_extra_buffer_of(resolved_view(server_args))
and not server_args.disable_radix_cache
and server_args.speculative_algorithm is None
):
@@ -1519,12 +1582,57 @@ def _mla_backend_page_constraints(view: Any) -> dict:
return {}
@register_post_process
def _mla_kv_cache_dtype_checks(view: Any) -> dict:
"""Read-only validation pass in the attention-backend compatibility
handler: the TRT-LLM and tokenspeed MLA backends constrain the resolved
kv-cache dtype (declarations never reach the field, so the checks read
the view)."""
if (
view.attention_backend == "trtllm_mla"
or view.decode_attention_backend == "trtllm_mla"
):
if not is_blackwell_supported():
raise ValueError(
"TRTLLM MLA backend is only supported on Blackwell GPUs (SM100/SM12x). Please use a different backend."
)
if view.kv_cache_dtype not in ["fp8_e4m3", "fp4_e2m1", "bf16", "auto"]:
raise ValueError(
"TensorRT-LLM MLA backend only supports kv-cache-dtype of fp8_e4m3, fp4_e2m1, bf16, or auto."
)
if (
view.attention_backend == "tokenspeed_mla"
or view.decode_attention_backend == "tokenspeed_mla"
):
if not is_blackwell_supported():
raise ValueError(
"tokenspeed_mla backend is only supported on Blackwell GPUs (SM100/SM12x)."
)
if view.kv_cache_dtype not in ["fp8_e4m3"]:
raise ValueError(
"tokenspeed_mla backend requires kv-cache-dtype=fp8_e4m3, "
f"got {view.kv_cache_dtype}."
)
return {}
@register_post_process
def _hisparse_validation(view: Any) -> dict:
"""Read-only validation pass: --enable-hisparse constraints (model class,
radix cache, kv dtype, DSA backends) read the resolved values through the
view."""
from sglang.srt.arg_groups.hisparse_hook import validate_hisparse
validate_hisparse(view)
return {}
@register_post_process
def _cutedsl_prefill_backend_fill(view: Any) -> dict:
"""Slot pass in the attention-backend compatibility handler: CuteDSL MLA
is decode-only, so validate the combination and default the prefill side
to trtllm_mla. The trtllm_mha check that follows at the legacy slot reads
the dual-applied value."""
the resolved value through the view."""
if not (
view.attention_backend == "cutedsl_mla"
or view.decode_attention_backend == "cutedsl_mla"
@@ -1610,7 +1718,7 @@ def _attention_backend_platform_fallbacks(view: Any) -> dict:
@register_post_process
def _intel_xpu_page_constraint(view: Any) -> dict:
_, decode_backend = view.get_attention_backends()
_, decode_backend = attention_backends_of(view)
if decode_backend == "intel_xpu":
if view.use_mla_backend():
supported_page_sizes = [16, 32, 64, 128]
@@ -1674,6 +1782,17 @@ def _data_parallelism_defaults(view: Any) -> dict:
return {}
@register_post_process
def _dp_lm_head_validation(view: Any) -> dict:
"""Read-only validation pass: dp-attention is a prerequisite for the
dp LM head. Reads the mid-resolution values through the view."""
if view.enable_dp_lm_head:
assert (
view.enable_dp_attention
), "Please enable dp attention when setting enable_dp_lm_head. "
return {}
@register_post_process
def _moe_runner_backend_quant_constraints(view: Any) -> dict:
"""The quantization-driven moe_runner_backend resolutions at the head of
@@ -1728,6 +1847,49 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
@register_post_process
def _moe_runner_fusion_disable(view: Any) -> dict:
"""FlashInfer CuteDSL / TRT-LLM / TRT-LLM-routed MoE runners require the
shared-experts fusion disabled; declared at the legacy write slots in
_handle_moe_kernel_config (before the deprecated cutlass env override, so
the runner value observed is the pre-override one)."""
runner = view.moe_runner_backend
if runner == "flashinfer_cutedsl":
logger.warning(
"FlashInfer CuteDSL MoE is enabled. --disable-shared-experts-fusion is automatically set."
)
return {"disable_shared_experts_fusion": True}
if runner in ("flashinfer_trtllm", "experimental_sgl_trtllm"):
logger.warning(
"FlashInfer TRTLLM MoE is enabled. --disable-shared-experts-fusion is automatically set."
)
return {"disable_shared_experts_fusion": True}
if runner == "flashinfer_trtllm_routed":
logger.warning(
"FlashInfer TRTLLM routed MoE is enabled. --disable-shared-experts-fusion is automatically set."
)
return {"disable_shared_experts_fusion": True}
return {}
def _a2a_fusion_adjustments(view: Any) -> dict:
"""A2A-backend-driven shared-experts fusion adjustments, declared at the
legacy write slots in _handle_a2a_moe: DeepEP Waterfill requires the
fusion enabled; FlashInfer A2A requires it disabled."""
if view.moe_a2a_backend == "deepep" and view.enable_deepep_waterfill:
if view.disable_shared_experts_fusion:
logger.warning(
"disable_shared_experts_fusion is overridden to False because DeepEP Waterfill requires shared expert fusion."
)
return {"disable_shared_experts_fusion": False}
return {}
if view.moe_a2a_backend == "flashinfer":
logger.warning(
"Flashinfer MoE A2A is enabled. --disable-shared-experts-fusion is automatically set."
)
return {"disable_shared_experts_fusion": True}
return {}
def _cutlass_moe_env_override(view: Any) -> dict:
from sglang.srt.environ import envs
@@ -1846,16 +2008,25 @@ def _dllm_overlap_disable(view: Any) -> dict:
@register_post_process
def _dllm_page_size(view: Any) -> dict:
if view.dllm_algorithm is None or view.disable_radix_cache:
if view.dllm_algorithm is None:
return {}
from sglang.srt.dllm.config import DllmConfig
config = DllmConfig.from_server_args(view)
if view.page_size % config.block_size != 0:
if not view.disable_radix_cache and view.page_size % config.block_size != 0:
logger.warning(
f"Setting page size to {config.block_size} for diffusion LLM inference"
)
return {"page_size": config.block_size}
if view.page_size > config.block_size:
# Legacy scheduler-init fallback, folded into the pass: the page
# size must not exceed the dllm block size.
logger.warning(
"WARNING: "
f"The page size {view.page_size} should not be larger than dllm block size {config.block_size}."
f"Page size now falls back to {config.block_size}"
)
return {"page_size": config.block_size}
return {}
@@ -1937,45 +2108,35 @@ def apply_model_overrides(
return records
def apply_declarations_to_server_args(
def validate_declarations(
server_args: Any,
declarations: Sequence[Tuple[str, Dict[str, Any]]],
*,
terminal: Sequence[Tuple[str, Dict[str, Any]]] = (),
) -> None:
"""Transition-period dual-apply: replay declarations onto ``server_args``
in gate order, byte-identical to the legacy imperative writes.
Retired per field once that field's readers have all flipped to the flags
tier (at which point the server_args field returns to pristine).
Validates against the same whitelist as the publish gate BEFORE any write:
a registry typo or a not-yet-resolvable field must fail fast here, not
mutate ``server_args`` and only be rejected at publish time.
"""Fail-fast whitelist check at declaration time: a registry typo or a
not-yet-resolvable field must be rejected at its slot, not only at
publish time. Declarations never mutate ``server_args``.
"""
# Non-dataclass fixtures carry no Arg metadata (mirrors the
# resolvable_fields escape); only real ServerArgs is validated.
if dataclasses.is_dataclass(type(server_args)):
whitelist = resolvable_fields(type(server_args))
for source, decl in list(declarations) + list(terminal):
unknown = set(decl) - whitelist
if unknown:
raise ValueError(
f"{source}: {sorted(unknown)} not model-overridable; the "
"transition dual-apply refuses fields the publish gate "
"would reject."
)
for _source, decl in list(declarations) + list(terminal):
for field, value in decl.items():
setattr(server_args, field, value)
if not dataclasses.is_dataclass(type(server_args)):
return
whitelist = resolvable_fields(type(server_args))
for source, decl in declarations:
unknown = set(decl) - whitelist
if unknown:
raise ValueError(
f"{source}: {sorted(unknown)} not model-overridable; "
"declarations are limited to the fields the publish gate "
"accepts."
)
def refresh_declared_fields(server_args: Any, fields: Iterable[str]) -> None:
"""Transition helper for legacy code that overwrites a resolved field
AFTER the override collection in ``__post_init__`` (e.g.
``ModelRunner.model_specific_adjustment`` forcing ``attention_backend``
for HRM-Text). Redeclares the live value so publish parity holds and the
flags tier materializes the adjusted end state.
"""Helper for legacy code that overwrites a resolved field AFTER
materialization (e.g. ``ModelRunner.model_specific_adjustment`` forcing
``attention_backend`` for HRM-Text). Redeclares the live value so the
publish parity holds and the flags tier materializes the adjusted end
state.
"""
_missing = object()
declarations = server_args._resolved_overrides
@@ -1998,8 +2159,8 @@ def assert_flag_parity(
*,
leaf_map: Optional[Dict[str, str]] = None,
) -> None:
"""Dual-apply drift guard: each migrated field's flag leaf must equal the
(dual-applied) ``server_args`` value."""
"""Drift guard: each declared field's flag leaf must equal the
(materialized) ``server_args`` value."""
mismatches = []
for field in fields:
owner, leaf = resolve_flag_leaf(flags, field, leaf_map=leaf_map)
@@ -135,7 +135,9 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
def _handle_dflash(server_args: ServerArgs) -> None:
if server_args.enable_dp_attention:
from sglang.srt.arg_groups.overrides import resolved_view
if resolved_view(server_args).enable_dp_attention:
raise ValueError(
"Currently DFLASH speculative decoding does not support dp attention."
)
@@ -264,7 +266,12 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
draft_backend = server_args.speculative_draft_attention_backend
if draft_backend is None:
draft_backend, _ = server_args.get_attention_backends()
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
resolved_view,
)
draft_backend, _ = attention_backends_of(resolved_view(server_args))
if draft_backend is None:
draft_backend = fallback_backend
elif draft_backend == "trtllm_mha":
@@ -305,9 +312,14 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None:
def _handle_eagle_family(server_args: ServerArgs) -> None:
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
resolved_view,
)
if (
server_args.speculative_algorithm == "STANDALONE"
and server_args.enable_dp_attention
and resolved_view(server_args).enable_dp_attention
):
# TODO: support dp attention for standalone speculative decoding
raise ValueError(
@@ -320,7 +332,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests."
)
if server_args.disable_overlap_schedule:
if resolved_view(server_args).disable_overlap_schedule:
logger.warning(
"Non-overlap (synchronous) spec v2 is used for eagle/eagle3/standalone "
"speculative decoding."
@@ -375,11 +387,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
server_args.speculative_num_draft_tokens,
) = _auto_choose_speculative_params(server_args, model_arch)
if (
server_args.attention_backend == "trtllm_mha"
or server_args.decode_attention_backend == "trtllm_mha"
or server_args.prefill_attention_backend == "trtllm_mha"
):
if "trtllm_mha" in attention_backends_of(resolved_view(server_args)):
if server_args.speculative_eagle_topk > 1:
raise ValueError(
"trtllm_mha backend only supports topk = 1 for speculative decoding."
@@ -416,7 +424,9 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"--enable-deterministic-inference; the sampling kernel draws "
"coins from the global RNG and is not batch-invariant."
)
if server_args.enable_multi_layer_eagle:
from sglang.srt.arg_groups.overrides import resolved_view
if resolved_view(server_args).enable_multi_layer_eagle:
raise NotImplementedError(
"--speculative-use-rejection-sampling is not supported with "
"multi-layer EAGLE (--enable-multi-layer-eagle)."
@@ -440,15 +450,16 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
# pass + per-branch expand pass with prefix-tail dup). Only these backends implement
# it; flashmla / trtllm_mla / cutlass_mla can't express the per-branch tree, so reject.
_PAGE_TREE_SPEC_BACKENDS = ("flashinfer", "fa3", "triton")
view = resolved_view(server_args)
if (
server_args.speculative_eagle_topk > 1
and server_args.page_size > 1
and server_args.attention_backend not in _PAGE_TREE_SPEC_BACKENDS
and view.page_size > 1
and view.attention_backend not in _PAGE_TREE_SPEC_BACKENDS
):
raise ValueError(
f"speculative_eagle_topk > 1 with page_size > 1 is only supported on "
f"{_PAGE_TREE_SPEC_BACKENDS}; got attention_backend="
f"{server_args.attention_backend!r}. Use page_size == 1 or one of those backends."
f"{view.attention_backend!r}. Use page_size == 1 or one of those backends."
)
@@ -499,18 +510,21 @@ def _handle_ngram(server_args: ServerArgs) -> None:
"using ngram speculative decoding."
)
from sglang.srt.arg_groups.overrides import resolved_view
view = resolved_view(server_args)
if (
server_args.speculative_eagle_topk > 1
and server_args.page_size > 1
and server_args.attention_backend != "flashinfer"
and view.page_size > 1
and view.attention_backend != "flashinfer"
):
raise ValueError(
f"speculative_eagle_topk({server_args.speculative_eagle_topk}) > 1 "
f"with page_size({server_args.page_size}) > 1 is unstable "
f"with page_size({view.page_size}) > 1 is unstable "
"and produces incorrect results for paged attention backends. "
"This combination is only supported for the 'flashinfer' backend."
)
if server_args.enable_dp_attention:
if view.enable_dp_attention:
# TODO: support dp attention for ngram speculative decoding
raise ValueError(
"Currently ngram speculative decoding does not support dp attention."
File diff suppressed because it is too large Load Diff
@@ -49,6 +49,8 @@ DEFAULT_ADAPTIVE_CONFIG: dict[str, dict] = {
def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
"""Return why adaptive spec cannot run under the given server args, or None if supported."""
from sglang.srt.arg_groups.overrides import resolved_view
if server_args.speculative_algorithm not in ("EAGLE", "EAGLE3"):
return (
f"speculative_algorithm={server_args.speculative_algorithm} "
@@ -62,12 +64,12 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
f"speculative_eagle_topk={server_args.speculative_eagle_topk} "
"(only topk=1 is supported)"
)
if server_args.enable_dp_attention:
if resolved_view(server_args).enable_dp_attention:
return (
"enable_dp_attention=True is not supported "
"(adaptive tier decisions are not synchronized across DP ranks)"
)
if server_args.enable_multi_layer_eagle:
if resolved_view(server_args).enable_multi_layer_eagle:
return (
"enable_multi_layer_eagle=True is not supported "
"(MultiLayerEagleWorkerV2 does not implement adaptive)"