[refactor] Resolve config declarations onto server_args at the end of __post_init__ (#30297)

This commit is contained in:
Cheng Wan
2026-07-06 18:04:44 -07:00
committed by GitHub
parent cf4edda956
commit c861896721
11 changed files with 722 additions and 393 deletions
+22 -7
View File
@@ -43,21 +43,30 @@ def _hisparse_allowed_backends(kv_cache_dtype: str) -> set[str]:
def validate_hisparse_dsa_backend( def validate_hisparse_dsa_backend(
server_args: ServerArgs, attr: str, label: str server_args: ServerArgs, attr: str, label: str
) -> None: ) -> None:
backend = getattr(server_args, attr) from sglang.srt.arg_groups.overrides import resolved_view
allowed_backends = _hisparse_allowed_backends(server_args.kv_cache_dtype)
# Invoked after the DSA kv-cache-dtype / split-backend declarations:
# read the resolving state through the view.
view = resolved_view(server_args)
backend = getattr(view, attr)
kv_cache_dtype = view.kv_cache_dtype
allowed_backends = _hisparse_allowed_backends(kv_cache_dtype)
if backend is not None and backend not in allowed_backends: if backend is not None and backend not in allowed_backends:
raise ValueError( raise ValueError(
f"HiSparse supports DSA {label} backend(s) {sorted(allowed_backends)} " f"HiSparse supports DSA {label} backend(s) {sorted(allowed_backends)} "
f"on this platform with --kv-cache-dtype={server_args.kv_cache_dtype}, " f"on this platform with --kv-cache-dtype={kv_cache_dtype}, "
f"but got --dsa-{label}-backend={backend}. " f"but got --dsa-{label}-backend={backend}. "
f"Please use --dsa-{label}-backend=" f"Please use --dsa-{label}-backend="
f"{_hisparse_default_backend(server_args.kv_cache_dtype)} " f"{_hisparse_default_backend(kv_cache_dtype)} "
"or omit it." "or omit it."
) )
def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None: def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
if server_args.kv_cache_dtype in HISPARSE_KV_CACHE_DTYPES: from sglang.srt.arg_groups.overrides import resolved_view
kv_cache_dtype = resolved_view(server_args).kv_cache_dtype
if kv_cache_dtype in HISPARSE_KV_CACHE_DTYPES:
return return
choices = " or ".join( choices = " or ".join(
@@ -65,7 +74,7 @@ def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
) )
raise ValueError( raise ValueError(
f"HiSparse requires one of {HISPARSE_KV_CACHE_DTYPES} KV cache dtypes, " f"HiSparse requires one of {HISPARSE_KV_CACHE_DTYPES} KV cache dtypes, "
f"but got --kv-cache-dtype={server_args.kv_cache_dtype}. Please use {choices}." f"but got --kv-cache-dtype={kv_cache_dtype}. Please use {choices}."
) )
@@ -112,7 +121,13 @@ def validate_hisparse(server_args: ServerArgs) -> None:
) )
return return
if server_args.kv_cache_dtype not in ("bfloat16", "auto", "fp8_e4m3"): from sglang.srt.arg_groups.overrides import resolved_view
if resolved_view(server_args).kv_cache_dtype not in (
"bfloat16",
"auto",
"fp8_e4m3",
):
validate_hisparse_kv_cache_dtype(server_args) validate_hisparse_kv_cache_dtype(server_args)
for attr, label in [ for attr, label in [
+213 -52
View File
@@ -117,14 +117,10 @@ def _invoke_provider(
class ResolvedView: class ResolvedView:
"""Read-only view of the resolving configuration handed to post-process """Read-only view of the resolving configuration handed to post-process
passes. passes: the accumulated declarations overlaid on the pristine
``server_args`` (residual imperative writes of non-resolved fields show
During the dual-apply transition the view forwards every read to the live through the fallthrough) — exactly the state the legacy handler at the
``server_args`` — the pristine input plus the declarations replayed so far same slot observed. Writes are rejected: passes return declarations.
plus any residual imperative writes — which is exactly the state the
legacy handler at the same slot observed. In the end state (dual-apply
retired) the same type overlays the accumulated declarations on the
pristine object. Writes are rejected: passes return declarations.
""" """
__slots__ = ("_server_args", "_overlay") __slots__ = ("_server_args", "_overlay")
@@ -162,14 +158,27 @@ def register_post_process(fn: Callable[..., dict]) -> Callable[..., dict]:
return fn return fn
def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None: def _declaration_overlay(server_args: Any) -> Dict[str, Any]:
"""Transition-period invocation of one pass at its legacy handler slot. """Accumulated declared values: declarations never mutate
``server_args``, so mid-resolution readers overlay them from the
declaration stash (last writer wins, like the gate)."""
overlay: Dict[str, Any] = {}
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
overlay.update(declared)
return overlay
Evaluates the pass on the live state (through a read-only view), appends
its declaration to the declaration stash, and dual-applies it in place — def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
byte-identical to the imperative handler write this replaces. """Invoke one pass at its legacy handler slot.
Evaluates the pass on the resolving state (a read-only view with the
accumulated declarations overlaid from the stash) and appends its
declaration to the stash. During ``__post_init__`` the fields stay
untouched — ``materialize_declarations`` applies the whole stash once at
the end of resolution; a pass invoked after materialization (a post-init
slot) writes through immediately.
""" """
declared = fn(ResolvedView(server_args)) declared = fn(ResolvedView(server_args, overlay=_declaration_overlay(server_args)))
if not isinstance(declared, dict): if not isinstance(declared, dict):
raise TypeError( raise TypeError(
f"post-process pass {fn.__qualname__} must return a dict, " f"post-process pass {fn.__qualname__} must return a dict, "
@@ -186,19 +195,70 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
# must sit at or after it in __post_init__ order. # must sit at or after it in __post_init__ order.
stash = server_args._resolved_overrides = [] stash = server_args._resolved_overrides = []
stash.append(entry) stash.append(entry)
apply_declarations_to_server_args(server_args, [entry]) validate_declarations(server_args, [entry])
if getattr(server_args, "_declarations_materialized", False):
for field, value in declared.items():
setattr(server_args, field, value)
def materialize_declarations(server_args: Any) -> None:
"""Apply the accumulated declarations onto ``server_args`` once, at the
end of ``__post_init__`` (gate order: last writer wins). After this the
fields carry the resolved configuration — every post-init reader, in any
process, reads them directly; ``resolved_view`` remains an internal
helper for mid-resolution code only."""
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
for field, value in declared.items():
setattr(server_args, field, value)
server_args._declarations_materialized = True
def resolved_view(server_args: Any) -> ResolvedView:
"""Read-only view of the resolving configuration for mid-resolution code
that is not a pass (``__post_init__`` handlers and hooks). Internal to
the resolution pipeline: after ``materialize_declarations`` runs, the
fields themselves carry the resolved values — read them directly."""
return ResolvedView(server_args, overlay=_declaration_overlay(server_args))
def attention_backends_of(cfg: Any) -> tuple:
"""(prefill, decode) attention backends of a config-shaped object (a
ResolvedView mid-resolution, or pristine server_args at dispatch time):
split fields fall back to the base backend."""
prefill = (
cfg.prefill_attention_backend
if cfg.prefill_attention_backend
else cfg.attention_backend
)
decode = (
cfg.decode_attention_backend
if cfg.decode_attention_backend
else cfg.attention_backend
)
return prefill, decode
def mamba_extra_buffer_of(cfg: Any) -> bool:
"""Mid-resolution equivalent of runtime_context.mamba_extra_buffer_enabled:
reads the (possibly overlaid) strategy from a config-shaped object."""
return cfg.disable_radix_cache is False and cfg.mamba_radix_cache_strategy in (
"extra_buffer",
"extra_buffer_lazy",
)
def declare_load_time_override(source: str, declared: Dict[str, Any]) -> None: def declare_load_time_override(source: str, declared: Dict[str, Any]) -> None:
"""Transition helper for load-time resolved fields (model-file config """Declare a load-time resolved field (model-file config overrides,
overrides, weight-resolved dtypes): dual-apply the declaration onto the weight-resolved dtypes): apply it onto the published ``server_args`` —
published ``server_args`` — byte-identical to the imperative write this resolution has already materialized, so post-init declarations write
replaces — and record it into the flags tier through the runtime gate.""" through — and record it into the flags tier through the runtime gate."""
from sglang.srt.runtime_context import get_context from sglang.srt.runtime_context import get_context
ctx = get_context() ctx = get_context()
entry = (source, dict(declared)) entry = (source, dict(declared))
apply_declarations_to_server_args(ctx.server_args, [entry]) validate_declarations(ctx.server_args, [entry])
for field, value in declared.items():
setattr(ctx.server_args, field, value)
ctx.record_runtime_overrides([entry]) ctx.record_runtime_overrides([entry])
@@ -741,8 +801,11 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict:
use_mla_backend=server_args.use_mla_backend(), use_mla_backend=server_args.use_mla_backend(),
model_config=server_args.get_model_config(), model_config=server_args.get_model_config(),
) )
# The mamba radix-cache pass runs before this dispatch: read the
# declared strategy through the view (the legacy branch observed the
# already-written field here).
if default_attn_backend == "trtllm_mha" and not ( if default_attn_backend == "trtllm_mha" and not (
not server_args.enable_mamba_extra_buffer() not mamba_extra_buffer_of(resolved_view(server_args))
and not server_args.disable_radix_cache and not server_args.disable_radix_cache
and server_args.speculative_algorithm is None and server_args.speculative_algorithm is None
): ):
@@ -1519,12 +1582,57 @@ def _mla_backend_page_constraints(view: Any) -> dict:
return {} return {}
@register_post_process
def _mla_kv_cache_dtype_checks(view: Any) -> dict:
"""Read-only validation pass in the attention-backend compatibility
handler: the TRT-LLM and tokenspeed MLA backends constrain the resolved
kv-cache dtype (declarations never reach the field, so the checks read
the view)."""
if (
view.attention_backend == "trtllm_mla"
or view.decode_attention_backend == "trtllm_mla"
):
if not is_blackwell_supported():
raise ValueError(
"TRTLLM MLA backend is only supported on Blackwell GPUs (SM100/SM12x). Please use a different backend."
)
if view.kv_cache_dtype not in ["fp8_e4m3", "fp4_e2m1", "bf16", "auto"]:
raise ValueError(
"TensorRT-LLM MLA backend only supports kv-cache-dtype of fp8_e4m3, fp4_e2m1, bf16, or auto."
)
if (
view.attention_backend == "tokenspeed_mla"
or view.decode_attention_backend == "tokenspeed_mla"
):
if not is_blackwell_supported():
raise ValueError(
"tokenspeed_mla backend is only supported on Blackwell GPUs (SM100/SM12x)."
)
if view.kv_cache_dtype not in ["fp8_e4m3"]:
raise ValueError(
"tokenspeed_mla backend requires kv-cache-dtype=fp8_e4m3, "
f"got {view.kv_cache_dtype}."
)
return {}
@register_post_process
def _hisparse_validation(view: Any) -> dict:
"""Read-only validation pass: --enable-hisparse constraints (model class,
radix cache, kv dtype, DSA backends) read the resolved values through the
view."""
from sglang.srt.arg_groups.hisparse_hook import validate_hisparse
validate_hisparse(view)
return {}
@register_post_process @register_post_process
def _cutedsl_prefill_backend_fill(view: Any) -> dict: def _cutedsl_prefill_backend_fill(view: Any) -> dict:
"""Slot pass in the attention-backend compatibility handler: CuteDSL MLA """Slot pass in the attention-backend compatibility handler: CuteDSL MLA
is decode-only, so validate the combination and default the prefill side is decode-only, so validate the combination and default the prefill side
to trtllm_mla. The trtllm_mha check that follows at the legacy slot reads to trtllm_mla. The trtllm_mha check that follows at the legacy slot reads
the dual-applied value.""" the resolved value through the view."""
if not ( if not (
view.attention_backend == "cutedsl_mla" view.attention_backend == "cutedsl_mla"
or view.decode_attention_backend == "cutedsl_mla" or view.decode_attention_backend == "cutedsl_mla"
@@ -1610,7 +1718,7 @@ def _attention_backend_platform_fallbacks(view: Any) -> dict:
@register_post_process @register_post_process
def _intel_xpu_page_constraint(view: Any) -> dict: def _intel_xpu_page_constraint(view: Any) -> dict:
_, decode_backend = view.get_attention_backends() _, decode_backend = attention_backends_of(view)
if decode_backend == "intel_xpu": if decode_backend == "intel_xpu":
if view.use_mla_backend(): if view.use_mla_backend():
supported_page_sizes = [16, 32, 64, 128] supported_page_sizes = [16, 32, 64, 128]
@@ -1674,6 +1782,17 @@ def _data_parallelism_defaults(view: Any) -> dict:
return {} return {}
@register_post_process
def _dp_lm_head_validation(view: Any) -> dict:
"""Read-only validation pass: dp-attention is a prerequisite for the
dp LM head. Reads the mid-resolution values through the view."""
if view.enable_dp_lm_head:
assert (
view.enable_dp_attention
), "Please enable dp attention when setting enable_dp_lm_head. "
return {}
@register_post_process @register_post_process
def _moe_runner_backend_quant_constraints(view: Any) -> dict: def _moe_runner_backend_quant_constraints(view: Any) -> dict:
"""The quantization-driven moe_runner_backend resolutions at the head of """The quantization-driven moe_runner_backend resolutions at the head of
@@ -1728,6 +1847,49 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
@register_post_process @register_post_process
def _moe_runner_fusion_disable(view: Any) -> dict:
"""FlashInfer CuteDSL / TRT-LLM / TRT-LLM-routed MoE runners require the
shared-experts fusion disabled; declared at the legacy write slots in
_handle_moe_kernel_config (before the deprecated cutlass env override, so
the runner value observed is the pre-override one)."""
runner = view.moe_runner_backend
if runner == "flashinfer_cutedsl":
logger.warning(
"FlashInfer CuteDSL MoE is enabled. --disable-shared-experts-fusion is automatically set."
)
return {"disable_shared_experts_fusion": True}
if runner in ("flashinfer_trtllm", "experimental_sgl_trtllm"):
logger.warning(
"FlashInfer TRTLLM MoE is enabled. --disable-shared-experts-fusion is automatically set."
)
return {"disable_shared_experts_fusion": True}
if runner == "flashinfer_trtllm_routed":
logger.warning(
"FlashInfer TRTLLM routed MoE is enabled. --disable-shared-experts-fusion is automatically set."
)
return {"disable_shared_experts_fusion": True}
return {}
def _a2a_fusion_adjustments(view: Any) -> dict:
"""A2A-backend-driven shared-experts fusion adjustments, declared at the
legacy write slots in _handle_a2a_moe: DeepEP Waterfill requires the
fusion enabled; FlashInfer A2A requires it disabled."""
if view.moe_a2a_backend == "deepep" and view.enable_deepep_waterfill:
if view.disable_shared_experts_fusion:
logger.warning(
"disable_shared_experts_fusion is overridden to False because DeepEP Waterfill requires shared expert fusion."
)
return {"disable_shared_experts_fusion": False}
return {}
if view.moe_a2a_backend == "flashinfer":
logger.warning(
"Flashinfer MoE A2A is enabled. --disable-shared-experts-fusion is automatically set."
)
return {"disable_shared_experts_fusion": True}
return {}
def _cutlass_moe_env_override(view: Any) -> dict: def _cutlass_moe_env_override(view: Any) -> dict:
from sglang.srt.environ import envs from sglang.srt.environ import envs
@@ -1846,16 +2008,25 @@ def _dllm_overlap_disable(view: Any) -> dict:
@register_post_process @register_post_process
def _dllm_page_size(view: Any) -> dict: def _dllm_page_size(view: Any) -> dict:
if view.dllm_algorithm is None or view.disable_radix_cache: if view.dllm_algorithm is None:
return {} return {}
from sglang.srt.dllm.config import DllmConfig from sglang.srt.dllm.config import DllmConfig
config = DllmConfig.from_server_args(view) config = DllmConfig.from_server_args(view)
if view.page_size % config.block_size != 0: if not view.disable_radix_cache and view.page_size % config.block_size != 0:
logger.warning( logger.warning(
f"Setting page size to {config.block_size} for diffusion LLM inference" f"Setting page size to {config.block_size} for diffusion LLM inference"
) )
return {"page_size": config.block_size} return {"page_size": config.block_size}
if view.page_size > config.block_size:
# Legacy scheduler-init fallback, folded into the pass: the page
# size must not exceed the dllm block size.
logger.warning(
"WARNING: "
f"The page size {view.page_size} should not be larger than dllm block size {config.block_size}."
f"Page size now falls back to {config.block_size}"
)
return {"page_size": config.block_size}
return {} return {}
@@ -1937,45 +2108,35 @@ def apply_model_overrides(
return records return records
def apply_declarations_to_server_args( def validate_declarations(
server_args: Any, server_args: Any,
declarations: Sequence[Tuple[str, Dict[str, Any]]], declarations: Sequence[Tuple[str, Dict[str, Any]]],
*,
terminal: Sequence[Tuple[str, Dict[str, Any]]] = (),
) -> None: ) -> None:
"""Transition-period dual-apply: replay declarations onto ``server_args`` """Fail-fast whitelist check at declaration time: a registry typo or a
in gate order, byte-identical to the legacy imperative writes. not-yet-resolvable field must be rejected at its slot, not only at
publish time. Declarations never mutate ``server_args``.
Retired per field once that field's readers have all flipped to the flags
tier (at which point the server_args field returns to pristine).
Validates against the same whitelist as the publish gate BEFORE any write:
a registry typo or a not-yet-resolvable field must fail fast here, not
mutate ``server_args`` and only be rejected at publish time.
""" """
# Non-dataclass fixtures carry no Arg metadata (mirrors the # Non-dataclass fixtures carry no Arg metadata (mirrors the
# resolvable_fields escape); only real ServerArgs is validated. # resolvable_fields escape); only real ServerArgs is validated.
if dataclasses.is_dataclass(type(server_args)): if not dataclasses.is_dataclass(type(server_args)):
return
whitelist = resolvable_fields(type(server_args)) whitelist = resolvable_fields(type(server_args))
for source, decl in list(declarations) + list(terminal): for source, decl in declarations:
unknown = set(decl) - whitelist unknown = set(decl) - whitelist
if unknown: if unknown:
raise ValueError( raise ValueError(
f"{source}: {sorted(unknown)} not model-overridable; the " f"{source}: {sorted(unknown)} not model-overridable; "
"transition dual-apply refuses fields the publish gate " "declarations are limited to the fields the publish gate "
"would reject." "accepts."
) )
for _source, decl in list(declarations) + list(terminal):
for field, value in decl.items():
setattr(server_args, field, value)
def refresh_declared_fields(server_args: Any, fields: Iterable[str]) -> None: def refresh_declared_fields(server_args: Any, fields: Iterable[str]) -> None:
"""Transition helper for legacy code that overwrites a resolved field """Helper for legacy code that overwrites a resolved field AFTER
AFTER the override collection in ``__post_init__`` (e.g. materialization (e.g. ``ModelRunner.model_specific_adjustment`` forcing
``ModelRunner.model_specific_adjustment`` forcing ``attention_backend`` ``attention_backend`` for HRM-Text). Redeclares the live value so the
for HRM-Text). Redeclares the live value so publish parity holds and the publish parity holds and the flags tier materializes the adjusted end
flags tier materializes the adjusted end state. state.
""" """
_missing = object() _missing = object()
declarations = server_args._resolved_overrides declarations = server_args._resolved_overrides
@@ -1998,8 +2159,8 @@ def assert_flag_parity(
*, *,
leaf_map: Optional[Dict[str, str]] = None, leaf_map: Optional[Dict[str, str]] = None,
) -> None: ) -> None:
"""Dual-apply drift guard: each migrated field's flag leaf must equal the """Drift guard: each declared field's flag leaf must equal the
(dual-applied) ``server_args`` value.""" (materialized) ``server_args`` value."""
mismatches = [] mismatches = []
for field in fields: for field in fields:
owner, leaf = resolve_flag_leaf(flags, field, leaf_map=leaf_map) owner, leaf = resolve_flag_leaf(flags, field, leaf_map=leaf_map)
@@ -135,7 +135,9 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
def _handle_dflash(server_args: ServerArgs) -> None: def _handle_dflash(server_args: ServerArgs) -> None:
if server_args.enable_dp_attention: from sglang.srt.arg_groups.overrides import resolved_view
if resolved_view(server_args).enable_dp_attention:
raise ValueError( raise ValueError(
"Currently DFLASH speculative decoding does not support dp attention." "Currently DFLASH speculative decoding does not support dp attention."
) )
@@ -264,7 +266,12 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
draft_backend = server_args.speculative_draft_attention_backend draft_backend = server_args.speculative_draft_attention_backend
if draft_backend is None: if draft_backend is None:
draft_backend, _ = server_args.get_attention_backends() from sglang.srt.arg_groups.overrides import (
attention_backends_of,
resolved_view,
)
draft_backend, _ = attention_backends_of(resolved_view(server_args))
if draft_backend is None: if draft_backend is None:
draft_backend = fallback_backend draft_backend = fallback_backend
elif draft_backend == "trtllm_mha": elif draft_backend == "trtllm_mha":
@@ -305,9 +312,14 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None:
def _handle_eagle_family(server_args: ServerArgs) -> None: def _handle_eagle_family(server_args: ServerArgs) -> None:
from sglang.srt.arg_groups.overrides import (
attention_backends_of,
resolved_view,
)
if ( if (
server_args.speculative_algorithm == "STANDALONE" server_args.speculative_algorithm == "STANDALONE"
and server_args.enable_dp_attention and resolved_view(server_args).enable_dp_attention
): ):
# TODO: support dp attention for standalone speculative decoding # TODO: support dp attention for standalone speculative decoding
raise ValueError( raise ValueError(
@@ -320,7 +332,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests." "Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests."
) )
if server_args.disable_overlap_schedule: if resolved_view(server_args).disable_overlap_schedule:
logger.warning( logger.warning(
"Non-overlap (synchronous) spec v2 is used for eagle/eagle3/standalone " "Non-overlap (synchronous) spec v2 is used for eagle/eagle3/standalone "
"speculative decoding." "speculative decoding."
@@ -375,11 +387,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
server_args.speculative_num_draft_tokens, server_args.speculative_num_draft_tokens,
) = _auto_choose_speculative_params(server_args, model_arch) ) = _auto_choose_speculative_params(server_args, model_arch)
if ( if "trtllm_mha" in attention_backends_of(resolved_view(server_args)):
server_args.attention_backend == "trtllm_mha"
or server_args.decode_attention_backend == "trtllm_mha"
or server_args.prefill_attention_backend == "trtllm_mha"
):
if server_args.speculative_eagle_topk > 1: if server_args.speculative_eagle_topk > 1:
raise ValueError( raise ValueError(
"trtllm_mha backend only supports topk = 1 for speculative decoding." "trtllm_mha backend only supports topk = 1 for speculative decoding."
@@ -416,7 +424,9 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
"--enable-deterministic-inference; the sampling kernel draws " "--enable-deterministic-inference; the sampling kernel draws "
"coins from the global RNG and is not batch-invariant." "coins from the global RNG and is not batch-invariant."
) )
if server_args.enable_multi_layer_eagle: from sglang.srt.arg_groups.overrides import resolved_view
if resolved_view(server_args).enable_multi_layer_eagle:
raise NotImplementedError( raise NotImplementedError(
"--speculative-use-rejection-sampling is not supported with " "--speculative-use-rejection-sampling is not supported with "
"multi-layer EAGLE (--enable-multi-layer-eagle)." "multi-layer EAGLE (--enable-multi-layer-eagle)."
@@ -440,15 +450,16 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
# pass + per-branch expand pass with prefix-tail dup). Only these backends implement # pass + per-branch expand pass with prefix-tail dup). Only these backends implement
# it; flashmla / trtllm_mla / cutlass_mla can't express the per-branch tree, so reject. # it; flashmla / trtllm_mla / cutlass_mla can't express the per-branch tree, so reject.
_PAGE_TREE_SPEC_BACKENDS = ("flashinfer", "fa3", "triton") _PAGE_TREE_SPEC_BACKENDS = ("flashinfer", "fa3", "triton")
view = resolved_view(server_args)
if ( if (
server_args.speculative_eagle_topk > 1 server_args.speculative_eagle_topk > 1
and server_args.page_size > 1 and view.page_size > 1
and server_args.attention_backend not in _PAGE_TREE_SPEC_BACKENDS and view.attention_backend not in _PAGE_TREE_SPEC_BACKENDS
): ):
raise ValueError( raise ValueError(
f"speculative_eagle_topk > 1 with page_size > 1 is only supported on " f"speculative_eagle_topk > 1 with page_size > 1 is only supported on "
f"{_PAGE_TREE_SPEC_BACKENDS}; got attention_backend=" f"{_PAGE_TREE_SPEC_BACKENDS}; got attention_backend="
f"{server_args.attention_backend!r}. Use page_size == 1 or one of those backends." f"{view.attention_backend!r}. Use page_size == 1 or one of those backends."
) )
@@ -499,18 +510,21 @@ def _handle_ngram(server_args: ServerArgs) -> None:
"using ngram speculative decoding." "using ngram speculative decoding."
) )
from sglang.srt.arg_groups.overrides import resolved_view
view = resolved_view(server_args)
if ( if (
server_args.speculative_eagle_topk > 1 server_args.speculative_eagle_topk > 1
and server_args.page_size > 1 and view.page_size > 1
and server_args.attention_backend != "flashinfer" and view.attention_backend != "flashinfer"
): ):
raise ValueError( raise ValueError(
f"speculative_eagle_topk({server_args.speculative_eagle_topk}) > 1 " f"speculative_eagle_topk({server_args.speculative_eagle_topk}) > 1 "
f"with page_size({server_args.page_size}) > 1 is unstable " f"with page_size({view.page_size}) > 1 is unstable "
"and produces incorrect results for paged attention backends. " "and produces incorrect results for paged attention backends. "
"This combination is only supported for the 'flashinfer' backend." "This combination is only supported for the 'flashinfer' backend."
) )
if server_args.enable_dp_attention: if view.enable_dp_attention:
# TODO: support dp attention for ngram speculative decoding # TODO: support dp attention for ngram speculative decoding
raise ValueError( raise ValueError(
"Currently ngram speculative decoding does not support dp attention." "Currently ngram speculative decoding does not support dp attention."
File diff suppressed because it is too large Load Diff
@@ -49,6 +49,8 @@ DEFAULT_ADAPTIVE_CONFIG: dict[str, dict] = {
def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None: def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
"""Return why adaptive spec cannot run under the given server args, or None if supported.""" """Return why adaptive spec cannot run under the given server args, or None if supported."""
from sglang.srt.arg_groups.overrides import resolved_view
if server_args.speculative_algorithm not in ("EAGLE", "EAGLE3"): if server_args.speculative_algorithm not in ("EAGLE", "EAGLE3"):
return ( return (
f"speculative_algorithm={server_args.speculative_algorithm} " f"speculative_algorithm={server_args.speculative_algorithm} "
@@ -62,12 +64,12 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
f"speculative_eagle_topk={server_args.speculative_eagle_topk} " f"speculative_eagle_topk={server_args.speculative_eagle_topk} "
"(only topk=1 is supported)" "(only topk=1 is supported)"
) )
if server_args.enable_dp_attention: if resolved_view(server_args).enable_dp_attention:
return ( return (
"enable_dp_attention=True is not supported " "enable_dp_attention=True is not supported "
"(adaptive tier decisions are not synchronized across DP ranks)" "(adaptive tier decisions are not synchronized across DP ranks)"
) )
if server_args.enable_multi_layer_eagle: if resolved_view(server_args).enable_multi_layer_eagle:
return ( return (
"enable_multi_layer_eagle=True is not supported " "enable_multi_layer_eagle=True is not supported "
"(MultiLayerEagleWorkerV2 does not implement adaptive)" "(MultiLayerEagleWorkerV2 does not implement adaptive)"
@@ -17,6 +17,10 @@ register_amd_ci(est_time=60, suite="extra-a-test-1-gpu-small-amd")
def _make_server_args(*, sampling_backend: str) -> SimpleNamespace: def _make_server_args(*, sampling_backend: str) -> SimpleNamespace:
# The install gate reads the resolved backend from the flags tier.
from sglang.srt.runtime_context import get_flags
get_flags().sampling_backend = sampling_backend
return SimpleNamespace(sampling_backend=sampling_backend) return SimpleNamespace(sampling_backend=sampling_backend)
@@ -102,6 +102,10 @@ def _make_model_runner(
sa = SimpleNamespace() sa = SimpleNamespace()
sa.swa_full_tokens_ratio = swa_full_tokens_ratio sa.swa_full_tokens_ratio = swa_full_tokens_ratio
# The configurator reads the resolved ratio from the flags tier.
from sglang.srt.runtime_context import get_flags
get_flags().swa_full_tokens_ratio = swa_full_tokens_ratio
sa.page_size = page_size sa.page_size = page_size
sa.disable_radix_cache = disable_radix_cache sa.disable_radix_cache = disable_radix_cache
sa.chunked_prefill_size = chunked_prefill_size sa.chunked_prefill_size = chunked_prefill_size
@@ -42,7 +42,7 @@ class TestDeepseekV4SharedExpertFusionPolicy(unittest.TestCase):
self.assertEqual(model.num_fused_shared_experts, 0) self.assertEqual(model.num_fused_shared_experts, 0)
self.assertTrue(get_flags().disable_shared_experts_fusion) self.assertTrue(get_flags().disable_shared_experts_fusion)
# dual-apply transition: the published config carries the value too # post-init declaration writes through to the published config
self.assertTrue(server_args.disable_shared_experts_fusion) self.assertTrue(server_args.disable_shared_experts_fusion)
def test_enables_shared_fusion_when_enforced(self): def test_enables_shared_fusion_when_enforced(self):
@@ -354,7 +354,10 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
args._handle_attention_backend_compatibility() args._handle_attention_backend_compatibility()
self.assertEqual(args.page_size, 128) from sglang.srt.arg_groups.overrides import resolved_view
self.assertEqual(args.page_size, 1) # dual-apply retired: pristine
self.assertEqual(resolved_view(args).page_size, 128)
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True) @patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
@patch("sglang.srt.server_args.ServerArgs.use_mla_backend", return_value=False) @patch("sglang.srt.server_args.ServerArgs.use_mla_backend", return_value=False)
@@ -364,7 +367,10 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
args._handle_attention_backend_compatibility() args._handle_attention_backend_compatibility()
self.assertEqual(args.page_size, 128) from sglang.srt.arg_groups.overrides import resolved_view
self.assertEqual(args.page_size, 1) # dual-apply retired: pristine
self.assertEqual(resolved_view(args).page_size, 128)
class TestContextParallelServerArgs(CustomTestCase): class TestContextParallelServerArgs(CustomTestCase):
@@ -1119,7 +1125,11 @@ class TestDeepEPWaterfillArgs(CustomTestCase):
# dummy-model path short-circuits __post_init__; invoke the handler directly. # dummy-model path short-circuits __post_init__; invoke the handler directly.
server_args._handle_a2a_moe() server_args._handle_a2a_moe()
self.assertFalse(server_args.disable_shared_experts_fusion) from sglang.srt.arg_groups.overrides import resolved_view
# dual-apply retired: the fields stay pristine, the declarations win
self.assertTrue(server_args.disable_shared_experts_fusion)
self.assertFalse(resolved_view(server_args).disable_shared_experts_fusion)
self.assertTrue(server_args.enforce_shared_experts_fusion) self.assertTrue(server_args.enforce_shared_experts_fusion)
def test_waterfill_overrides_moe_a2a_backend_to_deepep(self): def test_waterfill_overrides_moe_a2a_backend_to_deepep(self):
@@ -1131,7 +1141,10 @@ class TestDeepEPWaterfillArgs(CustomTestCase):
# dummy-model path short-circuits __post_init__; invoke the handler directly. # dummy-model path short-circuits __post_init__; invoke the handler directly.
server_args._handle_a2a_moe() server_args._handle_a2a_moe()
self.assertEqual(server_args.moe_a2a_backend, "deepep") from sglang.srt.arg_groups.overrides import resolved_view
self.assertEqual(server_args.moe_a2a_backend, "none") # pristine
self.assertEqual(resolved_view(server_args).moe_a2a_backend, "deepep")
self.assertTrue(server_args.enforce_shared_experts_fusion) self.assertTrue(server_args.enforce_shared_experts_fusion)
def test_waterfill_supports_deepep_low_latency_mode(self): def test_waterfill_supports_deepep_low_latency_mode(self):
+128 -62
View File
@@ -19,11 +19,10 @@ from sglang.srt.arg_groups import overrides as overrides_module
from sglang.srt.arg_groups.arg_utils import A, Arg, resolvable_fields from sglang.srt.arg_groups.arg_utils import A, Arg, resolvable_fields
from sglang.srt.arg_groups.overrides import ( from sglang.srt.arg_groups.overrides import (
OverrideRecord, OverrideRecord,
apply_declarations_to_server_args,
apply_model_overrides, apply_model_overrides,
assert_flag_parity,
collect_model_override_declarations, collect_model_override_declarations,
register_model_override, register_model_override,
validate_declarations,
) )
from sglang.srt.runtime_context import ( from sglang.srt.runtime_context import (
_StaticFlags, _StaticFlags,
@@ -207,7 +206,7 @@ class TestResolvedViewAndPasses(CustomTestCase):
self.assertEqual(view.a, 10) self.assertEqual(view.a, 10)
self.assertEqual(view.b, 2) self.assertEqual(view.b, 2)
def test_run_pass_appends_stash_and_dual_applies(self): def test_run_pass_appends_stash_and_stays_pristine(self):
from sglang.srt.arg_groups.overrides import run_post_process_pass from sglang.srt.arg_groups.overrides import run_post_process_pass
live = SimpleNamespace(x=None, _resolved_overrides=[]) live = SimpleNamespace(x=None, _resolved_overrides=[])
@@ -216,11 +215,12 @@ class TestResolvedViewAndPasses(CustomTestCase):
return {"x": "filled"} if view.x is None else {} return {"x": "filled"} if view.x is None else {}
run_post_process_pass(live, _fill_x) run_post_process_pass(live, _fill_x)
self.assertEqual(live.x, "filled") # dual-applied in place self.assertIsNone(live.x) # never applied in place
self.assertEqual( self.assertEqual(
live._resolved_overrides, [(_fill_x.__qualname__, {"x": "filled"})] live._resolved_overrides, [(_fill_x.__qualname__, {"x": "filled"})]
) )
run_post_process_pass(live, _fill_x) # now a no-op # the next invocation sees the declared value through the overlay
run_post_process_pass(live, _fill_x)
self.assertEqual(len(live._resolved_overrides), 1) self.assertEqual(len(live._resolved_overrides), 1)
def test_run_pass_rejects_non_dict(self): def test_run_pass_rejects_non_dict(self):
@@ -415,7 +415,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def test_mistral_large3_forces_bfloat16(self): def test_mistral_large3_forces_bfloat16(self):
sa = self._construct("MistralLarge3ForCausalLM", "mistral") sa = self._construct("MistralLarge3ForCausalLM", "mistral")
self.assertEqual(sa.dtype, "bfloat16") # dual-apply == legacy write self.assertEqual(sa.dtype, "bfloat16") # materialized at end of resolution
self.assertIn( self.assertIn(
("MODEL_OVERRIDES['MistralLarge3ForCausalLM']", {"dtype": "bfloat16"}), ("MODEL_OVERRIDES['MistralLarge3ForCausalLM']", {"dtype": "bfloat16"}),
sa._resolved_overrides, sa._resolved_overrides,
@@ -424,15 +424,15 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def test_pixtral_forces_bfloat16(self): def test_pixtral_forces_bfloat16(self):
sa = self._construct("PixtralForConditionalGeneration", "pixtral") sa = self._construct("PixtralForConditionalGeneration", "pixtral")
self.assertEqual(sa.dtype, "bfloat16") self.assertEqual(sa.dtype, "bfloat16") # materialized
self.assertEqual(self._publish(sa).dtype, "bfloat16") self.assertEqual(self._publish(sa).dtype, "bfloat16")
def test_user_requested_dtype_is_still_overridden(self): def test_user_requested_dtype_is_still_overridden(self):
# Legacy fidelity: the arch branch overwrote dtype unconditionally, # Legacy fidelity: the arch branch overwrote dtype unconditionally,
# so the declaration must too. The pristine request survives only on # so the declaration must too. The pristine request survives on
# provenance (and, post-V3, as the un-overridden server_args field). # provenance; the materialized field carries the override.
sa = self._construct("MistralLarge3ForCausalLM", "mistral", dtype="float16") sa = self._construct("MistralLarge3ForCausalLM", "mistral", dtype="float16")
self.assertEqual(sa.dtype, "bfloat16") self.assertEqual(sa.dtype, "bfloat16") # materialized
self.assertEqual(self._publish(sa).dtype, "bfloat16") self.assertEqual(self._publish(sa).dtype, "bfloat16")
def test_control_arch_keeps_pristine_dtype(self): def test_control_arch_keeps_pristine_dtype(self):
@@ -446,7 +446,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def test_minimax_m2_enables_tf32_matmul(self): def test_minimax_m2_enables_tf32_matmul(self):
sa = self._construct("MiniMaxM2ForCausalLM", "llama") sa = self._construct("MiniMaxM2ForCausalLM", "llama")
self.assertTrue(sa.enable_tf32_matmul) # dual-apply == legacy write self.assertTrue(sa.enable_tf32_matmul) # materialized
self.assertIn( self.assertIn(
("_minimax_m2_overrides", {"enable_tf32_matmul": True}), ("_minimax_m2_overrides", {"enable_tf32_matmul": True}),
sa._resolved_overrides, sa._resolved_overrides,
@@ -491,7 +491,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
config_extra=config_extra, config_extra=config_extra,
enable_hierarchical_cache=True, enable_hierarchical_cache=True,
) )
# dual-apply == legacy writes # materialized at the end of resolution
self.assertEqual(sa.swa_full_tokens_ratio, 1.0) self.assertEqual(sa.swa_full_tokens_ratio, 1.0)
self.assertTrue(sa.disable_hybrid_swa_memory) self.assertTrue(sa.disable_hybrid_swa_memory)
flags = self._publish(sa) flags = self._publish(sa)
@@ -500,7 +500,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def test_gemma2_disables_hybrid_swa_memory(self): def test_gemma2_disables_hybrid_swa_memory(self):
sa = self._construct("Gemma2ForCausalLM", "llama") sa = self._construct("Gemma2ForCausalLM", "llama")
self.assertTrue(sa.disable_hybrid_swa_memory) # dual-apply == legacy self.assertTrue(sa.disable_hybrid_swa_memory) # materialized
self.assertIn( self.assertIn(
("_gemma2_gemma3_overrides", {"disable_hybrid_swa_memory": True}), ("_gemma2_gemma3_overrides", {"disable_hybrid_swa_memory": True}),
sa._resolved_overrides, sa._resolved_overrides,
@@ -509,7 +509,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def test_olmo2_disables_hybrid_swa_memory(self): def test_olmo2_disables_hybrid_swa_memory(self):
sa = self._construct("Olmo2ForCausalLM", "llama") sa = self._construct("Olmo2ForCausalLM", "llama")
self.assertTrue(sa.disable_hybrid_swa_memory) self.assertTrue(sa.disable_hybrid_swa_memory) # materialized
self.assertTrue(self._publish(sa).disable_hybrid_swa_memory) self.assertTrue(self._publish(sa).disable_hybrid_swa_memory)
def test_exaone_conditional_on_sliding_window_pattern(self): def test_exaone_conditional_on_sliding_window_pattern(self):
@@ -520,7 +520,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
config_extra={"sliding_window_pattern": "LLLG"}, config_extra={"sliding_window_pattern": "LLLG"},
attention_backend="fa3", attention_backend="fa3",
) )
self.assertTrue(sa.disable_hybrid_swa_memory) self.assertTrue(sa.disable_hybrid_swa_memory) # materialized
self.assertTrue(self._publish(sa).disable_hybrid_swa_memory) self.assertTrue(self._publish(sa).disable_hybrid_swa_memory)
def test_exaone_without_pattern_declares_nothing(self): def test_exaone_without_pattern_declares_nothing(self):
@@ -543,7 +543,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
"llama", "llama",
config_extra={"quantization_config": {"quant_method": "mxfp4"}}, config_extra={"quantization_config": {"quant_method": "mxfp4"}},
) )
self.assertEqual(sa.dtype, "bfloat16") # dual-apply == legacy self.assertEqual(sa.dtype, "bfloat16") # materialized
self.assertEqual(self._publish(sa).dtype, "bfloat16") self.assertEqual(self._publish(sa).dtype, "bfloat16")
def test_gpt_oss_without_mxfp4_keeps_pristine_dtype(self): def test_gpt_oss_without_mxfp4_keeps_pristine_dtype(self):
@@ -569,7 +569,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
sa = self._construct("LlamaForCausalLM", "llama") sa = self._construct("LlamaForCausalLM", "llama")
expected = "flashinfer" if is_flashinfer_available() else "pytorch" expected = "flashinfer" if is_flashinfer_available() else "pytorch"
self.assertEqual(sa.sampling_backend, expected) self.assertEqual(sa.sampling_backend, expected) # materialized
self.assertIn( self.assertIn(
("_sampling_backend_default", {"sampling_backend": expected}), ("_sampling_backend_default", {"sampling_backend": expected}),
sa._resolved_overrides, sa._resolved_overrides,
@@ -587,20 +587,20 @@ class TestGoldenModelOverrides(_IsolatedPublish):
"LlamaForCausalLM", "llama", enable_deterministic_inference=True "LlamaForCausalLM", "llama", enable_deterministic_inference=True
) )
# two pass writers chain: default fill, then the deterministic force — # two pass writers chain: default fill, then the deterministic force —
# last writer wins on the flags leaf and parity holds end-to-end. # last writer wins; materialization lands the end state on the fields.
self.assertEqual(sa.sampling_backend, "pytorch") self.assertEqual(sa.sampling_backend, "pytorch")
flags = self._publish(sa) flags = self._publish(sa)
self.assertEqual(flags.sampling_backend, "pytorch") self.assertEqual(flags.sampling_backend, "pytorch")
# the deterministic attention fill declared a compatible backend and # the deterministic attention fill declared a compatible backend and
# the compatibility default-fill then had nothing to do # the compatibility default-fill then had nothing to do
self.assertIn( deterministic_fills = [
( decl["attention_backend"]
"_deterministic_attention_backend", for source, decl in sa._resolved_overrides
{"attention_backend": sa.attention_backend}, if source == "_deterministic_attention_backend"
), ]
sa._resolved_overrides, self.assertEqual(len(deterministic_fills), 1)
) self.assertEqual(sa.attention_backend, deterministic_fills[0])
self.assertEqual(flags.attn.backend, sa.attention_backend) self.assertEqual(flags.attn.backend, deterministic_fills[0])
def test_deterministic_incompatible_backend_raises(self): def test_deterministic_incompatible_backend_raises(self):
from sglang.srt.arg_groups.overrides import ( from sglang.srt.arg_groups.overrides import (
@@ -631,13 +631,15 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def test_dllm_forces_flashinfer_with_cuda_graph(self): def test_dllm_forces_flashinfer_with_cuda_graph(self):
# CUDA path: cuda graph enabled by default -> dllm forces flashinfer. # CUDA path: cuda graph enabled by default -> dllm forces flashinfer.
# A real dllm arch: the page pass now runs regardless of the radix
# switch and builds DllmConfig for it.
sa = self._construct( sa = self._construct(
"LlamaForCausalLM", "SDARForCausalLM",
"llama", "llama",
dllm_algorithm="LowConfidence", dllm_algorithm="LowConfidence",
disable_radix_cache=True, disable_radix_cache=True,
) )
self.assertEqual(sa.attention_backend, "flashinfer") self.assertEqual(sa.attention_backend, "flashinfer") # materialized
self.assertIn( self.assertIn(
("_dllm_attention_backend", {"attention_backend": "flashinfer"}), ("_dllm_attention_backend", {"attention_backend": "flashinfer"}),
sa._resolved_overrides, sa._resolved_overrides,
@@ -647,24 +649,35 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def test_attention_backend_leaf_materializes_end_state(self): def test_attention_backend_leaf_materializes_end_state(self):
# The default-fill pass declares the platform-selected backend; the # The default-fill pass declares the platform-selected backend; the
# leaf must equal the final server_args value (publish parity). # leaf must equal the last declared value while the server_args field
# stays pristine (dual-apply retired).
sa = self._construct("LlamaForCausalLM", "llama") sa = self._construct("LlamaForCausalLM", "llama")
declared = {f for _s, d in sa._resolved_overrides for f in d} declared_values = [
self.assertIn("attention_backend", declared) # default fill declared d["attention_backend"]
self.assertEqual(self._publish(sa).attn.backend, sa.attention_backend) for _s, d in sa._resolved_overrides
if "attention_backend" in d
]
self.assertTrue(declared_values) # default fill declared
self.assertEqual(sa.attention_backend, declared_values[-1]) # materialized
self.assertEqual(self._publish(sa).attn.backend, declared_values[-1])
def test_runner_side_adjustment_can_refresh_declaration(self): def test_post_materialize_pass_writes_through(self):
from sglang.srt.arg_groups.overrides import refresh_declared_fields from sglang.srt.arg_groups.overrides import run_post_process_pass
# A pass invoked after materialization (a post-init slot, like the
# legacy runner-side adjustments) declares AND writes through, so
# field readers and the publish see the same end state.
sa = self._construct("LlamaForCausalLM", "llama") sa = self._construct("LlamaForCausalLM", "llama")
declared = {f for _s, d in sa._resolved_overrides for f in d} resolved_before = sa.attention_backend
self.assertIn("attention_backend", declared)
# Simulate a legacy runner-side overwrite between collection and publish def _force_triton(view):
# (model_specific_adjustment forces attention_backend for HRM-Text). if view.attention_backend != "triton":
sa.attention_backend = "fa3" if sa.attention_backend != "fa3" else "triton" return {"attention_backend": "triton"}
with self.assertRaises(AssertionError): return {}
self._publish(sa) # stale declaration breaks parity
refresh_declared_fields(sa, ("attention_backend",)) run_post_process_pass(sa, _force_triton)
if resolved_before != "triton":
self.assertEqual(sa.attention_backend, "triton")
self.assertEqual(self._publish(sa).attn.backend, sa.attention_backend) self.assertEqual(self._publish(sa).attn.backend, sa.attention_backend)
def test_attention_backend_user_choice_declares_nothing_extra(self): def test_attention_backend_user_choice_declares_nothing_extra(self):
@@ -798,9 +811,41 @@ class TestGoldenModelOverrides(_IsolatedPublish):
return_value=SimpleNamespace(block_size=32), return_value=SimpleNamespace(block_size=32),
): ):
self.assertEqual(_view() and _dllm_page_size(_view()), {"page_size": 32}) self.assertEqual(_view() and _dllm_page_size(_view()), {"page_size": 32})
self.assertEqual(_dllm_page_size(_view(page_size=64)), {}) # aligned # aligned but larger than the block: the scheduler-init fallback
self.assertEqual(_dllm_page_size(_view(dllm_algorithm=None)), {}) # (folded into this pass) still caps the page at the block size
self.assertEqual(_dllm_page_size(_view(page_size=64)), {"page_size": 32})
self.assertEqual(_dllm_page_size(_view(page_size=32)), {}) # equal
# radix disabled skips the alignment fill but keeps the cap
self.assertEqual(_dllm_page_size(_view(disable_radix_cache=True)), {}) self.assertEqual(_dllm_page_size(_view(disable_radix_cache=True)), {})
self.assertEqual(
_dllm_page_size(_view(disable_radix_cache=True, page_size=64)),
{"page_size": 32},
)
self.assertEqual(_dllm_page_size(_view(dllm_algorithm=None)), {})
def test_declaration_overlay_mechanics(self):
from sglang.srt.arg_groups.overrides import run_post_process_pass
live = SimpleNamespace(x="user", y=None, _resolved_overrides=[])
def _resolve_x(view):
return {"x": "resolved"} if view.x == "user" else {}
def _read_x(view):
return {"y": view.x}
run_post_process_pass(live, _resolve_x)
# declaration recorded, but server_args stays pristine
self.assertEqual(
live._resolved_overrides, [(_resolve_x.__qualname__, {"x": "resolved"})]
)
self.assertEqual(live.x, "user")
# a later pass sees the resolved value through the view overlay
run_post_process_pass(live, _read_x)
self.assertEqual(
live._resolved_overrides[-1], (_read_x.__qualname__, {"y": "resolved"})
)
self.assertIsNone(live.y) # never applied in place
def test_overlap_disable_passes(self): def test_overlap_disable_passes(self):
from sglang.srt.arg_groups.overrides import ( from sglang.srt.arg_groups.overrides import (
@@ -1531,9 +1576,12 @@ class TestGoldenModelOverrides(_IsolatedPublish):
def test_page_size_leaf_materializes_end_state(self): def test_page_size_leaf_materializes_end_state(self):
sa = self._construct("LlamaForCausalLM", "llama") sa = self._construct("LlamaForCausalLM", "llama")
declared = {f for _s, d in sa._resolved_overrides for f in d} declared_values = [
self.assertIn("page_size", declared) # default fill declared d["page_size"] for _s, d in sa._resolved_overrides if "page_size" in d
self.assertEqual(self._publish(sa).page_size, sa.page_size) ]
self.assertTrue(declared_values) # default fill declared
self.assertEqual(sa.page_size, declared_values[-1]) # materialized
self.assertEqual(self._publish(sa).page_size, declared_values[-1])
def test_qwen3_5_hybrid_coupled_declaration(self): def test_qwen3_5_hybrid_coupled_declaration(self):
from sglang.srt.arg_groups.overrides import _qwen3_5_hybrid_overrides from sglang.srt.arg_groups.overrides import _qwen3_5_hybrid_overrides
@@ -1544,7 +1592,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
_get_default_attn_backend=lambda **_: default_backend, _get_default_attn_backend=lambda **_: default_backend,
use_mla_backend=lambda: False, use_mla_backend=lambda: False,
get_model_config=lambda: None, get_model_config=lambda: None,
enable_mamba_extra_buffer=lambda: False, mamba_radix_cache_strategy="auto",
disable_radix_cache=False, disable_radix_cache=False,
speculative_algorithm=None, speculative_algorithm=None,
) )
@@ -1571,6 +1619,24 @@ class TestGoldenModelOverrides(_IsolatedPublish):
), ),
{}, {},
) )
# the mamba pass ran before this dispatch and stashed the
# extra-buffer strategy: the callable must see it through the
# view (SM100 hybrid keeps trtllm_mha + page 64)
self.assertEqual(
_qwen3_5_hybrid_overrides(
_args(
"trtllm_mha",
_resolved_overrides=[
(
"_mamba_radix_cache_declarations",
{"mamba_radix_cache_strategy": "extra_buffer"},
)
],
),
None,
),
{"attention_backend": "trtllm_mha", "page_size": 64},
)
with patch.object(overrides_module, "is_sm100_supported", return_value=False): with patch.object(overrides_module, "is_sm100_supported", return_value=False):
self.assertEqual(_qwen3_5_hybrid_overrides(_args("fa3"), None), {}) self.assertEqual(_qwen3_5_hybrid_overrides(_args("fa3"), None), {})
@@ -1738,7 +1804,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
self.assertEqual( self.assertEqual(
_intel_xpu_page_constraint( _intel_xpu_page_constraint(
_view( _view(
get_attention_backends=lambda: (None, "intel_xpu"), decode_attention_backend="intel_xpu",
use_mla_backend=lambda: False, use_mla_backend=lambda: False,
) )
), ),
@@ -1747,7 +1813,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
self.assertEqual( self.assertEqual(
_intel_xpu_page_constraint( _intel_xpu_page_constraint(
_view( _view(
get_attention_backends=lambda: (None, "intel_xpu"), decode_attention_backend="intel_xpu",
use_mla_backend=lambda: True, use_mla_backend=lambda: True,
page_size=16, # MLA decode accepts 16 page_size=16, # MLA decode accepts 16
) )
@@ -2089,22 +2155,22 @@ class TestGoldenModelOverrides(_IsolatedPublish):
self.assertEqual(_step3p_overrides(_args(), None), {}) self.assertEqual(_step3p_overrides(_args(), None), {})
class TestDualApplyParity(CustomTestCase): class TestDeclarationValidation(CustomTestCase):
def test_dual_apply_replays_and_parity_holds(self): def test_declarations_never_mutate_server_args(self):
flags, args = _FakeFlags(), _FakeArgs() flags, args = _FakeFlags(), _FakeArgs()
declarations = [("src", {"resolved_by_model": "dsv4", "also_resolved": 7})] declarations = [("src", {"resolved_by_model": "dsv4", "also_resolved": 7})]
apply_model_overrides(flags, args, declarations) apply_model_overrides(flags, args, declarations)
apply_declarations_to_server_args(args, declarations) validate_declarations(args, declarations)
self.assertEqual(args.resolved_by_model, "dsv4") # the leaves carry the declared values; the fields stay pristine
self.assertEqual(args.also_resolved, 7) self.assertEqual(flags.resolved_by_model, "dsv4")
assert_flag_parity(flags, args, ["resolved_by_model", "also_resolved"]) self.assertEqual(flags.also_resolved, 7)
self.assertEqual(args.resolved_by_model, _FakeArgs.resolved_by_model)
self.assertEqual(args.also_resolved, _FakeArgs.also_resolved)
def test_parity_detects_drift(self): def test_validation_rejects_unknown_fields(self):
flags, args = _FakeFlags(), _FakeArgs() args = _FakeArgs()
apply_model_overrides(flags, args, [("src", {"resolved_by_model": "x"})]) with self.assertRaises(ValueError):
# dual-apply skipped -> server_args still pristine -> drift is caught validate_declarations(args, [("src", {"nope": 1})])
with self.assertRaises(AssertionError):
assert_flag_parity(flags, args, ["resolved_by_model"])
if __name__ == "__main__": if __name__ == "__main__":
+20 -12
View File
@@ -339,15 +339,6 @@ class TestRuntimeResolutionStages(_IsolatedServerArgs):
self.assertEqual(get_flags().sampling_backend, "pytorch") self.assertEqual(get_flags().sampling_backend, "pytorch")
self.assertEqual(get_flags().page_size, 64) # earlier stage survives self.assertEqual(get_flags().page_size, 64) # earlier stage survives
def test_record_parity_failure_rolls_back(self):
self._publish(page_size=1)
flags_before = get_flags()
with self.assertRaises(AssertionError):
# declared value diverges from the live server_args (no dual-apply)
get_context().record_runtime_overrides([("bad", {"page_size": 64})])
self.assertIs(get_flags(), flags_before) # previous flags intact
self.assertEqual(get_context()._runtime_overrides, []) # rolled back
def test_record_whitelist_violation_rolls_back(self): def test_record_whitelist_violation_rolls_back(self):
self._publish() self._publish()
with self.assertRaises(ValueError): with self.assertRaises(ValueError):
@@ -366,13 +357,15 @@ class TestRuntimeResolutionStages(_IsolatedServerArgs):
finally: finally:
reset_context() reset_context()
def test_declare_load_time_override_dual_applies_and_records(self): def test_declare_load_time_override_applies_and_records(self):
from sglang.srt.arg_groups.overrides import declare_load_time_override from sglang.srt.arg_groups.overrides import declare_load_time_override
args = self._publish(page_size=1) args = self._publish(page_size=1)
declare_load_time_override("model.load_time", {"page_size": 64}) declare_load_time_override("model.load_time", {"page_size": 64})
self.assertEqual(args.page_size, 64) # dual-applied onto server_args # post-init declaration: written through to the field and resolved
self.assertEqual(get_flags().page_size, 64) # resolved into the leaf # into the leaf
self.assertEqual(args.page_size, 64)
self.assertEqual(get_flags().page_size, 64)
self.assertEqual( self.assertEqual(
get_context()._runtime_overrides, get_context()._runtime_overrides,
[("model.load_time", {"page_size": 64})], [("model.load_time", {"page_size": 64})],
@@ -415,6 +408,21 @@ class TestRuntimeResolutionStages(_IsolatedServerArgs):
finally: finally:
reset_context() reset_context()
def test_declared_leaf_wins_over_stale_field(self):
# A stash entry always drives the leaf at publish, even if the field
# value diverged (e.g. a fixture that skipped materialization).
@dataclasses.dataclass
class _Args:
enable_dp_lm_head: A[bool, Arg(help="d", resolvable=True)] = True
_resolved_overrides: list = dataclasses.field(default_factory=list)
args = _Args()
args._resolved_overrides = [("dp", {"enable_dp_lm_head": False})]
args._declarations_materialized = True
args.enable_dp_lm_head = False
get_context().set_server_args(args)
self.assertFalse(get_flags().enable_dp_lm_head)
def test_bare_dataclass_publish_skips_materialization(self): def test_bare_dataclass_publish_skips_materialization(self):
# object.__new__(ServerArgs) fixtures (no __init__, no field values) # object.__new__(ServerArgs) fixtures (no __init__, no field values)
# must publish without touching the flags tier — dataclass defaults # must publish without touching the flags tier — dataclass defaults