[refactor] Resolve config declarations onto server_args at the end of __post_init__ (#30297)
This commit is contained in:
@@ -43,21 +43,30 @@ def _hisparse_allowed_backends(kv_cache_dtype: str) -> set[str]:
|
||||
def validate_hisparse_dsa_backend(
|
||||
server_args: ServerArgs, attr: str, label: str
|
||||
) -> None:
|
||||
backend = getattr(server_args, attr)
|
||||
allowed_backends = _hisparse_allowed_backends(server_args.kv_cache_dtype)
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
# Invoked after the DSA kv-cache-dtype / split-backend declarations:
|
||||
# read the resolving state through the view.
|
||||
view = resolved_view(server_args)
|
||||
backend = getattr(view, attr)
|
||||
kv_cache_dtype = view.kv_cache_dtype
|
||||
allowed_backends = _hisparse_allowed_backends(kv_cache_dtype)
|
||||
if backend is not None and backend not in allowed_backends:
|
||||
raise ValueError(
|
||||
f"HiSparse supports DSA {label} backend(s) {sorted(allowed_backends)} "
|
||||
f"on this platform with --kv-cache-dtype={server_args.kv_cache_dtype}, "
|
||||
f"on this platform with --kv-cache-dtype={kv_cache_dtype}, "
|
||||
f"but got --dsa-{label}-backend={backend}. "
|
||||
f"Please use --dsa-{label}-backend="
|
||||
f"{_hisparse_default_backend(server_args.kv_cache_dtype)} "
|
||||
f"{_hisparse_default_backend(kv_cache_dtype)} "
|
||||
"or omit it."
|
||||
)
|
||||
|
||||
|
||||
def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
|
||||
if server_args.kv_cache_dtype in HISPARSE_KV_CACHE_DTYPES:
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
kv_cache_dtype = resolved_view(server_args).kv_cache_dtype
|
||||
if kv_cache_dtype in HISPARSE_KV_CACHE_DTYPES:
|
||||
return
|
||||
|
||||
choices = " or ".join(
|
||||
@@ -65,7 +74,7 @@ def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
|
||||
)
|
||||
raise ValueError(
|
||||
f"HiSparse requires one of {HISPARSE_KV_CACHE_DTYPES} KV cache dtypes, "
|
||||
f"but got --kv-cache-dtype={server_args.kv_cache_dtype}. Please use {choices}."
|
||||
f"but got --kv-cache-dtype={kv_cache_dtype}. Please use {choices}."
|
||||
)
|
||||
|
||||
|
||||
@@ -112,7 +121,13 @@ def validate_hisparse(server_args: ServerArgs) -> None:
|
||||
)
|
||||
return
|
||||
|
||||
if server_args.kv_cache_dtype not in ("bfloat16", "auto", "fp8_e4m3"):
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
if resolved_view(server_args).kv_cache_dtype not in (
|
||||
"bfloat16",
|
||||
"auto",
|
||||
"fp8_e4m3",
|
||||
):
|
||||
validate_hisparse_kv_cache_dtype(server_args)
|
||||
|
||||
for attr, label in [
|
||||
|
||||
@@ -117,14 +117,10 @@ def _invoke_provider(
|
||||
|
||||
class ResolvedView:
|
||||
"""Read-only view of the resolving configuration handed to post-process
|
||||
passes.
|
||||
|
||||
During the dual-apply transition the view forwards every read to the live
|
||||
``server_args`` — the pristine input plus the declarations replayed so far
|
||||
plus any residual imperative writes — which is exactly the state the
|
||||
legacy handler at the same slot observed. In the end state (dual-apply
|
||||
retired) the same type overlays the accumulated declarations on the
|
||||
pristine object. Writes are rejected: passes return declarations.
|
||||
passes: the accumulated declarations overlaid on the pristine
|
||||
``server_args`` (residual imperative writes of non-resolved fields show
|
||||
through the fallthrough) — exactly the state the legacy handler at the
|
||||
same slot observed. Writes are rejected: passes return declarations.
|
||||
"""
|
||||
|
||||
__slots__ = ("_server_args", "_overlay")
|
||||
@@ -162,14 +158,27 @@ def register_post_process(fn: Callable[..., dict]) -> Callable[..., dict]:
|
||||
return fn
|
||||
|
||||
|
||||
def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
"""Transition-period invocation of one pass at its legacy handler slot.
|
||||
def _declaration_overlay(server_args: Any) -> Dict[str, Any]:
|
||||
"""Accumulated declared values: declarations never mutate
|
||||
``server_args``, so mid-resolution readers overlay them from the
|
||||
declaration stash (last writer wins, like the gate)."""
|
||||
overlay: Dict[str, Any] = {}
|
||||
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
|
||||
overlay.update(declared)
|
||||
return overlay
|
||||
|
||||
Evaluates the pass on the live state (through a read-only view), appends
|
||||
its declaration to the declaration stash, and dual-applies it in place —
|
||||
byte-identical to the imperative handler write this replaces.
|
||||
|
||||
def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
"""Invoke one pass at its legacy handler slot.
|
||||
|
||||
Evaluates the pass on the resolving state (a read-only view with the
|
||||
accumulated declarations overlaid from the stash) and appends its
|
||||
declaration to the stash. During ``__post_init__`` the fields stay
|
||||
untouched — ``materialize_declarations`` applies the whole stash once at
|
||||
the end of resolution; a pass invoked after materialization (a post-init
|
||||
slot) writes through immediately.
|
||||
"""
|
||||
declared = fn(ResolvedView(server_args))
|
||||
declared = fn(ResolvedView(server_args, overlay=_declaration_overlay(server_args)))
|
||||
if not isinstance(declared, dict):
|
||||
raise TypeError(
|
||||
f"post-process pass {fn.__qualname__} must return a dict, "
|
||||
@@ -186,19 +195,70 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
# must sit at or after it in __post_init__ order.
|
||||
stash = server_args._resolved_overrides = []
|
||||
stash.append(entry)
|
||||
apply_declarations_to_server_args(server_args, [entry])
|
||||
validate_declarations(server_args, [entry])
|
||||
if getattr(server_args, "_declarations_materialized", False):
|
||||
for field, value in declared.items():
|
||||
setattr(server_args, field, value)
|
||||
|
||||
|
||||
def materialize_declarations(server_args: Any) -> None:
|
||||
"""Apply the accumulated declarations onto ``server_args`` once, at the
|
||||
end of ``__post_init__`` (gate order: last writer wins). After this the
|
||||
fields carry the resolved configuration — every post-init reader, in any
|
||||
process, reads them directly; ``resolved_view`` remains an internal
|
||||
helper for mid-resolution code only."""
|
||||
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
|
||||
for field, value in declared.items():
|
||||
setattr(server_args, field, value)
|
||||
server_args._declarations_materialized = True
|
||||
|
||||
|
||||
def resolved_view(server_args: Any) -> ResolvedView:
|
||||
"""Read-only view of the resolving configuration for mid-resolution code
|
||||
that is not a pass (``__post_init__`` handlers and hooks). Internal to
|
||||
the resolution pipeline: after ``materialize_declarations`` runs, the
|
||||
fields themselves carry the resolved values — read them directly."""
|
||||
return ResolvedView(server_args, overlay=_declaration_overlay(server_args))
|
||||
|
||||
|
||||
def attention_backends_of(cfg: Any) -> tuple:
|
||||
"""(prefill, decode) attention backends of a config-shaped object (a
|
||||
ResolvedView mid-resolution, or pristine server_args at dispatch time):
|
||||
split fields fall back to the base backend."""
|
||||
prefill = (
|
||||
cfg.prefill_attention_backend
|
||||
if cfg.prefill_attention_backend
|
||||
else cfg.attention_backend
|
||||
)
|
||||
decode = (
|
||||
cfg.decode_attention_backend
|
||||
if cfg.decode_attention_backend
|
||||
else cfg.attention_backend
|
||||
)
|
||||
return prefill, decode
|
||||
|
||||
|
||||
def mamba_extra_buffer_of(cfg: Any) -> bool:
|
||||
"""Mid-resolution equivalent of runtime_context.mamba_extra_buffer_enabled:
|
||||
reads the (possibly overlaid) strategy from a config-shaped object."""
|
||||
return cfg.disable_radix_cache is False and cfg.mamba_radix_cache_strategy in (
|
||||
"extra_buffer",
|
||||
"extra_buffer_lazy",
|
||||
)
|
||||
|
||||
|
||||
def declare_load_time_override(source: str, declared: Dict[str, Any]) -> None:
|
||||
"""Transition helper for load-time resolved fields (model-file config
|
||||
overrides, weight-resolved dtypes): dual-apply the declaration onto the
|
||||
published ``server_args`` — byte-identical to the imperative write this
|
||||
replaces — and record it into the flags tier through the runtime gate."""
|
||||
"""Declare a load-time resolved field (model-file config overrides,
|
||||
weight-resolved dtypes): apply it onto the published ``server_args`` —
|
||||
resolution has already materialized, so post-init declarations write
|
||||
through — and record it into the flags tier through the runtime gate."""
|
||||
from sglang.srt.runtime_context import get_context
|
||||
|
||||
ctx = get_context()
|
||||
entry = (source, dict(declared))
|
||||
apply_declarations_to_server_args(ctx.server_args, [entry])
|
||||
validate_declarations(ctx.server_args, [entry])
|
||||
for field, value in declared.items():
|
||||
setattr(ctx.server_args, field, value)
|
||||
ctx.record_runtime_overrides([entry])
|
||||
|
||||
|
||||
@@ -741,8 +801,11 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
use_mla_backend=server_args.use_mla_backend(),
|
||||
model_config=server_args.get_model_config(),
|
||||
)
|
||||
# The mamba radix-cache pass runs before this dispatch: read the
|
||||
# declared strategy through the view (the legacy branch observed the
|
||||
# already-written field here).
|
||||
if default_attn_backend == "trtllm_mha" and not (
|
||||
not server_args.enable_mamba_extra_buffer()
|
||||
not mamba_extra_buffer_of(resolved_view(server_args))
|
||||
and not server_args.disable_radix_cache
|
||||
and server_args.speculative_algorithm is None
|
||||
):
|
||||
@@ -1519,12 +1582,57 @@ def _mla_backend_page_constraints(view: Any) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _mla_kv_cache_dtype_checks(view: Any) -> dict:
|
||||
"""Read-only validation pass in the attention-backend compatibility
|
||||
handler: the TRT-LLM and tokenspeed MLA backends constrain the resolved
|
||||
kv-cache dtype (declarations never reach the field, so the checks read
|
||||
the view)."""
|
||||
if (
|
||||
view.attention_backend == "trtllm_mla"
|
||||
or view.decode_attention_backend == "trtllm_mla"
|
||||
):
|
||||
if not is_blackwell_supported():
|
||||
raise ValueError(
|
||||
"TRTLLM MLA backend is only supported on Blackwell GPUs (SM100/SM12x). Please use a different backend."
|
||||
)
|
||||
if view.kv_cache_dtype not in ["fp8_e4m3", "fp4_e2m1", "bf16", "auto"]:
|
||||
raise ValueError(
|
||||
"TensorRT-LLM MLA backend only supports kv-cache-dtype of fp8_e4m3, fp4_e2m1, bf16, or auto."
|
||||
)
|
||||
if (
|
||||
view.attention_backend == "tokenspeed_mla"
|
||||
or view.decode_attention_backend == "tokenspeed_mla"
|
||||
):
|
||||
if not is_blackwell_supported():
|
||||
raise ValueError(
|
||||
"tokenspeed_mla backend is only supported on Blackwell GPUs (SM100/SM12x)."
|
||||
)
|
||||
if view.kv_cache_dtype not in ["fp8_e4m3"]:
|
||||
raise ValueError(
|
||||
"tokenspeed_mla backend requires kv-cache-dtype=fp8_e4m3, "
|
||||
f"got {view.kv_cache_dtype}."
|
||||
)
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _hisparse_validation(view: Any) -> dict:
|
||||
"""Read-only validation pass: --enable-hisparse constraints (model class,
|
||||
radix cache, kv dtype, DSA backends) read the resolved values through the
|
||||
view."""
|
||||
from sglang.srt.arg_groups.hisparse_hook import validate_hisparse
|
||||
|
||||
validate_hisparse(view)
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _cutedsl_prefill_backend_fill(view: Any) -> dict:
|
||||
"""Slot pass in the attention-backend compatibility handler: CuteDSL MLA
|
||||
is decode-only, so validate the combination and default the prefill side
|
||||
to trtllm_mla. The trtllm_mha check that follows at the legacy slot reads
|
||||
the dual-applied value."""
|
||||
the resolved value through the view."""
|
||||
if not (
|
||||
view.attention_backend == "cutedsl_mla"
|
||||
or view.decode_attention_backend == "cutedsl_mla"
|
||||
@@ -1610,7 +1718,7 @@ def _attention_backend_platform_fallbacks(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _intel_xpu_page_constraint(view: Any) -> dict:
|
||||
_, decode_backend = view.get_attention_backends()
|
||||
_, decode_backend = attention_backends_of(view)
|
||||
if decode_backend == "intel_xpu":
|
||||
if view.use_mla_backend():
|
||||
supported_page_sizes = [16, 32, 64, 128]
|
||||
@@ -1674,6 +1782,17 @@ def _data_parallelism_defaults(view: Any) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _dp_lm_head_validation(view: Any) -> dict:
|
||||
"""Read-only validation pass: dp-attention is a prerequisite for the
|
||||
dp LM head. Reads the mid-resolution values through the view."""
|
||||
if view.enable_dp_lm_head:
|
||||
assert (
|
||||
view.enable_dp_attention
|
||||
), "Please enable dp attention when setting enable_dp_lm_head. "
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
||||
"""The quantization-driven moe_runner_backend resolutions at the head of
|
||||
@@ -1728,6 +1847,49 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _moe_runner_fusion_disable(view: Any) -> dict:
|
||||
"""FlashInfer CuteDSL / TRT-LLM / TRT-LLM-routed MoE runners require the
|
||||
shared-experts fusion disabled; declared at the legacy write slots in
|
||||
_handle_moe_kernel_config (before the deprecated cutlass env override, so
|
||||
the runner value observed is the pre-override one)."""
|
||||
runner = view.moe_runner_backend
|
||||
if runner == "flashinfer_cutedsl":
|
||||
logger.warning(
|
||||
"FlashInfer CuteDSL MoE is enabled. --disable-shared-experts-fusion is automatically set."
|
||||
)
|
||||
return {"disable_shared_experts_fusion": True}
|
||||
if runner in ("flashinfer_trtllm", "experimental_sgl_trtllm"):
|
||||
logger.warning(
|
||||
"FlashInfer TRTLLM MoE is enabled. --disable-shared-experts-fusion is automatically set."
|
||||
)
|
||||
return {"disable_shared_experts_fusion": True}
|
||||
if runner == "flashinfer_trtllm_routed":
|
||||
logger.warning(
|
||||
"FlashInfer TRTLLM routed MoE is enabled. --disable-shared-experts-fusion is automatically set."
|
||||
)
|
||||
return {"disable_shared_experts_fusion": True}
|
||||
return {}
|
||||
|
||||
|
||||
def _a2a_fusion_adjustments(view: Any) -> dict:
|
||||
"""A2A-backend-driven shared-experts fusion adjustments, declared at the
|
||||
legacy write slots in _handle_a2a_moe: DeepEP Waterfill requires the
|
||||
fusion enabled; FlashInfer A2A requires it disabled."""
|
||||
if view.moe_a2a_backend == "deepep" and view.enable_deepep_waterfill:
|
||||
if view.disable_shared_experts_fusion:
|
||||
logger.warning(
|
||||
"disable_shared_experts_fusion is overridden to False because DeepEP Waterfill requires shared expert fusion."
|
||||
)
|
||||
return {"disable_shared_experts_fusion": False}
|
||||
return {}
|
||||
if view.moe_a2a_backend == "flashinfer":
|
||||
logger.warning(
|
||||
"Flashinfer MoE A2A is enabled. --disable-shared-experts-fusion is automatically set."
|
||||
)
|
||||
return {"disable_shared_experts_fusion": True}
|
||||
return {}
|
||||
|
||||
|
||||
def _cutlass_moe_env_override(view: Any) -> dict:
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
@@ -1846,16 +2008,25 @@ def _dllm_overlap_disable(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _dllm_page_size(view: Any) -> dict:
|
||||
if view.dllm_algorithm is None or view.disable_radix_cache:
|
||||
if view.dllm_algorithm is None:
|
||||
return {}
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
|
||||
config = DllmConfig.from_server_args(view)
|
||||
if view.page_size % config.block_size != 0:
|
||||
if not view.disable_radix_cache and view.page_size % config.block_size != 0:
|
||||
logger.warning(
|
||||
f"Setting page size to {config.block_size} for diffusion LLM inference"
|
||||
)
|
||||
return {"page_size": config.block_size}
|
||||
if view.page_size > config.block_size:
|
||||
# Legacy scheduler-init fallback, folded into the pass: the page
|
||||
# size must not exceed the dllm block size.
|
||||
logger.warning(
|
||||
"WARNING: "
|
||||
f"The page size {view.page_size} should not be larger than dllm block size {config.block_size}."
|
||||
f"Page size now falls back to {config.block_size}"
|
||||
)
|
||||
return {"page_size": config.block_size}
|
||||
return {}
|
||||
|
||||
|
||||
@@ -1937,45 +2108,35 @@ def apply_model_overrides(
|
||||
return records
|
||||
|
||||
|
||||
def apply_declarations_to_server_args(
|
||||
def validate_declarations(
|
||||
server_args: Any,
|
||||
declarations: Sequence[Tuple[str, Dict[str, Any]]],
|
||||
*,
|
||||
terminal: Sequence[Tuple[str, Dict[str, Any]]] = (),
|
||||
) -> None:
|
||||
"""Transition-period dual-apply: replay declarations onto ``server_args``
|
||||
in gate order, byte-identical to the legacy imperative writes.
|
||||
|
||||
Retired per field once that field's readers have all flipped to the flags
|
||||
tier (at which point the server_args field returns to pristine).
|
||||
|
||||
Validates against the same whitelist as the publish gate BEFORE any write:
|
||||
a registry typo or a not-yet-resolvable field must fail fast here, not
|
||||
mutate ``server_args`` and only be rejected at publish time.
|
||||
"""Fail-fast whitelist check at declaration time: a registry typo or a
|
||||
not-yet-resolvable field must be rejected at its slot, not only at
|
||||
publish time. Declarations never mutate ``server_args``.
|
||||
"""
|
||||
# Non-dataclass fixtures carry no Arg metadata (mirrors the
|
||||
# resolvable_fields escape); only real ServerArgs is validated.
|
||||
if dataclasses.is_dataclass(type(server_args)):
|
||||
whitelist = resolvable_fields(type(server_args))
|
||||
for source, decl in list(declarations) + list(terminal):
|
||||
unknown = set(decl) - whitelist
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"{source}: {sorted(unknown)} not model-overridable; the "
|
||||
"transition dual-apply refuses fields the publish gate "
|
||||
"would reject."
|
||||
)
|
||||
for _source, decl in list(declarations) + list(terminal):
|
||||
for field, value in decl.items():
|
||||
setattr(server_args, field, value)
|
||||
if not dataclasses.is_dataclass(type(server_args)):
|
||||
return
|
||||
whitelist = resolvable_fields(type(server_args))
|
||||
for source, decl in declarations:
|
||||
unknown = set(decl) - whitelist
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"{source}: {sorted(unknown)} not model-overridable; "
|
||||
"declarations are limited to the fields the publish gate "
|
||||
"accepts."
|
||||
)
|
||||
|
||||
|
||||
def refresh_declared_fields(server_args: Any, fields: Iterable[str]) -> None:
|
||||
"""Transition helper for legacy code that overwrites a resolved field
|
||||
AFTER the override collection in ``__post_init__`` (e.g.
|
||||
``ModelRunner.model_specific_adjustment`` forcing ``attention_backend``
|
||||
for HRM-Text). Redeclares the live value so publish parity holds and the
|
||||
flags tier materializes the adjusted end state.
|
||||
"""Helper for legacy code that overwrites a resolved field AFTER
|
||||
materialization (e.g. ``ModelRunner.model_specific_adjustment`` forcing
|
||||
``attention_backend`` for HRM-Text). Redeclares the live value so the
|
||||
publish parity holds and the flags tier materializes the adjusted end
|
||||
state.
|
||||
"""
|
||||
_missing = object()
|
||||
declarations = server_args._resolved_overrides
|
||||
@@ -1998,8 +2159,8 @@ def assert_flag_parity(
|
||||
*,
|
||||
leaf_map: Optional[Dict[str, str]] = None,
|
||||
) -> None:
|
||||
"""Dual-apply drift guard: each migrated field's flag leaf must equal the
|
||||
(dual-applied) ``server_args`` value."""
|
||||
"""Drift guard: each declared field's flag leaf must equal the
|
||||
(materialized) ``server_args`` value."""
|
||||
mismatches = []
|
||||
for field in fields:
|
||||
owner, leaf = resolve_flag_leaf(flags, field, leaf_map=leaf_map)
|
||||
|
||||
@@ -135,7 +135,9 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
|
||||
|
||||
|
||||
def _handle_dflash(server_args: ServerArgs) -> None:
|
||||
if server_args.enable_dp_attention:
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
if resolved_view(server_args).enable_dp_attention:
|
||||
raise ValueError(
|
||||
"Currently DFLASH speculative decoding does not support dp attention."
|
||||
)
|
||||
@@ -264,7 +266,12 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
|
||||
|
||||
draft_backend = server_args.speculative_draft_attention_backend
|
||||
if draft_backend is None:
|
||||
draft_backend, _ = server_args.get_attention_backends()
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
attention_backends_of,
|
||||
resolved_view,
|
||||
)
|
||||
|
||||
draft_backend, _ = attention_backends_of(resolved_view(server_args))
|
||||
if draft_backend is None:
|
||||
draft_backend = fallback_backend
|
||||
elif draft_backend == "trtllm_mha":
|
||||
@@ -305,9 +312,14 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None:
|
||||
|
||||
|
||||
def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
attention_backends_of,
|
||||
resolved_view,
|
||||
)
|
||||
|
||||
if (
|
||||
server_args.speculative_algorithm == "STANDALONE"
|
||||
and server_args.enable_dp_attention
|
||||
and resolved_view(server_args).enable_dp_attention
|
||||
):
|
||||
# TODO: support dp attention for standalone speculative decoding
|
||||
raise ValueError(
|
||||
@@ -320,7 +332,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
"Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests."
|
||||
)
|
||||
|
||||
if server_args.disable_overlap_schedule:
|
||||
if resolved_view(server_args).disable_overlap_schedule:
|
||||
logger.warning(
|
||||
"Non-overlap (synchronous) spec v2 is used for eagle/eagle3/standalone "
|
||||
"speculative decoding."
|
||||
@@ -375,11 +387,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
server_args.speculative_num_draft_tokens,
|
||||
) = _auto_choose_speculative_params(server_args, model_arch)
|
||||
|
||||
if (
|
||||
server_args.attention_backend == "trtllm_mha"
|
||||
or server_args.decode_attention_backend == "trtllm_mha"
|
||||
or server_args.prefill_attention_backend == "trtllm_mha"
|
||||
):
|
||||
if "trtllm_mha" in attention_backends_of(resolved_view(server_args)):
|
||||
if server_args.speculative_eagle_topk > 1:
|
||||
raise ValueError(
|
||||
"trtllm_mha backend only supports topk = 1 for speculative decoding."
|
||||
@@ -416,7 +424,9 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
"--enable-deterministic-inference; the sampling kernel draws "
|
||||
"coins from the global RNG and is not batch-invariant."
|
||||
)
|
||||
if server_args.enable_multi_layer_eagle:
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
if resolved_view(server_args).enable_multi_layer_eagle:
|
||||
raise NotImplementedError(
|
||||
"--speculative-use-rejection-sampling is not supported with "
|
||||
"multi-layer EAGLE (--enable-multi-layer-eagle)."
|
||||
@@ -440,15 +450,16 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
# pass + per-branch expand pass with prefix-tail dup). Only these backends implement
|
||||
# it; flashmla / trtllm_mla / cutlass_mla can't express the per-branch tree, so reject.
|
||||
_PAGE_TREE_SPEC_BACKENDS = ("flashinfer", "fa3", "triton")
|
||||
view = resolved_view(server_args)
|
||||
if (
|
||||
server_args.speculative_eagle_topk > 1
|
||||
and server_args.page_size > 1
|
||||
and server_args.attention_backend not in _PAGE_TREE_SPEC_BACKENDS
|
||||
and view.page_size > 1
|
||||
and view.attention_backend not in _PAGE_TREE_SPEC_BACKENDS
|
||||
):
|
||||
raise ValueError(
|
||||
f"speculative_eagle_topk > 1 with page_size > 1 is only supported on "
|
||||
f"{_PAGE_TREE_SPEC_BACKENDS}; got attention_backend="
|
||||
f"{server_args.attention_backend!r}. Use page_size == 1 or one of those backends."
|
||||
f"{view.attention_backend!r}. Use page_size == 1 or one of those backends."
|
||||
)
|
||||
|
||||
|
||||
@@ -499,18 +510,21 @@ def _handle_ngram(server_args: ServerArgs) -> None:
|
||||
"using ngram speculative decoding."
|
||||
)
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
view = resolved_view(server_args)
|
||||
if (
|
||||
server_args.speculative_eagle_topk > 1
|
||||
and server_args.page_size > 1
|
||||
and server_args.attention_backend != "flashinfer"
|
||||
and view.page_size > 1
|
||||
and view.attention_backend != "flashinfer"
|
||||
):
|
||||
raise ValueError(
|
||||
f"speculative_eagle_topk({server_args.speculative_eagle_topk}) > 1 "
|
||||
f"with page_size({server_args.page_size}) > 1 is unstable "
|
||||
f"with page_size({view.page_size}) > 1 is unstable "
|
||||
"and produces incorrect results for paged attention backends. "
|
||||
"This combination is only supported for the 'flashinfer' backend."
|
||||
)
|
||||
if server_args.enable_dp_attention:
|
||||
if view.enable_dp_attention:
|
||||
# TODO: support dp attention for ngram speculative decoding
|
||||
raise ValueError(
|
||||
"Currently ngram speculative decoding does not support dp attention."
|
||||
|
||||
+273
-231
File diff suppressed because it is too large
Load Diff
@@ -49,6 +49,8 @@ DEFAULT_ADAPTIVE_CONFIG: dict[str, dict] = {
|
||||
|
||||
def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
|
||||
"""Return why adaptive spec cannot run under the given server args, or None if supported."""
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
if server_args.speculative_algorithm not in ("EAGLE", "EAGLE3"):
|
||||
return (
|
||||
f"speculative_algorithm={server_args.speculative_algorithm} "
|
||||
@@ -62,12 +64,12 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
|
||||
f"speculative_eagle_topk={server_args.speculative_eagle_topk} "
|
||||
"(only topk=1 is supported)"
|
||||
)
|
||||
if server_args.enable_dp_attention:
|
||||
if resolved_view(server_args).enable_dp_attention:
|
||||
return (
|
||||
"enable_dp_attention=True is not supported "
|
||||
"(adaptive tier decisions are not synchronized across DP ranks)"
|
||||
)
|
||||
if server_args.enable_multi_layer_eagle:
|
||||
if resolved_view(server_args).enable_multi_layer_eagle:
|
||||
return (
|
||||
"enable_multi_layer_eagle=True is not supported "
|
||||
"(MultiLayerEagleWorkerV2 does not implement adaptive)"
|
||||
|
||||
@@ -17,6 +17,10 @@ register_amd_ci(est_time=60, suite="extra-a-test-1-gpu-small-amd")
|
||||
|
||||
|
||||
def _make_server_args(*, sampling_backend: str) -> SimpleNamespace:
|
||||
# The install gate reads the resolved backend from the flags tier.
|
||||
from sglang.srt.runtime_context import get_flags
|
||||
|
||||
get_flags().sampling_backend = sampling_backend
|
||||
return SimpleNamespace(sampling_backend=sampling_backend)
|
||||
|
||||
|
||||
|
||||
@@ -102,6 +102,10 @@ def _make_model_runner(
|
||||
|
||||
sa = SimpleNamespace()
|
||||
sa.swa_full_tokens_ratio = swa_full_tokens_ratio
|
||||
# The configurator reads the resolved ratio from the flags tier.
|
||||
from sglang.srt.runtime_context import get_flags
|
||||
|
||||
get_flags().swa_full_tokens_ratio = swa_full_tokens_ratio
|
||||
sa.page_size = page_size
|
||||
sa.disable_radix_cache = disable_radix_cache
|
||||
sa.chunked_prefill_size = chunked_prefill_size
|
||||
|
||||
@@ -42,7 +42,7 @@ class TestDeepseekV4SharedExpertFusionPolicy(unittest.TestCase):
|
||||
|
||||
self.assertEqual(model.num_fused_shared_experts, 0)
|
||||
self.assertTrue(get_flags().disable_shared_experts_fusion)
|
||||
# dual-apply transition: the published config carries the value too
|
||||
# post-init declaration writes through to the published config
|
||||
self.assertTrue(server_args.disable_shared_experts_fusion)
|
||||
|
||||
def test_enables_shared_fusion_when_enforced(self):
|
||||
|
||||
@@ -354,7 +354,10 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
|
||||
|
||||
args._handle_attention_backend_compatibility()
|
||||
|
||||
self.assertEqual(args.page_size, 128)
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
self.assertEqual(args.page_size, 1) # dual-apply retired: pristine
|
||||
self.assertEqual(resolved_view(args).page_size, 128)
|
||||
|
||||
@patch("sglang.srt.arg_groups.overrides.is_sm100_supported", return_value=True)
|
||||
@patch("sglang.srt.server_args.ServerArgs.use_mla_backend", return_value=False)
|
||||
@@ -364,7 +367,10 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
|
||||
|
||||
args._handle_attention_backend_compatibility()
|
||||
|
||||
self.assertEqual(args.page_size, 128)
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
self.assertEqual(args.page_size, 1) # dual-apply retired: pristine
|
||||
self.assertEqual(resolved_view(args).page_size, 128)
|
||||
|
||||
|
||||
class TestContextParallelServerArgs(CustomTestCase):
|
||||
@@ -1119,7 +1125,11 @@ class TestDeepEPWaterfillArgs(CustomTestCase):
|
||||
# dummy-model path short-circuits __post_init__; invoke the handler directly.
|
||||
server_args._handle_a2a_moe()
|
||||
|
||||
self.assertFalse(server_args.disable_shared_experts_fusion)
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
# dual-apply retired: the fields stay pristine, the declarations win
|
||||
self.assertTrue(server_args.disable_shared_experts_fusion)
|
||||
self.assertFalse(resolved_view(server_args).disable_shared_experts_fusion)
|
||||
self.assertTrue(server_args.enforce_shared_experts_fusion)
|
||||
|
||||
def test_waterfill_overrides_moe_a2a_backend_to_deepep(self):
|
||||
@@ -1131,7 +1141,10 @@ class TestDeepEPWaterfillArgs(CustomTestCase):
|
||||
# dummy-model path short-circuits __post_init__; invoke the handler directly.
|
||||
server_args._handle_a2a_moe()
|
||||
|
||||
self.assertEqual(server_args.moe_a2a_backend, "deepep")
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
self.assertEqual(server_args.moe_a2a_backend, "none") # pristine
|
||||
self.assertEqual(resolved_view(server_args).moe_a2a_backend, "deepep")
|
||||
self.assertTrue(server_args.enforce_shared_experts_fusion)
|
||||
|
||||
def test_waterfill_supports_deepep_low_latency_mode(self):
|
||||
|
||||
@@ -19,11 +19,10 @@ from sglang.srt.arg_groups import overrides as overrides_module
|
||||
from sglang.srt.arg_groups.arg_utils import A, Arg, resolvable_fields
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
OverrideRecord,
|
||||
apply_declarations_to_server_args,
|
||||
apply_model_overrides,
|
||||
assert_flag_parity,
|
||||
collect_model_override_declarations,
|
||||
register_model_override,
|
||||
validate_declarations,
|
||||
)
|
||||
from sglang.srt.runtime_context import (
|
||||
_StaticFlags,
|
||||
@@ -207,7 +206,7 @@ class TestResolvedViewAndPasses(CustomTestCase):
|
||||
self.assertEqual(view.a, 10)
|
||||
self.assertEqual(view.b, 2)
|
||||
|
||||
def test_run_pass_appends_stash_and_dual_applies(self):
|
||||
def test_run_pass_appends_stash_and_stays_pristine(self):
|
||||
from sglang.srt.arg_groups.overrides import run_post_process_pass
|
||||
|
||||
live = SimpleNamespace(x=None, _resolved_overrides=[])
|
||||
@@ -216,11 +215,12 @@ class TestResolvedViewAndPasses(CustomTestCase):
|
||||
return {"x": "filled"} if view.x is None else {}
|
||||
|
||||
run_post_process_pass(live, _fill_x)
|
||||
self.assertEqual(live.x, "filled") # dual-applied in place
|
||||
self.assertIsNone(live.x) # never applied in place
|
||||
self.assertEqual(
|
||||
live._resolved_overrides, [(_fill_x.__qualname__, {"x": "filled"})]
|
||||
)
|
||||
run_post_process_pass(live, _fill_x) # now a no-op
|
||||
# the next invocation sees the declared value through the overlay
|
||||
run_post_process_pass(live, _fill_x)
|
||||
self.assertEqual(len(live._resolved_overrides), 1)
|
||||
|
||||
def test_run_pass_rejects_non_dict(self):
|
||||
@@ -415,7 +415,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
def test_mistral_large3_forces_bfloat16(self):
|
||||
sa = self._construct("MistralLarge3ForCausalLM", "mistral")
|
||||
self.assertEqual(sa.dtype, "bfloat16") # dual-apply == legacy write
|
||||
self.assertEqual(sa.dtype, "bfloat16") # materialized at end of resolution
|
||||
self.assertIn(
|
||||
("MODEL_OVERRIDES['MistralLarge3ForCausalLM']", {"dtype": "bfloat16"}),
|
||||
sa._resolved_overrides,
|
||||
@@ -424,15 +424,15 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
def test_pixtral_forces_bfloat16(self):
|
||||
sa = self._construct("PixtralForConditionalGeneration", "pixtral")
|
||||
self.assertEqual(sa.dtype, "bfloat16")
|
||||
self.assertEqual(sa.dtype, "bfloat16") # materialized
|
||||
self.assertEqual(self._publish(sa).dtype, "bfloat16")
|
||||
|
||||
def test_user_requested_dtype_is_still_overridden(self):
|
||||
# Legacy fidelity: the arch branch overwrote dtype unconditionally,
|
||||
# so the declaration must too. The pristine request survives only on
|
||||
# provenance (and, post-V3, as the un-overridden server_args field).
|
||||
# so the declaration must too. The pristine request survives on
|
||||
# provenance; the materialized field carries the override.
|
||||
sa = self._construct("MistralLarge3ForCausalLM", "mistral", dtype="float16")
|
||||
self.assertEqual(sa.dtype, "bfloat16")
|
||||
self.assertEqual(sa.dtype, "bfloat16") # materialized
|
||||
self.assertEqual(self._publish(sa).dtype, "bfloat16")
|
||||
|
||||
def test_control_arch_keeps_pristine_dtype(self):
|
||||
@@ -446,7 +446,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
def test_minimax_m2_enables_tf32_matmul(self):
|
||||
sa = self._construct("MiniMaxM2ForCausalLM", "llama")
|
||||
self.assertTrue(sa.enable_tf32_matmul) # dual-apply == legacy write
|
||||
self.assertTrue(sa.enable_tf32_matmul) # materialized
|
||||
self.assertIn(
|
||||
("_minimax_m2_overrides", {"enable_tf32_matmul": True}),
|
||||
sa._resolved_overrides,
|
||||
@@ -491,7 +491,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
config_extra=config_extra,
|
||||
enable_hierarchical_cache=True,
|
||||
)
|
||||
# dual-apply == legacy writes
|
||||
# materialized at the end of resolution
|
||||
self.assertEqual(sa.swa_full_tokens_ratio, 1.0)
|
||||
self.assertTrue(sa.disable_hybrid_swa_memory)
|
||||
flags = self._publish(sa)
|
||||
@@ -500,7 +500,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
def test_gemma2_disables_hybrid_swa_memory(self):
|
||||
sa = self._construct("Gemma2ForCausalLM", "llama")
|
||||
self.assertTrue(sa.disable_hybrid_swa_memory) # dual-apply == legacy
|
||||
self.assertTrue(sa.disable_hybrid_swa_memory) # materialized
|
||||
self.assertIn(
|
||||
("_gemma2_gemma3_overrides", {"disable_hybrid_swa_memory": True}),
|
||||
sa._resolved_overrides,
|
||||
@@ -509,7 +509,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
def test_olmo2_disables_hybrid_swa_memory(self):
|
||||
sa = self._construct("Olmo2ForCausalLM", "llama")
|
||||
self.assertTrue(sa.disable_hybrid_swa_memory)
|
||||
self.assertTrue(sa.disable_hybrid_swa_memory) # materialized
|
||||
self.assertTrue(self._publish(sa).disable_hybrid_swa_memory)
|
||||
|
||||
def test_exaone_conditional_on_sliding_window_pattern(self):
|
||||
@@ -520,7 +520,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
config_extra={"sliding_window_pattern": "LLLG"},
|
||||
attention_backend="fa3",
|
||||
)
|
||||
self.assertTrue(sa.disable_hybrid_swa_memory)
|
||||
self.assertTrue(sa.disable_hybrid_swa_memory) # materialized
|
||||
self.assertTrue(self._publish(sa).disable_hybrid_swa_memory)
|
||||
|
||||
def test_exaone_without_pattern_declares_nothing(self):
|
||||
@@ -543,7 +543,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
"llama",
|
||||
config_extra={"quantization_config": {"quant_method": "mxfp4"}},
|
||||
)
|
||||
self.assertEqual(sa.dtype, "bfloat16") # dual-apply == legacy
|
||||
self.assertEqual(sa.dtype, "bfloat16") # materialized
|
||||
self.assertEqual(self._publish(sa).dtype, "bfloat16")
|
||||
|
||||
def test_gpt_oss_without_mxfp4_keeps_pristine_dtype(self):
|
||||
@@ -569,7 +569,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
sa = self._construct("LlamaForCausalLM", "llama")
|
||||
expected = "flashinfer" if is_flashinfer_available() else "pytorch"
|
||||
self.assertEqual(sa.sampling_backend, expected)
|
||||
self.assertEqual(sa.sampling_backend, expected) # materialized
|
||||
self.assertIn(
|
||||
("_sampling_backend_default", {"sampling_backend": expected}),
|
||||
sa._resolved_overrides,
|
||||
@@ -587,20 +587,20 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
"LlamaForCausalLM", "llama", enable_deterministic_inference=True
|
||||
)
|
||||
# two pass writers chain: default fill, then the deterministic force —
|
||||
# last writer wins on the flags leaf and parity holds end-to-end.
|
||||
# last writer wins; materialization lands the end state on the fields.
|
||||
self.assertEqual(sa.sampling_backend, "pytorch")
|
||||
flags = self._publish(sa)
|
||||
self.assertEqual(flags.sampling_backend, "pytorch")
|
||||
# the deterministic attention fill declared a compatible backend and
|
||||
# the compatibility default-fill then had nothing to do
|
||||
self.assertIn(
|
||||
(
|
||||
"_deterministic_attention_backend",
|
||||
{"attention_backend": sa.attention_backend},
|
||||
),
|
||||
sa._resolved_overrides,
|
||||
)
|
||||
self.assertEqual(flags.attn.backend, sa.attention_backend)
|
||||
deterministic_fills = [
|
||||
decl["attention_backend"]
|
||||
for source, decl in sa._resolved_overrides
|
||||
if source == "_deterministic_attention_backend"
|
||||
]
|
||||
self.assertEqual(len(deterministic_fills), 1)
|
||||
self.assertEqual(sa.attention_backend, deterministic_fills[0])
|
||||
self.assertEqual(flags.attn.backend, deterministic_fills[0])
|
||||
|
||||
def test_deterministic_incompatible_backend_raises(self):
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
@@ -631,13 +631,15 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
def test_dllm_forces_flashinfer_with_cuda_graph(self):
|
||||
# CUDA path: cuda graph enabled by default -> dllm forces flashinfer.
|
||||
# A real dllm arch: the page pass now runs regardless of the radix
|
||||
# switch and builds DllmConfig for it.
|
||||
sa = self._construct(
|
||||
"LlamaForCausalLM",
|
||||
"SDARForCausalLM",
|
||||
"llama",
|
||||
dllm_algorithm="LowConfidence",
|
||||
disable_radix_cache=True,
|
||||
)
|
||||
self.assertEqual(sa.attention_backend, "flashinfer")
|
||||
self.assertEqual(sa.attention_backend, "flashinfer") # materialized
|
||||
self.assertIn(
|
||||
("_dllm_attention_backend", {"attention_backend": "flashinfer"}),
|
||||
sa._resolved_overrides,
|
||||
@@ -647,24 +649,35 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
def test_attention_backend_leaf_materializes_end_state(self):
|
||||
# The default-fill pass declares the platform-selected backend; the
|
||||
# leaf must equal the final server_args value (publish parity).
|
||||
# leaf must equal the last declared value while the server_args field
|
||||
# stays pristine (dual-apply retired).
|
||||
sa = self._construct("LlamaForCausalLM", "llama")
|
||||
declared = {f for _s, d in sa._resolved_overrides for f in d}
|
||||
self.assertIn("attention_backend", declared) # default fill declared
|
||||
self.assertEqual(self._publish(sa).attn.backend, sa.attention_backend)
|
||||
declared_values = [
|
||||
d["attention_backend"]
|
||||
for _s, d in sa._resolved_overrides
|
||||
if "attention_backend" in d
|
||||
]
|
||||
self.assertTrue(declared_values) # default fill declared
|
||||
self.assertEqual(sa.attention_backend, declared_values[-1]) # materialized
|
||||
self.assertEqual(self._publish(sa).attn.backend, declared_values[-1])
|
||||
|
||||
def test_runner_side_adjustment_can_refresh_declaration(self):
|
||||
from sglang.srt.arg_groups.overrides import refresh_declared_fields
|
||||
def test_post_materialize_pass_writes_through(self):
|
||||
from sglang.srt.arg_groups.overrides import run_post_process_pass
|
||||
|
||||
# A pass invoked after materialization (a post-init slot, like the
|
||||
# legacy runner-side adjustments) declares AND writes through, so
|
||||
# field readers and the publish see the same end state.
|
||||
sa = self._construct("LlamaForCausalLM", "llama")
|
||||
declared = {f for _s, d in sa._resolved_overrides for f in d}
|
||||
self.assertIn("attention_backend", declared)
|
||||
# Simulate a legacy runner-side overwrite between collection and publish
|
||||
# (model_specific_adjustment forces attention_backend for HRM-Text).
|
||||
sa.attention_backend = "fa3" if sa.attention_backend != "fa3" else "triton"
|
||||
with self.assertRaises(AssertionError):
|
||||
self._publish(sa) # stale declaration breaks parity
|
||||
refresh_declared_fields(sa, ("attention_backend",))
|
||||
resolved_before = sa.attention_backend
|
||||
|
||||
def _force_triton(view):
|
||||
if view.attention_backend != "triton":
|
||||
return {"attention_backend": "triton"}
|
||||
return {}
|
||||
|
||||
run_post_process_pass(sa, _force_triton)
|
||||
if resolved_before != "triton":
|
||||
self.assertEqual(sa.attention_backend, "triton")
|
||||
self.assertEqual(self._publish(sa).attn.backend, sa.attention_backend)
|
||||
|
||||
def test_attention_backend_user_choice_declares_nothing_extra(self):
|
||||
@@ -798,9 +811,41 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
return_value=SimpleNamespace(block_size=32),
|
||||
):
|
||||
self.assertEqual(_view() and _dllm_page_size(_view()), {"page_size": 32})
|
||||
self.assertEqual(_dllm_page_size(_view(page_size=64)), {}) # aligned
|
||||
# aligned but larger than the block: the scheduler-init fallback
|
||||
# (folded into this pass) still caps the page at the block size
|
||||
self.assertEqual(_dllm_page_size(_view(page_size=64)), {"page_size": 32})
|
||||
self.assertEqual(_dllm_page_size(_view(page_size=32)), {}) # equal
|
||||
# radix disabled skips the alignment fill but keeps the cap
|
||||
self.assertEqual(_dllm_page_size(_view(disable_radix_cache=True)), {})
|
||||
self.assertEqual(
|
||||
_dllm_page_size(_view(disable_radix_cache=True, page_size=64)),
|
||||
{"page_size": 32},
|
||||
)
|
||||
self.assertEqual(_dllm_page_size(_view(dllm_algorithm=None)), {})
|
||||
self.assertEqual(_dllm_page_size(_view(disable_radix_cache=True)), {})
|
||||
|
||||
def test_declaration_overlay_mechanics(self):
|
||||
from sglang.srt.arg_groups.overrides import run_post_process_pass
|
||||
|
||||
live = SimpleNamespace(x="user", y=None, _resolved_overrides=[])
|
||||
|
||||
def _resolve_x(view):
|
||||
return {"x": "resolved"} if view.x == "user" else {}
|
||||
|
||||
def _read_x(view):
|
||||
return {"y": view.x}
|
||||
|
||||
run_post_process_pass(live, _resolve_x)
|
||||
# declaration recorded, but server_args stays pristine
|
||||
self.assertEqual(
|
||||
live._resolved_overrides, [(_resolve_x.__qualname__, {"x": "resolved"})]
|
||||
)
|
||||
self.assertEqual(live.x, "user")
|
||||
# a later pass sees the resolved value through the view overlay
|
||||
run_post_process_pass(live, _read_x)
|
||||
self.assertEqual(
|
||||
live._resolved_overrides[-1], (_read_x.__qualname__, {"y": "resolved"})
|
||||
)
|
||||
self.assertIsNone(live.y) # never applied in place
|
||||
|
||||
def test_overlap_disable_passes(self):
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
@@ -1531,9 +1576,12 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
|
||||
def test_page_size_leaf_materializes_end_state(self):
|
||||
sa = self._construct("LlamaForCausalLM", "llama")
|
||||
declared = {f for _s, d in sa._resolved_overrides for f in d}
|
||||
self.assertIn("page_size", declared) # default fill declared
|
||||
self.assertEqual(self._publish(sa).page_size, sa.page_size)
|
||||
declared_values = [
|
||||
d["page_size"] for _s, d in sa._resolved_overrides if "page_size" in d
|
||||
]
|
||||
self.assertTrue(declared_values) # default fill declared
|
||||
self.assertEqual(sa.page_size, declared_values[-1]) # materialized
|
||||
self.assertEqual(self._publish(sa).page_size, declared_values[-1])
|
||||
|
||||
def test_qwen3_5_hybrid_coupled_declaration(self):
|
||||
from sglang.srt.arg_groups.overrides import _qwen3_5_hybrid_overrides
|
||||
@@ -1544,7 +1592,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
_get_default_attn_backend=lambda **_: default_backend,
|
||||
use_mla_backend=lambda: False,
|
||||
get_model_config=lambda: None,
|
||||
enable_mamba_extra_buffer=lambda: False,
|
||||
mamba_radix_cache_strategy="auto",
|
||||
disable_radix_cache=False,
|
||||
speculative_algorithm=None,
|
||||
)
|
||||
@@ -1571,6 +1619,24 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
),
|
||||
{},
|
||||
)
|
||||
# the mamba pass ran before this dispatch and stashed the
|
||||
# extra-buffer strategy: the callable must see it through the
|
||||
# view (SM100 hybrid keeps trtllm_mha + page 64)
|
||||
self.assertEqual(
|
||||
_qwen3_5_hybrid_overrides(
|
||||
_args(
|
||||
"trtllm_mha",
|
||||
_resolved_overrides=[
|
||||
(
|
||||
"_mamba_radix_cache_declarations",
|
||||
{"mamba_radix_cache_strategy": "extra_buffer"},
|
||||
)
|
||||
],
|
||||
),
|
||||
None,
|
||||
),
|
||||
{"attention_backend": "trtllm_mha", "page_size": 64},
|
||||
)
|
||||
with patch.object(overrides_module, "is_sm100_supported", return_value=False):
|
||||
self.assertEqual(_qwen3_5_hybrid_overrides(_args("fa3"), None), {})
|
||||
|
||||
@@ -1738,7 +1804,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
self.assertEqual(
|
||||
_intel_xpu_page_constraint(
|
||||
_view(
|
||||
get_attention_backends=lambda: (None, "intel_xpu"),
|
||||
decode_attention_backend="intel_xpu",
|
||||
use_mla_backend=lambda: False,
|
||||
)
|
||||
),
|
||||
@@ -1747,7 +1813,7 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
self.assertEqual(
|
||||
_intel_xpu_page_constraint(
|
||||
_view(
|
||||
get_attention_backends=lambda: (None, "intel_xpu"),
|
||||
decode_attention_backend="intel_xpu",
|
||||
use_mla_backend=lambda: True,
|
||||
page_size=16, # MLA decode accepts 16
|
||||
)
|
||||
@@ -2089,22 +2155,22 @@ class TestGoldenModelOverrides(_IsolatedPublish):
|
||||
self.assertEqual(_step3p_overrides(_args(), None), {})
|
||||
|
||||
|
||||
class TestDualApplyParity(CustomTestCase):
|
||||
def test_dual_apply_replays_and_parity_holds(self):
|
||||
class TestDeclarationValidation(CustomTestCase):
|
||||
def test_declarations_never_mutate_server_args(self):
|
||||
flags, args = _FakeFlags(), _FakeArgs()
|
||||
declarations = [("src", {"resolved_by_model": "dsv4", "also_resolved": 7})]
|
||||
apply_model_overrides(flags, args, declarations)
|
||||
apply_declarations_to_server_args(args, declarations)
|
||||
self.assertEqual(args.resolved_by_model, "dsv4")
|
||||
self.assertEqual(args.also_resolved, 7)
|
||||
assert_flag_parity(flags, args, ["resolved_by_model", "also_resolved"])
|
||||
validate_declarations(args, declarations)
|
||||
# the leaves carry the declared values; the fields stay pristine
|
||||
self.assertEqual(flags.resolved_by_model, "dsv4")
|
||||
self.assertEqual(flags.also_resolved, 7)
|
||||
self.assertEqual(args.resolved_by_model, _FakeArgs.resolved_by_model)
|
||||
self.assertEqual(args.also_resolved, _FakeArgs.also_resolved)
|
||||
|
||||
def test_parity_detects_drift(self):
|
||||
flags, args = _FakeFlags(), _FakeArgs()
|
||||
apply_model_overrides(flags, args, [("src", {"resolved_by_model": "x"})])
|
||||
# dual-apply skipped -> server_args still pristine -> drift is caught
|
||||
with self.assertRaises(AssertionError):
|
||||
assert_flag_parity(flags, args, ["resolved_by_model"])
|
||||
def test_validation_rejects_unknown_fields(self):
|
||||
args = _FakeArgs()
|
||||
with self.assertRaises(ValueError):
|
||||
validate_declarations(args, [("src", {"nope": 1})])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -339,15 +339,6 @@ class TestRuntimeResolutionStages(_IsolatedServerArgs):
|
||||
self.assertEqual(get_flags().sampling_backend, "pytorch")
|
||||
self.assertEqual(get_flags().page_size, 64) # earlier stage survives
|
||||
|
||||
def test_record_parity_failure_rolls_back(self):
|
||||
self._publish(page_size=1)
|
||||
flags_before = get_flags()
|
||||
with self.assertRaises(AssertionError):
|
||||
# declared value diverges from the live server_args (no dual-apply)
|
||||
get_context().record_runtime_overrides([("bad", {"page_size": 64})])
|
||||
self.assertIs(get_flags(), flags_before) # previous flags intact
|
||||
self.assertEqual(get_context()._runtime_overrides, []) # rolled back
|
||||
|
||||
def test_record_whitelist_violation_rolls_back(self):
|
||||
self._publish()
|
||||
with self.assertRaises(ValueError):
|
||||
@@ -366,13 +357,15 @@ class TestRuntimeResolutionStages(_IsolatedServerArgs):
|
||||
finally:
|
||||
reset_context()
|
||||
|
||||
def test_declare_load_time_override_dual_applies_and_records(self):
|
||||
def test_declare_load_time_override_applies_and_records(self):
|
||||
from sglang.srt.arg_groups.overrides import declare_load_time_override
|
||||
|
||||
args = self._publish(page_size=1)
|
||||
declare_load_time_override("model.load_time", {"page_size": 64})
|
||||
self.assertEqual(args.page_size, 64) # dual-applied onto server_args
|
||||
self.assertEqual(get_flags().page_size, 64) # resolved into the leaf
|
||||
# post-init declaration: written through to the field and resolved
|
||||
# into the leaf
|
||||
self.assertEqual(args.page_size, 64)
|
||||
self.assertEqual(get_flags().page_size, 64)
|
||||
self.assertEqual(
|
||||
get_context()._runtime_overrides,
|
||||
[("model.load_time", {"page_size": 64})],
|
||||
@@ -415,6 +408,21 @@ class TestRuntimeResolutionStages(_IsolatedServerArgs):
|
||||
finally:
|
||||
reset_context()
|
||||
|
||||
def test_declared_leaf_wins_over_stale_field(self):
|
||||
# A stash entry always drives the leaf at publish, even if the field
|
||||
# value diverged (e.g. a fixture that skipped materialization).
|
||||
@dataclasses.dataclass
|
||||
class _Args:
|
||||
enable_dp_lm_head: A[bool, Arg(help="d", resolvable=True)] = True
|
||||
_resolved_overrides: list = dataclasses.field(default_factory=list)
|
||||
|
||||
args = _Args()
|
||||
args._resolved_overrides = [("dp", {"enable_dp_lm_head": False})]
|
||||
args._declarations_materialized = True
|
||||
args.enable_dp_lm_head = False
|
||||
get_context().set_server_args(args)
|
||||
self.assertFalse(get_flags().enable_dp_lm_head)
|
||||
|
||||
def test_bare_dataclass_publish_skips_materialization(self):
|
||||
# object.__new__(ServerArgs) fixtures (no __init__, no field values)
|
||||
# must publish without touching the flags tier — dataclass defaults
|
||||
|
||||
Reference in New Issue
Block a user