[refactor] Resolve config declarations onto server_args at the end of __post_init__ (#30297)
This commit is contained in:
@@ -43,21 +43,30 @@ def _hisparse_allowed_backends(kv_cache_dtype: str) -> set[str]:
|
||||
def validate_hisparse_dsa_backend(
|
||||
server_args: ServerArgs, attr: str, label: str
|
||||
) -> None:
|
||||
backend = getattr(server_args, attr)
|
||||
allowed_backends = _hisparse_allowed_backends(server_args.kv_cache_dtype)
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
# Invoked after the DSA kv-cache-dtype / split-backend declarations:
|
||||
# read the resolving state through the view.
|
||||
view = resolved_view(server_args)
|
||||
backend = getattr(view, attr)
|
||||
kv_cache_dtype = view.kv_cache_dtype
|
||||
allowed_backends = _hisparse_allowed_backends(kv_cache_dtype)
|
||||
if backend is not None and backend not in allowed_backends:
|
||||
raise ValueError(
|
||||
f"HiSparse supports DSA {label} backend(s) {sorted(allowed_backends)} "
|
||||
f"on this platform with --kv-cache-dtype={server_args.kv_cache_dtype}, "
|
||||
f"on this platform with --kv-cache-dtype={kv_cache_dtype}, "
|
||||
f"but got --dsa-{label}-backend={backend}. "
|
||||
f"Please use --dsa-{label}-backend="
|
||||
f"{_hisparse_default_backend(server_args.kv_cache_dtype)} "
|
||||
f"{_hisparse_default_backend(kv_cache_dtype)} "
|
||||
"or omit it."
|
||||
)
|
||||
|
||||
|
||||
def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
|
||||
if server_args.kv_cache_dtype in HISPARSE_KV_CACHE_DTYPES:
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
kv_cache_dtype = resolved_view(server_args).kv_cache_dtype
|
||||
if kv_cache_dtype in HISPARSE_KV_CACHE_DTYPES:
|
||||
return
|
||||
|
||||
choices = " or ".join(
|
||||
@@ -65,7 +74,7 @@ def validate_hisparse_kv_cache_dtype(server_args: ServerArgs) -> None:
|
||||
)
|
||||
raise ValueError(
|
||||
f"HiSparse requires one of {HISPARSE_KV_CACHE_DTYPES} KV cache dtypes, "
|
||||
f"but got --kv-cache-dtype={server_args.kv_cache_dtype}. Please use {choices}."
|
||||
f"but got --kv-cache-dtype={kv_cache_dtype}. Please use {choices}."
|
||||
)
|
||||
|
||||
|
||||
@@ -112,7 +121,13 @@ def validate_hisparse(server_args: ServerArgs) -> None:
|
||||
)
|
||||
return
|
||||
|
||||
if server_args.kv_cache_dtype not in ("bfloat16", "auto", "fp8_e4m3"):
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
if resolved_view(server_args).kv_cache_dtype not in (
|
||||
"bfloat16",
|
||||
"auto",
|
||||
"fp8_e4m3",
|
||||
):
|
||||
validate_hisparse_kv_cache_dtype(server_args)
|
||||
|
||||
for attr, label in [
|
||||
|
||||
@@ -117,14 +117,10 @@ def _invoke_provider(
|
||||
|
||||
class ResolvedView:
|
||||
"""Read-only view of the resolving configuration handed to post-process
|
||||
passes.
|
||||
|
||||
During the dual-apply transition the view forwards every read to the live
|
||||
``server_args`` — the pristine input plus the declarations replayed so far
|
||||
plus any residual imperative writes — which is exactly the state the
|
||||
legacy handler at the same slot observed. In the end state (dual-apply
|
||||
retired) the same type overlays the accumulated declarations on the
|
||||
pristine object. Writes are rejected: passes return declarations.
|
||||
passes: the accumulated declarations overlaid on the pristine
|
||||
``server_args`` (residual imperative writes of non-resolved fields show
|
||||
through the fallthrough) — exactly the state the legacy handler at the
|
||||
same slot observed. Writes are rejected: passes return declarations.
|
||||
"""
|
||||
|
||||
__slots__ = ("_server_args", "_overlay")
|
||||
@@ -162,14 +158,27 @@ def register_post_process(fn: Callable[..., dict]) -> Callable[..., dict]:
|
||||
return fn
|
||||
|
||||
|
||||
def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
"""Transition-period invocation of one pass at its legacy handler slot.
|
||||
def _declaration_overlay(server_args: Any) -> Dict[str, Any]:
|
||||
"""Accumulated declared values: declarations never mutate
|
||||
``server_args``, so mid-resolution readers overlay them from the
|
||||
declaration stash (last writer wins, like the gate)."""
|
||||
overlay: Dict[str, Any] = {}
|
||||
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
|
||||
overlay.update(declared)
|
||||
return overlay
|
||||
|
||||
Evaluates the pass on the live state (through a read-only view), appends
|
||||
its declaration to the declaration stash, and dual-applies it in place —
|
||||
byte-identical to the imperative handler write this replaces.
|
||||
|
||||
def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
"""Invoke one pass at its legacy handler slot.
|
||||
|
||||
Evaluates the pass on the resolving state (a read-only view with the
|
||||
accumulated declarations overlaid from the stash) and appends its
|
||||
declaration to the stash. During ``__post_init__`` the fields stay
|
||||
untouched — ``materialize_declarations`` applies the whole stash once at
|
||||
the end of resolution; a pass invoked after materialization (a post-init
|
||||
slot) writes through immediately.
|
||||
"""
|
||||
declared = fn(ResolvedView(server_args))
|
||||
declared = fn(ResolvedView(server_args, overlay=_declaration_overlay(server_args)))
|
||||
if not isinstance(declared, dict):
|
||||
raise TypeError(
|
||||
f"post-process pass {fn.__qualname__} must return a dict, "
|
||||
@@ -186,19 +195,70 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
|
||||
# must sit at or after it in __post_init__ order.
|
||||
stash = server_args._resolved_overrides = []
|
||||
stash.append(entry)
|
||||
apply_declarations_to_server_args(server_args, [entry])
|
||||
validate_declarations(server_args, [entry])
|
||||
if getattr(server_args, "_declarations_materialized", False):
|
||||
for field, value in declared.items():
|
||||
setattr(server_args, field, value)
|
||||
|
||||
|
||||
def materialize_declarations(server_args: Any) -> None:
|
||||
"""Apply the accumulated declarations onto ``server_args`` once, at the
|
||||
end of ``__post_init__`` (gate order: last writer wins). After this the
|
||||
fields carry the resolved configuration — every post-init reader, in any
|
||||
process, reads them directly; ``resolved_view`` remains an internal
|
||||
helper for mid-resolution code only."""
|
||||
for _source, declared in getattr(server_args, "_resolved_overrides", None) or ():
|
||||
for field, value in declared.items():
|
||||
setattr(server_args, field, value)
|
||||
server_args._declarations_materialized = True
|
||||
|
||||
|
||||
def resolved_view(server_args: Any) -> ResolvedView:
|
||||
"""Read-only view of the resolving configuration for mid-resolution code
|
||||
that is not a pass (``__post_init__`` handlers and hooks). Internal to
|
||||
the resolution pipeline: after ``materialize_declarations`` runs, the
|
||||
fields themselves carry the resolved values — read them directly."""
|
||||
return ResolvedView(server_args, overlay=_declaration_overlay(server_args))
|
||||
|
||||
|
||||
def attention_backends_of(cfg: Any) -> tuple:
|
||||
"""(prefill, decode) attention backends of a config-shaped object (a
|
||||
ResolvedView mid-resolution, or pristine server_args at dispatch time):
|
||||
split fields fall back to the base backend."""
|
||||
prefill = (
|
||||
cfg.prefill_attention_backend
|
||||
if cfg.prefill_attention_backend
|
||||
else cfg.attention_backend
|
||||
)
|
||||
decode = (
|
||||
cfg.decode_attention_backend
|
||||
if cfg.decode_attention_backend
|
||||
else cfg.attention_backend
|
||||
)
|
||||
return prefill, decode
|
||||
|
||||
|
||||
def mamba_extra_buffer_of(cfg: Any) -> bool:
|
||||
"""Mid-resolution equivalent of runtime_context.mamba_extra_buffer_enabled:
|
||||
reads the (possibly overlaid) strategy from a config-shaped object."""
|
||||
return cfg.disable_radix_cache is False and cfg.mamba_radix_cache_strategy in (
|
||||
"extra_buffer",
|
||||
"extra_buffer_lazy",
|
||||
)
|
||||
|
||||
|
||||
def declare_load_time_override(source: str, declared: Dict[str, Any]) -> None:
|
||||
"""Transition helper for load-time resolved fields (model-file config
|
||||
overrides, weight-resolved dtypes): dual-apply the declaration onto the
|
||||
published ``server_args`` — byte-identical to the imperative write this
|
||||
replaces — and record it into the flags tier through the runtime gate."""
|
||||
"""Declare a load-time resolved field (model-file config overrides,
|
||||
weight-resolved dtypes): apply it onto the published ``server_args`` —
|
||||
resolution has already materialized, so post-init declarations write
|
||||
through — and record it into the flags tier through the runtime gate."""
|
||||
from sglang.srt.runtime_context import get_context
|
||||
|
||||
ctx = get_context()
|
||||
entry = (source, dict(declared))
|
||||
apply_declarations_to_server_args(ctx.server_args, [entry])
|
||||
validate_declarations(ctx.server_args, [entry])
|
||||
for field, value in declared.items():
|
||||
setattr(ctx.server_args, field, value)
|
||||
ctx.record_runtime_overrides([entry])
|
||||
|
||||
|
||||
@@ -741,8 +801,11 @@ def _qwen3_5_hybrid_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
use_mla_backend=server_args.use_mla_backend(),
|
||||
model_config=server_args.get_model_config(),
|
||||
)
|
||||
# The mamba radix-cache pass runs before this dispatch: read the
|
||||
# declared strategy through the view (the legacy branch observed the
|
||||
# already-written field here).
|
||||
if default_attn_backend == "trtllm_mha" and not (
|
||||
not server_args.enable_mamba_extra_buffer()
|
||||
not mamba_extra_buffer_of(resolved_view(server_args))
|
||||
and not server_args.disable_radix_cache
|
||||
and server_args.speculative_algorithm is None
|
||||
):
|
||||
@@ -1519,12 +1582,57 @@ def _mla_backend_page_constraints(view: Any) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _mla_kv_cache_dtype_checks(view: Any) -> dict:
|
||||
"""Read-only validation pass in the attention-backend compatibility
|
||||
handler: the TRT-LLM and tokenspeed MLA backends constrain the resolved
|
||||
kv-cache dtype (declarations never reach the field, so the checks read
|
||||
the view)."""
|
||||
if (
|
||||
view.attention_backend == "trtllm_mla"
|
||||
or view.decode_attention_backend == "trtllm_mla"
|
||||
):
|
||||
if not is_blackwell_supported():
|
||||
raise ValueError(
|
||||
"TRTLLM MLA backend is only supported on Blackwell GPUs (SM100/SM12x). Please use a different backend."
|
||||
)
|
||||
if view.kv_cache_dtype not in ["fp8_e4m3", "fp4_e2m1", "bf16", "auto"]:
|
||||
raise ValueError(
|
||||
"TensorRT-LLM MLA backend only supports kv-cache-dtype of fp8_e4m3, fp4_e2m1, bf16, or auto."
|
||||
)
|
||||
if (
|
||||
view.attention_backend == "tokenspeed_mla"
|
||||
or view.decode_attention_backend == "tokenspeed_mla"
|
||||
):
|
||||
if not is_blackwell_supported():
|
||||
raise ValueError(
|
||||
"tokenspeed_mla backend is only supported on Blackwell GPUs (SM100/SM12x)."
|
||||
)
|
||||
if view.kv_cache_dtype not in ["fp8_e4m3"]:
|
||||
raise ValueError(
|
||||
"tokenspeed_mla backend requires kv-cache-dtype=fp8_e4m3, "
|
||||
f"got {view.kv_cache_dtype}."
|
||||
)
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _hisparse_validation(view: Any) -> dict:
|
||||
"""Read-only validation pass: --enable-hisparse constraints (model class,
|
||||
radix cache, kv dtype, DSA backends) read the resolved values through the
|
||||
view."""
|
||||
from sglang.srt.arg_groups.hisparse_hook import validate_hisparse
|
||||
|
||||
validate_hisparse(view)
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _cutedsl_prefill_backend_fill(view: Any) -> dict:
|
||||
"""Slot pass in the attention-backend compatibility handler: CuteDSL MLA
|
||||
is decode-only, so validate the combination and default the prefill side
|
||||
to trtllm_mla. The trtllm_mha check that follows at the legacy slot reads
|
||||
the dual-applied value."""
|
||||
the resolved value through the view."""
|
||||
if not (
|
||||
view.attention_backend == "cutedsl_mla"
|
||||
or view.decode_attention_backend == "cutedsl_mla"
|
||||
@@ -1610,7 +1718,7 @@ def _attention_backend_platform_fallbacks(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _intel_xpu_page_constraint(view: Any) -> dict:
|
||||
_, decode_backend = view.get_attention_backends()
|
||||
_, decode_backend = attention_backends_of(view)
|
||||
if decode_backend == "intel_xpu":
|
||||
if view.use_mla_backend():
|
||||
supported_page_sizes = [16, 32, 64, 128]
|
||||
@@ -1674,6 +1782,17 @@ def _data_parallelism_defaults(view: Any) -> dict:
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _dp_lm_head_validation(view: Any) -> dict:
|
||||
"""Read-only validation pass: dp-attention is a prerequisite for the
|
||||
dp LM head. Reads the mid-resolution values through the view."""
|
||||
if view.enable_dp_lm_head:
|
||||
assert (
|
||||
view.enable_dp_attention
|
||||
), "Please enable dp attention when setting enable_dp_lm_head. "
|
||||
return {}
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
||||
"""The quantization-driven moe_runner_backend resolutions at the head of
|
||||
@@ -1728,6 +1847,49 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
||||
|
||||
|
||||
@register_post_process
|
||||
def _moe_runner_fusion_disable(view: Any) -> dict:
|
||||
"""FlashInfer CuteDSL / TRT-LLM / TRT-LLM-routed MoE runners require the
|
||||
shared-experts fusion disabled; declared at the legacy write slots in
|
||||
_handle_moe_kernel_config (before the deprecated cutlass env override, so
|
||||
the runner value observed is the pre-override one)."""
|
||||
runner = view.moe_runner_backend
|
||||
if runner == "flashinfer_cutedsl":
|
||||
logger.warning(
|
||||
"FlashInfer CuteDSL MoE is enabled. --disable-shared-experts-fusion is automatically set."
|
||||
)
|
||||
return {"disable_shared_experts_fusion": True}
|
||||
if runner in ("flashinfer_trtllm", "experimental_sgl_trtllm"):
|
||||
logger.warning(
|
||||
"FlashInfer TRTLLM MoE is enabled. --disable-shared-experts-fusion is automatically set."
|
||||
)
|
||||
return {"disable_shared_experts_fusion": True}
|
||||
if runner == "flashinfer_trtllm_routed":
|
||||
logger.warning(
|
||||
"FlashInfer TRTLLM routed MoE is enabled. --disable-shared-experts-fusion is automatically set."
|
||||
)
|
||||
return {"disable_shared_experts_fusion": True}
|
||||
return {}
|
||||
|
||||
|
||||
def _a2a_fusion_adjustments(view: Any) -> dict:
|
||||
"""A2A-backend-driven shared-experts fusion adjustments, declared at the
|
||||
legacy write slots in _handle_a2a_moe: DeepEP Waterfill requires the
|
||||
fusion enabled; FlashInfer A2A requires it disabled."""
|
||||
if view.moe_a2a_backend == "deepep" and view.enable_deepep_waterfill:
|
||||
if view.disable_shared_experts_fusion:
|
||||
logger.warning(
|
||||
"disable_shared_experts_fusion is overridden to False because DeepEP Waterfill requires shared expert fusion."
|
||||
)
|
||||
return {"disable_shared_experts_fusion": False}
|
||||
return {}
|
||||
if view.moe_a2a_backend == "flashinfer":
|
||||
logger.warning(
|
||||
"Flashinfer MoE A2A is enabled. --disable-shared-experts-fusion is automatically set."
|
||||
)
|
||||
return {"disable_shared_experts_fusion": True}
|
||||
return {}
|
||||
|
||||
|
||||
def _cutlass_moe_env_override(view: Any) -> dict:
|
||||
from sglang.srt.environ import envs
|
||||
|
||||
@@ -1846,16 +2008,25 @@ def _dllm_overlap_disable(view: Any) -> dict:
|
||||
|
||||
@register_post_process
|
||||
def _dllm_page_size(view: Any) -> dict:
|
||||
if view.dllm_algorithm is None or view.disable_radix_cache:
|
||||
if view.dllm_algorithm is None:
|
||||
return {}
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
|
||||
config = DllmConfig.from_server_args(view)
|
||||
if view.page_size % config.block_size != 0:
|
||||
if not view.disable_radix_cache and view.page_size % config.block_size != 0:
|
||||
logger.warning(
|
||||
f"Setting page size to {config.block_size} for diffusion LLM inference"
|
||||
)
|
||||
return {"page_size": config.block_size}
|
||||
if view.page_size > config.block_size:
|
||||
# Legacy scheduler-init fallback, folded into the pass: the page
|
||||
# size must not exceed the dllm block size.
|
||||
logger.warning(
|
||||
"WARNING: "
|
||||
f"The page size {view.page_size} should not be larger than dllm block size {config.block_size}."
|
||||
f"Page size now falls back to {config.block_size}"
|
||||
)
|
||||
return {"page_size": config.block_size}
|
||||
return {}
|
||||
|
||||
|
||||
@@ -1937,45 +2108,35 @@ def apply_model_overrides(
|
||||
return records
|
||||
|
||||
|
||||
def apply_declarations_to_server_args(
|
||||
def validate_declarations(
|
||||
server_args: Any,
|
||||
declarations: Sequence[Tuple[str, Dict[str, Any]]],
|
||||
*,
|
||||
terminal: Sequence[Tuple[str, Dict[str, Any]]] = (),
|
||||
) -> None:
|
||||
"""Transition-period dual-apply: replay declarations onto ``server_args``
|
||||
in gate order, byte-identical to the legacy imperative writes.
|
||||
|
||||
Retired per field once that field's readers have all flipped to the flags
|
||||
tier (at which point the server_args field returns to pristine).
|
||||
|
||||
Validates against the same whitelist as the publish gate BEFORE any write:
|
||||
a registry typo or a not-yet-resolvable field must fail fast here, not
|
||||
mutate ``server_args`` and only be rejected at publish time.
|
||||
"""Fail-fast whitelist check at declaration time: a registry typo or a
|
||||
not-yet-resolvable field must be rejected at its slot, not only at
|
||||
publish time. Declarations never mutate ``server_args``.
|
||||
"""
|
||||
# Non-dataclass fixtures carry no Arg metadata (mirrors the
|
||||
# resolvable_fields escape); only real ServerArgs is validated.
|
||||
if dataclasses.is_dataclass(type(server_args)):
|
||||
whitelist = resolvable_fields(type(server_args))
|
||||
for source, decl in list(declarations) + list(terminal):
|
||||
unknown = set(decl) - whitelist
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"{source}: {sorted(unknown)} not model-overridable; the "
|
||||
"transition dual-apply refuses fields the publish gate "
|
||||
"would reject."
|
||||
)
|
||||
for _source, decl in list(declarations) + list(terminal):
|
||||
for field, value in decl.items():
|
||||
setattr(server_args, field, value)
|
||||
if not dataclasses.is_dataclass(type(server_args)):
|
||||
return
|
||||
whitelist = resolvable_fields(type(server_args))
|
||||
for source, decl in declarations:
|
||||
unknown = set(decl) - whitelist
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"{source}: {sorted(unknown)} not model-overridable; "
|
||||
"declarations are limited to the fields the publish gate "
|
||||
"accepts."
|
||||
)
|
||||
|
||||
|
||||
def refresh_declared_fields(server_args: Any, fields: Iterable[str]) -> None:
|
||||
"""Transition helper for legacy code that overwrites a resolved field
|
||||
AFTER the override collection in ``__post_init__`` (e.g.
|
||||
``ModelRunner.model_specific_adjustment`` forcing ``attention_backend``
|
||||
for HRM-Text). Redeclares the live value so publish parity holds and the
|
||||
flags tier materializes the adjusted end state.
|
||||
"""Helper for legacy code that overwrites a resolved field AFTER
|
||||
materialization (e.g. ``ModelRunner.model_specific_adjustment`` forcing
|
||||
``attention_backend`` for HRM-Text). Redeclares the live value so the
|
||||
publish parity holds and the flags tier materializes the adjusted end
|
||||
state.
|
||||
"""
|
||||
_missing = object()
|
||||
declarations = server_args._resolved_overrides
|
||||
@@ -1998,8 +2159,8 @@ def assert_flag_parity(
|
||||
*,
|
||||
leaf_map: Optional[Dict[str, str]] = None,
|
||||
) -> None:
|
||||
"""Dual-apply drift guard: each migrated field's flag leaf must equal the
|
||||
(dual-applied) ``server_args`` value."""
|
||||
"""Drift guard: each declared field's flag leaf must equal the
|
||||
(materialized) ``server_args`` value."""
|
||||
mismatches = []
|
||||
for field in fields:
|
||||
owner, leaf = resolve_flag_leaf(flags, field, leaf_map=leaf_map)
|
||||
|
||||
@@ -135,7 +135,9 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
|
||||
|
||||
|
||||
def _handle_dflash(server_args: ServerArgs) -> None:
|
||||
if server_args.enable_dp_attention:
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
if resolved_view(server_args).enable_dp_attention:
|
||||
raise ValueError(
|
||||
"Currently DFLASH speculative decoding does not support dp attention."
|
||||
)
|
||||
@@ -264,7 +266,12 @@ def _resolve_dflash_draft_attention_backend(server_args: ServerArgs) -> None:
|
||||
|
||||
draft_backend = server_args.speculative_draft_attention_backend
|
||||
if draft_backend is None:
|
||||
draft_backend, _ = server_args.get_attention_backends()
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
attention_backends_of,
|
||||
resolved_view,
|
||||
)
|
||||
|
||||
draft_backend, _ = attention_backends_of(resolved_view(server_args))
|
||||
if draft_backend is None:
|
||||
draft_backend = fallback_backend
|
||||
elif draft_backend == "trtllm_mha":
|
||||
@@ -305,9 +312,14 @@ def _handle_frozen_kv_mtp(server_args: ServerArgs) -> None:
|
||||
|
||||
|
||||
def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
from sglang.srt.arg_groups.overrides import (
|
||||
attention_backends_of,
|
||||
resolved_view,
|
||||
)
|
||||
|
||||
if (
|
||||
server_args.speculative_algorithm == "STANDALONE"
|
||||
and server_args.enable_dp_attention
|
||||
and resolved_view(server_args).enable_dp_attention
|
||||
):
|
||||
# TODO: support dp attention for standalone speculative decoding
|
||||
raise ValueError(
|
||||
@@ -320,7 +332,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
"Max running requests is reset to 48 for speculative decoding. You can override this by explicitly setting --max-running-requests."
|
||||
)
|
||||
|
||||
if server_args.disable_overlap_schedule:
|
||||
if resolved_view(server_args).disable_overlap_schedule:
|
||||
logger.warning(
|
||||
"Non-overlap (synchronous) spec v2 is used for eagle/eagle3/standalone "
|
||||
"speculative decoding."
|
||||
@@ -375,11 +387,7 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
server_args.speculative_num_draft_tokens,
|
||||
) = _auto_choose_speculative_params(server_args, model_arch)
|
||||
|
||||
if (
|
||||
server_args.attention_backend == "trtllm_mha"
|
||||
or server_args.decode_attention_backend == "trtllm_mha"
|
||||
or server_args.prefill_attention_backend == "trtllm_mha"
|
||||
):
|
||||
if "trtllm_mha" in attention_backends_of(resolved_view(server_args)):
|
||||
if server_args.speculative_eagle_topk > 1:
|
||||
raise ValueError(
|
||||
"trtllm_mha backend only supports topk = 1 for speculative decoding."
|
||||
@@ -416,7 +424,9 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
"--enable-deterministic-inference; the sampling kernel draws "
|
||||
"coins from the global RNG and is not batch-invariant."
|
||||
)
|
||||
if server_args.enable_multi_layer_eagle:
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
if resolved_view(server_args).enable_multi_layer_eagle:
|
||||
raise NotImplementedError(
|
||||
"--speculative-use-rejection-sampling is not supported with "
|
||||
"multi-layer EAGLE (--enable-multi-layer-eagle)."
|
||||
@@ -440,15 +450,16 @@ def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
# pass + per-branch expand pass with prefix-tail dup). Only these backends implement
|
||||
# it; flashmla / trtllm_mla / cutlass_mla can't express the per-branch tree, so reject.
|
||||
_PAGE_TREE_SPEC_BACKENDS = ("flashinfer", "fa3", "triton")
|
||||
view = resolved_view(server_args)
|
||||
if (
|
||||
server_args.speculative_eagle_topk > 1
|
||||
and server_args.page_size > 1
|
||||
and server_args.attention_backend not in _PAGE_TREE_SPEC_BACKENDS
|
||||
and view.page_size > 1
|
||||
and view.attention_backend not in _PAGE_TREE_SPEC_BACKENDS
|
||||
):
|
||||
raise ValueError(
|
||||
f"speculative_eagle_topk > 1 with page_size > 1 is only supported on "
|
||||
f"{_PAGE_TREE_SPEC_BACKENDS}; got attention_backend="
|
||||
f"{server_args.attention_backend!r}. Use page_size == 1 or one of those backends."
|
||||
f"{view.attention_backend!r}. Use page_size == 1 or one of those backends."
|
||||
)
|
||||
|
||||
|
||||
@@ -499,18 +510,21 @@ def _handle_ngram(server_args: ServerArgs) -> None:
|
||||
"using ngram speculative decoding."
|
||||
)
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
view = resolved_view(server_args)
|
||||
if (
|
||||
server_args.speculative_eagle_topk > 1
|
||||
and server_args.page_size > 1
|
||||
and server_args.attention_backend != "flashinfer"
|
||||
and view.page_size > 1
|
||||
and view.attention_backend != "flashinfer"
|
||||
):
|
||||
raise ValueError(
|
||||
f"speculative_eagle_topk({server_args.speculative_eagle_topk}) > 1 "
|
||||
f"with page_size({server_args.page_size}) > 1 is unstable "
|
||||
f"with page_size({view.page_size}) > 1 is unstable "
|
||||
"and produces incorrect results for paged attention backends. "
|
||||
"This combination is only supported for the 'flashinfer' backend."
|
||||
)
|
||||
if server_args.enable_dp_attention:
|
||||
if view.enable_dp_attention:
|
||||
# TODO: support dp attention for ngram speculative decoding
|
||||
raise ValueError(
|
||||
"Currently ngram speculative decoding does not support dp attention."
|
||||
|
||||
+273
-231
File diff suppressed because it is too large
Load Diff
@@ -49,6 +49,8 @@ DEFAULT_ADAPTIVE_CONFIG: dict[str, dict] = {
|
||||
|
||||
def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
|
||||
"""Return why adaptive spec cannot run under the given server args, or None if supported."""
|
||||
from sglang.srt.arg_groups.overrides import resolved_view
|
||||
|
||||
if server_args.speculative_algorithm not in ("EAGLE", "EAGLE3"):
|
||||
return (
|
||||
f"speculative_algorithm={server_args.speculative_algorithm} "
|
||||
@@ -62,12 +64,12 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
|
||||
f"speculative_eagle_topk={server_args.speculative_eagle_topk} "
|
||||
"(only topk=1 is supported)"
|
||||
)
|
||||
if server_args.enable_dp_attention:
|
||||
if resolved_view(server_args).enable_dp_attention:
|
||||
return (
|
||||
"enable_dp_attention=True is not supported "
|
||||
"(adaptive tier decisions are not synchronized across DP ranks)"
|
||||
)
|
||||
if server_args.enable_multi_layer_eagle:
|
||||
if resolved_view(server_args).enable_multi_layer_eagle:
|
||||
return (
|
||||
"enable_multi_layer_eagle=True is not supported "
|
||||
"(MultiLayerEagleWorkerV2 does not implement adaptive)"
|
||||
|
||||
Reference in New Issue
Block a user