config: the resolution callbacks into the record go to zero (#36972)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
48b88e1256
commit
b65e677e48
@@ -1672,7 +1672,7 @@ def max_prefill_buffer_tokens() -> int:
|
||||
|
||||
Every input is a published leaf (``schedule`` plus the configured PP size),
|
||||
so this derives from the bags and follows a post-publish override;
|
||||
``ServerArgs.max_prefill_buffer_tokens`` is the pre-publish equivalent and
|
||||
``overrides.max_prefill_buffer_tokens`` is the pre-publish equivalent and
|
||||
``TestDerivedPredicatesAgreeAcrossTiers`` pins the two equal.
|
||||
"""
|
||||
import math
|
||||
@@ -1726,17 +1726,19 @@ def pre_capture_activation_reserve_mb(gpu_mem: float | None) -> float:
|
||||
# --- Derived config accessors ------------------------------------------------
|
||||
#
|
||||
# A few values are computed from several config fields plus the HF config, so
|
||||
# they are ``ServerArgs`` members rather than namespace leaves. Business code
|
||||
# must not reach for the startup record to get them: these accessors are the
|
||||
# named home, and this module — which owns the slot — is the only place that
|
||||
# reads it. Each one keeps the member's exact semantics, including which model
|
||||
# they are derived accessors rather than namespace leaves. Business code must
|
||||
# not reach for the startup record to get them: these accessors are the named
|
||||
# home, and this module — which owns the slot — is the only place that reads
|
||||
# it. Each one keeps the pre-publish function's exact semantics, including which model
|
||||
# config it derives from (always the process's, i.e. the target's).
|
||||
|
||||
|
||||
def mamba_cache_chunk_size() -> int:
|
||||
"""The caching point granularity for mamba state: ``max(the model's mamba
|
||||
chunk size, page_size)``. Cached on the config after the first call."""
|
||||
return get_server_args().mamba_cache_chunk_size
|
||||
from sglang.srt.arg_groups.overrides import mamba_cache_chunk_size as _of
|
||||
|
||||
return _of(get_server_args())
|
||||
|
||||
|
||||
def mamba_checkpoint_grid(tree_page: int) -> int:
|
||||
@@ -1759,7 +1761,7 @@ def max_speculative_num_draft_tokens() -> int | None:
|
||||
"""The largest draft-token count speculative decoding may use.
|
||||
|
||||
All three inputs are ``spec`` leaves, so this derives from the bags and
|
||||
follows a post-publish override; ``ServerArgs.max_speculative_num_draft_tokens``
|
||||
follows a post-publish override; ``overrides.max_speculative_num_draft_tokens``
|
||||
is the pre-publish equivalent. Adaptive spec resolves the count from its
|
||||
candidate-step table instead of the flat field.
|
||||
"""
|
||||
@@ -1788,7 +1790,9 @@ def _adaptive_draft_token_bound(cfg_path: str | None) -> int:
|
||||
|
||||
def uses_mla_backend() -> bool:
|
||||
"""Whether this process's model runs the MLA attention path."""
|
||||
return get_server_args().use_mla_backend()
|
||||
from sglang.srt.arg_groups.overrides import use_mla_backend
|
||||
|
||||
return use_mla_backend(get_server_args())
|
||||
|
||||
|
||||
def attention_backends() -> tuple:
|
||||
@@ -1796,7 +1800,7 @@ def attention_backends() -> tuple:
|
||||
back to ``attention_backend``.
|
||||
|
||||
All three inputs are ``exec.kernel`` leaves, so this derives from the bags
|
||||
and follows a post-publish override; ``ServerArgs.get_attention_backends``
|
||||
and follows a post-publish override; ``overrides.attention_backends_of``
|
||||
is the pre-publish equivalent the resolution pipeline uses. A built runner
|
||||
stamps its own resolved pair (``ModelRunner.prefill_attention_backend_str``);
|
||||
read that when there is a runner in hand.
|
||||
@@ -1810,7 +1814,9 @@ def attention_backends() -> tuple:
|
||||
|
||||
def process_model_config():
|
||||
"""The process's ``ModelConfig`` (built once from the published config)."""
|
||||
return get_server_args().get_model_config()
|
||||
from sglang.srt.arg_groups.overrides import model_config_of
|
||||
|
||||
return model_config_of(get_server_args())
|
||||
|
||||
|
||||
def cutedsl_moe_max_num_tokens() -> int:
|
||||
@@ -1818,7 +1824,7 @@ def cutedsl_moe_max_num_tokens() -> int:
|
||||
|
||||
Every input is a published leaf (``spec``, ``schedule``, ``exec.graph``), so
|
||||
this derives from the bags and follows a post-publish override;
|
||||
``ServerArgs.cutedsl_moe_max_num_tokens`` is the pre-publish equivalent the
|
||||
``overrides.cutedsl_moe_max_num_tokens`` is the pre-publish equivalent the
|
||||
resolution pipeline uses. Max over the prefill bound, the piecewise-prefill
|
||||
capture, and the decode/verify bound.
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user