minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 15 Packages Projects Releases Wiki Activity

config: the record is not an object that gets passed around (#36622)

Browse Source 
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Cheng Wan
2026-08-27 12:57:10 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent fd40a331bf
commit 6ff2a20ccf
23 changed files with 234 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/arg_groups/overrides.py
+6 -10
View File
@@ -265,12 +265,8 @@ def run_post_process_pass(server_args: Any, fn: Callable[..., dict]) -> None:
def _apply_fields(server_args: Any, fields: Dict[str, Any]) -> None:
"""Write record fields past the guard that forbids post-resolution writes.
Resolution declares, so nothing in the pipeline calls this. It exists for
``RuntimeContext.override_server_args``, the launch stand-in tests use: there
the caller's values are both the operator's input and resolution's answer.
"""
"""Write fields on behalf of the pipeline (bypasses the strict bare-
assignment guard that protects post-resolution mutation)."""
object.__setattr__(server_args, "_internal_write", True)
try:
for field, value in fields.items():
@@ -299,7 +295,7 @@ def declare_resolution(server_args: Any, source: str, **fields: Any) -> None:
stash = getattr(server_args, "_resolved_overrides", None)
if stash is None:
stash = []
object.__setattr__(server_args, "_resolved_overrides", stash)
server_args._resolved_overrides = stash
stash.append((source, dict(fields)))
@@ -333,12 +329,12 @@ def declare_late_resolution(server_args: Any, source: str, **fields: Any) -> Non
log = getattr(server_args, "_runtime_mutations", None)
if log is None:
log = []
object.__setattr__(server_args, "_runtime_mutations", log)
server_args._runtime_mutations = log
log.append((source, dict(fields)))
stash = getattr(server_args, "_resolved_overrides", None)
if stash is None:
stash = []
object.__setattr__(server_args, "_resolved_overrides", stash)
server_args._resolved_overrides = stash
stash.append((source, dict(fields)))
@@ -376,7 +372,7 @@ def declare_direct_writes(
stash = getattr(server_args, "_resolved_overrides", None)
if stash is None:
stash = []
object.__setattr__(server_args, "_resolved_overrides", stash)
server_args._resolved_overrides = stash
# A resolver reached this way can also declare properly -- the in-tree
# implementations of these hooks do. Those fields are already explained, and
# recording them again would attribute them to the wrapper and bury an
python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py
-3
View File
@@ -26,7 +26,6 @@ from sglang.srt.runtime_context import (
get_schedule,
get_serving,
)
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils.common import ceil_align
if TYPE_CHECKING:
@@ -44,7 +43,6 @@ class DecodeKVCacheOffloadManager:
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
tp_group: torch.distributed.ProcessGroup,
tree_cache: BasePrefixCache,
server_args: ServerArgs,
) -> None:
self.req_to_token_pool = req_to_token_pool
self.token_to_kv_pool_allocator = token_to_kv_pool_allocator
@@ -64,7 +62,6 @@ class DecodeKVCacheOffloadManager:
self.decode_host_mem_pool = build_kv_host_pool(
kv_pool=kv_cache,
page_size=self.page_size,
server_args=server_args,
use_mla=isinstance(kv_cache, MLATokenToKVPool),
)
python/sglang/srt/elastic_ep/elastic_ep.py
+1 -1
View File
@@ -10,11 +10,11 @@ import torch
from sglang.srt.distributed import get_world_group, parallel_state
from sglang.srt.distributed.utils import get_global_tcp_store
from sglang.srt.eplb.expert_location import broadcast_global_expert_location_metadata
from sglang.srt.managers.schedule_batch import ServerArgs
from sglang.srt.runtime_context import (
get_exec,
get_parallel,
)
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import is_cpu, is_cuda
if TYPE_CHECKING:
python/sglang/srt/entrypoints/http_server.py
+4 -4
View File
@@ -228,13 +228,13 @@ async def init_multi_tokenizer() -> ServerArgs:
server_args: ServerArgs
port_args: PortArgs
publish(server_args, role="tokenizer")
# API key authentication is not supported in multi-tokenizer mode
assert (
server_args.api_key is None
get_serving().api_key is None
), "API key is not supported in multi-tokenizer mode"
publish(server_args, role="tokenizer")
# Create a new ipc name for the current process
port_args.tokenizer_ipc_name = (
f"ipc://{tempfile.NamedTemporaryFile(delete=False).name}"
@@ -2754,7 +2754,7 @@ def _start_native_grpc_server_for_runtime(
host=get_serving().host,
port=grpc_port,
runtime_handle=runtime_handle,
worker_threads=server_args.grpc_worker_threads,
worker_threads=get_serving().grpc_worker_threads,
)
logger.info(f"Native gRPC server started on {get_serving().host}:{grpc_port}")
return grpc_handle
python/sglang/srt/managers/rust_server.py
+1 -1
View File
@@ -187,7 +187,7 @@ class NativeMmHost:
import_processors("sglang.srt.multimodal.processors")
if mm_process_pkg := envs.SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE.get():
import_processors(mm_process_pkg, overwrite=True)
self._processor = processor or get_processor_wrapper(self.server_args)
self._processor = processor or get_processor_wrapper()
def resolve_native_spec(self) -> Optional[NativeMmSpec]:
"""The :class:`NativeMmSpec` for this model, or ``None`` when it has no
python/sglang/srt/managers/schedule_batch.py
+4 -14
View File
@@ -140,7 +140,6 @@ from sglang.srt.observability.req_time_stats import (
from sglang.srt.runtime_context import get_parallel
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
from sglang.srt.sampling.sampling_params import SamplingParams
from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import flatten_nested_list
from sglang.srt.utils.token_sequence_matcher import TokenSequenceMatcher
@@ -1960,7 +1959,6 @@ def release_req(
*,
req: Req,
remaing_req_count: int,
server_args: ServerArgs,
req_to_token_pool: ReqToTokenPool,
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
tree_cache: BasePrefixCache,
@@ -1997,7 +1995,6 @@ def release_req(
def retract_all(
*,
reqs: List[Req],
server_args: ServerArgs,
req_to_token_pool: ReqToTokenPool,
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
tree_cache: BasePrefixCache,
@@ -2008,7 +2005,6 @@ def retract_all(
release_req(
req=reqs[idx],
remaing_req_count=len(reqs) - idx,
server_args=server_args,
req_to_token_pool=req_to_token_pool,
token_to_kv_pool_allocator=token_to_kv_pool_allocator,
tree_cache=tree_cache,
@@ -2877,9 +2873,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
evict_from_tree_cache(self.tree_cache, num_tokens)
return self.token_to_kv_pool_allocator.available_size() >= num_tokens
def retract_decode(
self, server_args: ServerArgs
) -> Tuple[List[Req], float, List[Req]]:
def retract_decode(self) -> Tuple[List[Req], float, List[Req]]:
"""Retract the decoding requests when there is not enough memory."""
sorted_indices = self._get_decode_retraction_order(self.reqs)
sorted_indices = beam_retraction_order(sorted_indices, self.reqs)
@@ -2913,12 +2907,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
self.token_to_kv_pool_allocator,
)
# Aborting, so a host backup to resume from would be wasted.
self.release_req(
idx, len(sorted_indices), server_args, offload_kv=False
)
self.release_req(idx, len(sorted_indices), offload_kv=False)
continue
# release memory and don't insert into the tree because we need the space instantly
if self.release_req(idx, len(sorted_indices), server_args):
if self.release_req(idx, len(sorted_indices)):
retracted_reqs.append(req)
else:
# The retraction host pool could not hold the backup and the
@@ -2953,7 +2945,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
self.req_to_token_pool,
self.token_to_kv_pool_allocator,
)
self.release_req(last_idx, 0, server_args, offload_kv=False)
self.release_req(last_idx, 0, offload_kv=False)
logger.warning(
"retract_decode: aborted last request %s due to OOM", last_req.rid
)
@@ -3012,13 +3004,11 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
self,
idx: int,
remaing_req_count: int,
server_args: ServerArgs,
offload_kv: bool = True,
) -> bool:
return release_req(
req=self.reqs[idx],
remaing_req_count=remaing_req_count,
server_args=server_args,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
tree_cache=self.tree_cache,
python/sglang/srt/managers/schedule_policy.py
+1 -1
View File
@@ -1498,7 +1498,7 @@ class PrefillAdder:
)
release_counter += 1
self.running_batch.release_req(
i, len(self.running_batch.reqs) - release_counter, server_args
i, len(self.running_batch.reqs) - release_counter
)
else:
keep_indices.append(i)
python/sglang/srt/managers/scheduler.py
+3 -8
View File
@@ -598,7 +598,6 @@ class Scheduler(
else self.tp_cpu_group
),
tree_cache=self.tree_cache,
server_args=self.server_args,
)
else:
self.decode_offload_manager = None
@@ -3680,9 +3679,7 @@ class Scheduler(
if mamba_allocator is not None
else None
)
retracted_reqs, new_token_ratio, reqs_to_abort = batch.retract_decode(
self.server_args
)
retracted_reqs, new_token_ratio, reqs_to_abort = batch.retract_decode()
new_available_tokens = self.token_to_kv_pool_allocator.available_size()
new_token_gained = new_available_tokens - old_available_tokens
mamba_num_gained = (
@@ -4595,9 +4592,8 @@ class Scheduler(
if envs.SGLANG_EXPOSE_OWN_ENV_VARS.get():
ret["env_vars"] = exportable_env_vars()
# These fields are not msgpack-serializable (a config object and a bound
# signal handler); no reader consumes them.
ret.pop("model_config", None)
# A bound signal handler is not msgpack-serializable, and no reader
# consumes it.
ret.pop("custom_sigquit_handler", None)
return GetInternalStateReqOutput(internal_state=msgspec_to_builtins(ret))
@@ -4950,7 +4946,6 @@ class Scheduler(
# discarded. Non-decode modes ignore offload_kv (they never offload).
retract_all(
reqs=retract_reqs,
server_args=self.server_args,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
tree_cache=self.tree_cache,
python/sglang/srt/managers/tokenizer_manager.py
+2 -2
View File
@@ -490,7 +490,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
import_processors("sglang.srt.multimodal.processors")
if mm_process_pkg := envs.SGLANG_EXTERNAL_MM_PROCESSOR_PACKAGE.get():
import_processors(mm_process_pkg, overwrite=True)
_processor = get_processor_wrapper(server_args)
_processor = get_processor_wrapper()
transport_mode = determine_tensor_transport_mode()
# We want to parallelize the image pre-processing so we create an executor for it
@@ -3602,7 +3602,7 @@ async def print_exception_wrapper(func):
sys.exit(1)
def get_processor_wrapper(server_args):
def get_processor_wrapper():
return get_processor(
get_serving().tokenizer_path,
tokenizer_mode=get_serving().tokenizer_mode,
python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py
-20
View File
@@ -97,7 +97,6 @@ def build_kv_host_pool(
*,
kv_pool: Any,
page_size: int,
server_args: ServerArgs,
use_mla: bool,
override_kv_cache_dim: Optional[int] = None,
host_size: Optional[float] = None,
@@ -179,7 +178,6 @@ def build_pool_entry(
def build_kv_only_group(
*,
page_size: int,
server_args: ServerArgs,
kv_pool: Any,
full_layer_mapping: dict[int, int],
use_mla: bool,
@@ -192,7 +190,6 @@ def build_kv_only_group(
kv_host_pool = build_kv_host_pool(
kv_pool=kv_pool,
page_size=page_size,
server_args=server_args,
use_mla=use_mla,
override_kv_cache_dim=override_kv_cache_dim,
host_size=host_size,
@@ -223,7 +220,6 @@ def build_kv_only_group(
def build_hybrid_swa_group(
*,
page_size: int,
server_args: ServerArgs,
full_kv_pool: Any,
swa_kv_pool: Any,
full_layer_mapping: dict[int, int],
@@ -241,7 +237,6 @@ def build_hybrid_swa_group(
kv_host_pool = build_kv_host_pool(
kv_pool=full_kv_pool,
page_size=page_size,
server_args=server_args,
use_mla=use_mla,
host_size=kv_host_size,
pool_label="full",
@@ -249,7 +244,6 @@ def build_hybrid_swa_group(
swa_host_pool = build_kv_host_pool(
kv_pool=swa_kv_pool,
page_size=page_size,
server_args=server_args,
use_mla=use_mla,
host_size=swa_host_size,
mtp_draft_device_pools=mtp_swa_device_pools,
@@ -310,7 +304,6 @@ def build_kv_only_stack(
transfer_layer_num = len(full_layer_mapping)
host_pool_group = build_kv_only_group(
page_size=params.page_size,
server_args=server_args,
kv_pool=kv_pool,
full_layer_mapping=full_layer_mapping,
use_mla=use_mla,
@@ -371,7 +364,6 @@ def build_hybrid_swa_stack(
host_pool_group = build_hybrid_swa_group(
page_size=params.page_size,
server_args=server_args,
full_kv_pool=full_kv_pool,
swa_kv_pool=swa_kv_pool,
full_layer_mapping=full_layer_mapping,
@@ -723,7 +715,6 @@ def build_hybrid_mamba_stack(
kv_host_pool = build_kv_host_pool(
kv_pool=kv_pool,
page_size=params.page_size,
server_args=server_args,
use_mla=use_mla,
host_size=kv_host_size,
mtp_draft_device_pools=mtp_draft_device_pools,
@@ -826,7 +817,6 @@ def build_hybrid_mamba_swa_stack(
kv_host_pool = build_kv_host_pool(
kv_pool=full_kv_pool,
page_size=page_size,
server_args=server_args,
use_mla=False,
host_size=kv_host_size,
pool_label="full",
@@ -834,7 +824,6 @@ def build_hybrid_mamba_swa_stack(
swa_host_pool = build_kv_host_pool(
kv_pool=swa_kv_pool,
page_size=page_size,
server_args=server_args,
use_mla=False,
host_size=swa_host_size,
pool_label="swa",
@@ -925,7 +914,6 @@ def build_anchor_sidecar_stack(
kv_host_pool = build_kv_host_pool(
kv_pool=kv_pool,
page_size=params.page_size,
server_args=server_args,
use_mla=use_mla,
override_kv_cache_dim=override_kv_cache_dim,
mtp_draft_device_pools=mtp_draft_device_pools,
@@ -1013,7 +1001,6 @@ def build_full_draft_pools(
*,
draft_kv_pool: Any,
tree_cache: Any,
server_args: ServerArgs,
) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]:
"""Build draft KV/DSA sidecars whose indices follow target full KV."""
from sglang.srt.mem_cache.memory_pool import (
@@ -1089,7 +1076,6 @@ def build_swa_draft_pools(
*,
draft_kv_pool: Any,
tree_cache: Any,
server_args: ServerArgs,
) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]:
"""Build a draft SWA sidecar whose indices follow target SWA."""
draft_swa_pool = draft_kv_pool.swa_kv_pool
@@ -1143,7 +1129,6 @@ def build_hicache_draft_sidecars(
*,
draft_device_pools: tuple[Any, ...],
tree_cache: Any,
server_args: ServerArgs,
) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]:
"""Compose the full and SWA draft-sidecar paths."""
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
@@ -1158,7 +1143,6 @@ def build_hicache_draft_sidecars(
return builder(
draft_kv_pool=draft_kv_pool,
tree_cache=tree_cache,
server_args=server_args,
)
@@ -1561,7 +1545,6 @@ class _MiniMaxSparseStrategy(StackStrategy):
):
host_pool_group, cache_controller = build_minimax_sparse_hicache_stack(
params=params,
server_args=server_args,
sparse_pool=kvcache,
load_cache_event=load_cache_event,
storage_backend=storage_backend,
@@ -1754,7 +1737,6 @@ def attach_hybrid_pool_to_unified_cache(
def build_minimax_sparse_hicache_stack(
*,
params: CacheInitParams,
server_args: ServerArgs,
sparse_pool: Any,
load_cache_event,
storage_backend: Optional[str],
@@ -1787,7 +1769,6 @@ def build_minimax_sparse_hicache_stack(
kv_host_pool = build_kv_host_pool(
kv_pool=main_pool,
page_size=params.page_size,
server_args=server_args,
use_mla=False,
)
entries = [
@@ -1892,7 +1873,6 @@ def attach_hybrid_minimax_sparse_pool_to_hiradix_cache(
else:
host_pool_group, cache_controller = build_minimax_sparse_hicache_stack(
params=params,
server_args=server_args,
sparse_pool=sparse_pool,
load_cache_event=load_cache_event,
storage_backend=get_memory().hicache_storage_backend,
python/sglang/srt/mem_cache/kv_cache_builder.py
-3
View File
@@ -66,7 +66,6 @@ def maybe_register_hicache_draft(
*,
tree_cache,
draft_plan: HiCacheDraftPlan,
server_args: ServerArgs,
) -> None:
from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode
@@ -85,7 +84,6 @@ def maybe_register_hicache_draft(
specs, entries = build_hicache_draft_sidecars(
draft_device_pools=draft_plan.device_pools,
tree_cache=tree_cache,
server_args=server_args,
)
for spec, entry in zip(specs, entries, strict=True):
tree_cache.register_sidecar_pool(spec, entry)
@@ -322,7 +320,6 @@ def build_kv_cache(
maybe_register_hicache_draft(
tree_cache=tree_cache,
draft_plan=hicache_draft_plan,
server_args=server_args,
)
if retraction_backup == "host_pool":
python/sglang/srt/model_executor/model_runner.py
-2
View File
@@ -652,7 +652,6 @@ class ModelRunner:
prepare_moe_topk(
model=self.model,
model_config=self.model_config,
server_args=self.server_args,
moe_ep_size=self.ps.moe_ep_size,
moe_ep_rank=self.ps.moe_ep_rank,
)
@@ -1187,7 +1186,6 @@ class ModelRunner:
# before configure_kv_cache_dtype.)
load_kv_cache_scales(
model=self.model,
server_args=self.server_args,
kv_cache_dtype=get_model().kv_cache_dtype,
)
python/sglang/srt/model_executor/model_runner_components/load_model_utils.py
+1 -3
View File
@@ -105,9 +105,7 @@ def maybe_trigger_remote_instance_nccl_send_group(
t.start()
def load_kv_cache_scales(
*, model, server_args: ServerArgs, kv_cache_dtype: str
) -> None:
def load_kv_cache_scales(*, model, kv_cache_dtype: str) -> None:
"""``kv_cache_dtype`` is the caller's resolved value. Required rather than
defaulted: a fallback to ``server_args`` would be a hidden global read for
any future caller that forgets to pass one."""
python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py
-2
View File
@@ -18,7 +18,6 @@ from sglang.srt.utils import get_bool_env_var, is_hip, log_info_on_rank0
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.server_args import ServerArgs
logger = logging.getLogger(__name__)
@@ -29,7 +28,6 @@ def prepare_moe_topk(
*,
model,
model_config: ModelConfig,
server_args: ServerArgs,
moe_ep_size: int,
moe_ep_rank: int,
) -> None:
python/sglang/srt/server_args.py
+16 -21
View File
@@ -1279,6 +1279,8 @@ class ServerArgs:
"defaults to --port + 10000.",
NS("serving"),
] = None
# Env-only (SGLANG_GRPC_WORKER_THREADS); a field so the projection sees it.
grpc_worker_threads: A[Optional[int], Arg(no_cli=True), NS("serving")] = None
sidecar: A[
Optional[str],
"Start a locally managed sidecar against the native gRPC server. "
@@ -3696,7 +3698,7 @@ class ServerArgs:
except BaseException:
# The handlers that ran already declared, and they are not
# idempotent over their own output.
object.__setattr__(self, "_resolution_failed", True)
self._resolution_failed = True
raise
# Set here too, because the dummy/absent-model path returns before the
# end of the pipeline that normally sets it: the gate is about whether
@@ -3747,8 +3749,8 @@ class ServerArgs:
# Everything outside the fields, enumerated from the instance: the raw
# snapshot, the stash, and what resolution memoized -- including the
# `get_model_config()` cache, which a resolved copy can no longer fill
# (the read-only guard refuses the write).
# `get_model_config()` memo, which the copy carries over rather than
# rebuild.
field_names = {field.name for field in dataclasses.fields(self)}
for name, value in vars(self).items():
if name in field_names or name == "_resolution_finished":
@@ -4472,7 +4474,10 @@ class ServerArgs:
# Native gRPC tuning knob is env-only; --grpc-port (CLI) enables the
# native server, falling back to SGLANG_GRPC_PORT.
self.grpc_worker_threads = envs.SGLANG_GRPC_WORKER_THREADS.get()
self._declare(
"_handle_deprecated_args",
grpc_worker_threads=envs.SGLANG_GRPC_WORKER_THREADS.get(),
)
grpc_port_env = envs.SGLANG_GRPC_PORT.get()
if cfg.grpc_port is None and grpc_port_env is not None:
@@ -4496,10 +4501,10 @@ class ServerArgs:
"--grpc-port / SGLANG_GRPC_PORT "
f"({cfg.grpc_port}) must be between 1 and 65535"
)
if self.grpc_worker_threads < 1:
if cfg.grpc_worker_threads is not None and cfg.grpc_worker_threads < 1:
raise ValueError(
"SGLANG_GRPC_WORKER_THREADS "
f"({self.grpc_worker_threads}) must be >= 1"
f"({cfg.grpc_worker_threads}) must be >= 1"
)
# Native gRPC is incompatible with launch paths it doesn't wire into.
@@ -7271,7 +7276,6 @@ class ServerArgs:
"_handle_dwdp",
ep_size=cfg.dwdp_size,
)
self.moe_ep_size = cfg.dwdp_size
self._declare(
"_handle_dwdp",
moe_dp_size=1,
@@ -7290,7 +7294,7 @@ class ServerArgs:
logger.info(
f"DWDP enabled: dwdp_size={cfg.dwdp_size}, "
f"auto-forced dp_size={cfg.dp_size}, moe_ep_size={self.moe_ep_size}, "
f"auto-forced dp_size={cfg.dp_size}, ep_size={cfg.dwdp_size}, "
f"moe_dense_tp_size=1, moe_a2a_backend=none, "
f"dp_attention_local_control_broadcast=True, "
f"enable_dp_lm_head=True, SCHEDULER_SKIP_ALL_GATHER=True, "
@@ -10098,7 +10102,7 @@ class ServerArgs:
cfg = resolving_view(self)
from sglang.srt.configs.model_config import ModelConfig
memo = getattr(self, "model_config", None)
memo = getattr(self, "_model_config", None)
if memo is not None:
# The key is the path this record carried when the cache was
# filled. The GGUF and ModelScope handlers declare a different
@@ -10112,7 +10116,7 @@ class ServerArgs:
return memo
model_config = ModelConfig.from_server_args(self)
self.model_config = model_config
self._model_config = model_config
self._model_config_built_from = cfg.model_path
if model_config.is_hybrid_swa:
logger.info(
@@ -10147,7 +10151,6 @@ class ServerArgs:
if (
getattr(self, "_resolution_finished", False)
and not getattr(self, "_internal_write", False)
and name not in _CACHE_SLOTS
and (not name.startswith("_") or name in _underscore_field_names())
):
raise AttributeError(
@@ -10227,7 +10230,7 @@ class ServerArgs:
# is supported.
result = max(candidate_steps) + 1
if getattr(self, "_resolution_finished", False):
object.__setattr__(self, "_max_speculative_num_draft_tokens", result)
self._max_speculative_num_draft_tokens = result
return result
@property
@@ -10851,7 +10854,7 @@ class ServerArgs:
result = json.loads(self.modelexpress_config)
else:
result = self.modelexpress_config
object.__setattr__(self, "_mx_config_cache", result)
self._mx_config_cache = result
return result
@property
@@ -11058,14 +11061,6 @@ def m3_fp8_attn_gemm_enabled(args) -> bool:
)
# Caches, which the read-only guard lets through: a value the record derived
# from itself is not resolved configuration, and a key that can invalidate on a
# resolved record needs the refill to be storable there. Only the public-named
# ones are listed -- a cache key spelled with a leading underscore is already
# exempt.
_CACHE_SLOTS = frozenset({"model_config"})
# NOTE: The process-wide ServerArgs is owned by the runtime context
# (sglang.srt.runtime_context). The two functions below are LEGACY shims kept
# for the existing call-sites; they publish/read the same live object by