diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 3f46f5ac9..88eb3689a 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -261,17 +261,19 @@ def mamba_extra_buffer_of(cfg: Any) -> bool: def declare_load_time_override(source: str, declared: Dict[str, Any]) -> None: """Declare a load-time resolved field (model-file config overrides, - weight-resolved dtypes) after publish. it is written to the config - bags via ``get_context().override`` (namespace readers see it); server_args - stays the pristine startup record. Validated against the resolvable - whitelist first.""" + weight-resolved dtypes) on the published ``server_args``: resolution has + already materialized, so the declaration writes through, joining the + declaration stash for provenance and republish consistency.""" from sglang.srt.runtime_context import get_context - context = get_context() - validate_declarations(context.server_args, [(source, dict(declared))]) - # write the config bags (namespace readers see it); server_args - # stays the pristine startup record. - context.override(source, **declared) + server_args = get_context().server_args + validate_declarations(server_args, [(source, dict(declared))]) + override = getattr(server_args, "override", None) + if override is not None: + override(source, **declared) + else: + # Config-shaped fixtures without the mutation entry point. + _apply_fields(server_args, declared) def collect_model_override_declarations( diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 3829c4beb..ca44b0021 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -39,11 +39,7 @@ from sglang.srt.model_executor.forward_batch_info import ( compute_position, ) from sglang.srt.model_executor.forward_context import get_attn_backend -from sglang.srt.runtime_context import ( - get_device, - get_exec, - get_parallel, -) +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.speculative.spec_info import SpecInput from sglang.srt.utils import BumpAllocator, empty_context, get_bool_env_var, is_hip @@ -187,7 +183,7 @@ def _update_device_and_sum_field_from_cpu_field( cpu_value if isinstance(cpu_value, torch.Tensor) else torch.tensor(cpu_value, dtype=old_device_value.dtype) - ).to(device=get_device().device, non_blocking=True) + ).to(device=get_server_args().device, non_blocking=True) setattr(batch, device_field, new_device_value) if sum_field is not None: @@ -339,7 +335,7 @@ def compute_split_indices_for_cuda_graph_replay( class TboCudaGraphRunnerPlugin: def __init__(self): self._tbo_children_num_token_non_padded = torch.zeros( - (2,), dtype=torch.int32, device=get_device().device + (2,), dtype=torch.int32, device=get_server_args().device ) def capture_one_batch_size(self, batch: ForwardBatch, num_tokens: int): @@ -637,7 +633,7 @@ class TboForwardBatchPreparer: sum_field=None, ) _, child_b.extend_start_loc = compute_position( - get_exec().kernel.attention_backend, + get_server_args().attention_backend, child_b.extend_prefix_lens, child_b.extend_seq_lens, child_b.extend_num_tokens, @@ -761,7 +757,7 @@ class TboForwardBatchPreparer: # TODO improve, e.g. unify w/ `init_raw` if ( - get_parallel().moe_dense_tp_size == 1 + get_server_args().moe_dense_tp_size == 1 and batch.global_dp_buffer_len is not None ): sum_len = end_token_index - start_token_index @@ -836,7 +832,7 @@ class TboForwardBatchPreparer: value_a = min(tbo_split_token_index, num_token_non_padded) value_b = max(0, num_token_non_padded - tbo_split_token_index) return torch.tensor([value_a, value_b], dtype=torch.int32).to( - device=get_device().device, non_blocking=True + device=get_server_args().device, non_blocking=True ) @classmethod diff --git a/python/sglang/srt/configs/inkling.py b/python/sglang/srt/configs/inkling.py index c963ca9e9..b4f24592e 100644 --- a/python/sglang/srt/configs/inkling.py +++ b/python/sglang/srt/configs/inkling.py @@ -8,7 +8,6 @@ from transformers import CONFIG_MAPPING from transformers.configuration_utils import PretrainedConfig from sglang.srt.configs.mamba_utils import BaseLinearStateParams -from sglang.srt.runtime_context import get_exec class InklingModelConfig(PretrainedConfig): @@ -225,8 +224,9 @@ class InklingModelConfig(PretrainedConfig): self.swa_num_key_value_heads, self.swa_head_dim ) stream_dim = self.hidden_size + from sglang.srt.runtime_context import get_server_args - if get_exec().comm.enable_scattered_sconv: + if get_server_args().enable_scattered_sconv: # Scattered sconv: the attn/mlp output sconvs run on the [T, H/P] # hidden shard, so their conv-state caches shard with them. assert ( diff --git a/python/sglang/srt/constrained/grammar_manager.py b/python/sglang/srt/constrained/grammar_manager.py index 010185c25..b039020fd 100644 --- a/python/sglang/srt/constrained/grammar_manager.py +++ b/python/sglang/srt/constrained/grammar_manager.py @@ -14,7 +14,6 @@ from sglang.srt.constrained.base_grammar_backend import ( from sglang.srt.constrained.reasoner_grammar_backend import ReasonerGrammarObject from sglang.srt.distributed.communication_tags import P2PTag from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_serving if TYPE_CHECKING: from sglang.srt.managers.io_struct import AbortReq @@ -29,7 +28,7 @@ class GrammarManager: self.scheduler = scheduler self.server_args = scheduler.server_args self.grammar_queue: List[Req] = [] - if not get_serving().skip_tokenizer_init: + if not self.server_args.skip_tokenizer_init: self.grammar_backend = create_grammar_backend( self.server_args, scheduler.tokenizer, diff --git a/python/sglang/srt/disaggregation/common/conn.py b/python/sglang/srt/disaggregation/common/conn.py index 165db4315..064165f46 100644 --- a/python/sglang/srt/disaggregation/common/conn.py +++ b/python/sglang/srt/disaggregation/common/conn.py @@ -32,8 +32,11 @@ from sglang.srt.disaggregation.utils import ( ) from sglang.srt.distributed import get_pp_group, get_world_group from sglang.srt.environ import envs -from sglang.srt.layers.dp_attention import get_attention_dp_rank, get_attention_dp_size -from sglang.srt.runtime_context import get_model, get_parallel, get_serving +from sglang.srt.layers.dp_attention import ( + get_attention_dp_rank, + get_attention_dp_size, +) +from sglang.srt.runtime_context import get_model, get_parallel from sglang.srt.server_args import ServerArgs from sglang.srt.utils.network import ( NetworkAddress, @@ -572,7 +575,7 @@ class CommonKVManager(BaseKVManager): `Connection refused`, and the leader's `prefill_port_table` ends up missing rows. """ - if not self.dist_init_addr or get_parallel().nnodes == 1: + if not self.dist_init_addr or self.server_args.nnodes == 1: return local_port if not (dist.is_available() and dist.is_initialized()): @@ -624,14 +627,14 @@ class CommonKVManager(BaseKVManager): "rank_port": self.rank_port, "page_size": self.kv_args.page_size, "kv_cache_dtype": get_model().kv_cache_dtype, - "load_balance_method": get_parallel().load_balance_method, + "load_balance_method": self.server_args.load_balance_method, "enable_dsa_cache_layer_split": getattr( self.server_args, "enable_dsa_cache_layer_split", False ), # Self-register the HTTP API port so the decode can derive the PD # retract rebootstrap /generate URL from bootstrap info instead of a # router-injected pd_rebootstrap_prefill_url. - "prefill_http_port": get_serving().port, + "prefill_http_port": self.server_args.port, } max_retries, initial_delay, max_delay = 5, 1.0, 30.0 diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 25c5d0db8..5224920c4 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -86,7 +86,7 @@ from sglang.srt.observability.req_time_stats import ( set_schedule_time_batch, set_time_batch, ) -from sglang.srt.runtime_context import get_disagg, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import get_num_new_pages from sglang.srt.utils.network import NetworkAddress from sglang.srt.utils.nvtx_utils import scheduler_nvtx_method @@ -2151,7 +2151,7 @@ class SchedulerDisaggregationDecodeMixin: # Decode-radix path: new requests already matched in # `pop_preallocated`. Retracted requests reset `last_node`, # so re-match only when that state is missing. - if get_disagg().disaggregation_decode_enable_radix_cache: + if self.server_args.disaggregation_decode_enable_radix_cache: tree_cache = self.tree_cache if req.last_node is None else None else: tree_cache = self.tree_cache @@ -2191,7 +2191,7 @@ class SchedulerDisaggregationDecodeMixin: if self.enable_decode_hicache: self.tree_cache.check_hicache_events() - if get_disagg().disaggregation_decode_enable_offload_kvcache: + if self.server_args.disaggregation_decode_enable_offload_kvcache: self.decode_offload_manager.check_offload_progress() # try to resume retracted requests if there are enough space for another `num_reserved_decode_tokens` decode steps @@ -2203,7 +2203,9 @@ class SchedulerDisaggregationDecodeMixin: if not hasattr(self, "polling_count"): self.polling_count = 0 - self.polling_interval = get_disagg().disaggregation_decode_polling_interval + self.polling_interval = ( + self.server_args.disaggregation_decode_polling_interval + ) self.polling_count = (self.polling_count + 1) % self.polling_interval diff --git a/python/sglang/srt/disaggregation/encode_grpc_server.py b/python/sglang/srt/disaggregation/encode_grpc_server.py index 3516f56d4..5abb27473 100644 --- a/python/sglang/srt/disaggregation/encode_grpc_server.py +++ b/python/sglang/srt/disaggregation/encode_grpc_server.py @@ -28,7 +28,6 @@ from sglang.srt.disaggregation.encode_server import ( ) from sglang.srt.managers.io_struct import async_sock_send, wrap_as_pickle from sglang.srt.managers.schedule_batch import Modality -from sglang.srt.runtime_context import get_disagg from sglang.srt.server_args import PortArgs, ServerArgs from sglang.srt.utils import random_uuid from sglang.srt.utils.network import NetworkAddress, get_zmq_socket @@ -118,13 +117,13 @@ class SGLangEncoderServer(SGLangEncoderServicer): context.set_details(error_msg) return sglang_encoder_pb2.EncodeResponse() - if get_disagg().encoder_transfer_backend == "mooncake": + if self.server_args.encoder_transfer_backend == "mooncake": return sglang_encoder_pb2.EncodeResponse( embedding_size=nbytes, embedding_len=embedding_len, embedding_dim=embedding_dim, ) - elif get_disagg().encoder_transfer_backend == "zmq_to_scheduler": + elif self.server_args.encoder_transfer_backend == "zmq_to_scheduler": embedding_ports = list(request.embedding_port) logger.info(f"embedding_port = {embedding_ports}") if not embedding_ports: @@ -142,7 +141,7 @@ class SGLangEncoderServer(SGLangEncoderServicer): await asyncio.gather(*tasks) self.encoder.embedding_to_send.pop(request.req_id, None) return sglang_encoder_pb2.EncodeResponse() - elif get_disagg().encoder_transfer_backend == "zmq_to_tokenizer": + elif self.server_args.encoder_transfer_backend == "zmq_to_tokenizer": embedding_port = ( request.embedding_port[0] if request.embedding_port else 0 ) diff --git a/python/sglang/srt/disaggregation/encode_server.py b/python/sglang/srt/disaggregation/encode_server.py index 1b0a8af8c..626b75a1b 100644 --- a/python/sglang/srt/disaggregation/encode_server.py +++ b/python/sglang/srt/disaggregation/encode_server.py @@ -59,9 +59,15 @@ from sglang.srt.model_loader import get_model from sglang.srt.multimodal.processors.qwen_vl import preprocess_video from sglang.srt.observability.metrics_collector import EncoderMetricsCollector from sglang.srt.observability.req_time_stats import EncoderReqTimeStats -from sglang.srt.observability.trace import process_tracing_init, trace_set_thread_info -from sglang.srt.runtime_context import get_disagg, get_exec, get_mm -from sglang.srt.server_args import PortArgs, ServerArgs +from sglang.srt.observability.trace import ( + process_tracing_init, + trace_set_thread_info, +) +from sglang.srt.server_args import ( + PortArgs, + ServerArgs, + set_global_server_args_for_scheduler, +) from sglang.srt.utils import ( add_prometheus_middleware, configure_logger, @@ -256,9 +262,7 @@ class MMEncoder: ): logger.info(f"init MMEncoder {rank}/{server_args.tp_size}") self.server_args = server_args - from sglang.srt.runtime_context import publish - - publish(server_args, role="encoder") + set_global_server_args_for_scheduler(server_args) self.rank = rank # DP rank for metric labels; overridden by run_dp_worker in DP mode. # 0 in the single-instance (non-DP) path. @@ -345,7 +349,7 @@ class MMEncoder: [], dtype=self._embedding_dtype ).element_size() - if get_mm().enable_mm_global_cache: + if self.server_args.enable_mm_global_cache: from sglang.srt.mem_cache.storage.mooncake_store.embedding_cache_controller import ( EmbeddingCacheController, ) @@ -363,15 +367,15 @@ class MMEncoder: self.mm_global_cache = None # Pre-compute embedding metadata (needed by all ranks for mooncake) - if get_disagg().encoder_transfer_backend == "mooncake": + if self.server_args.encoder_transfer_backend == "mooncake": self._embedding_dims = self._infer_embedding_dims() if self.rank == 0: logger.info( - f"Using transfer backend: {get_disagg().encoder_transfer_backend}" + f"Using transfer backend: {self.server_args.encoder_transfer_backend}" ) - if get_disagg().encoder_transfer_backend == "mooncake": + if self.server_args.encoder_transfer_backend == "mooncake": self.local_ip = get_local_ip_auto() self.engine = get_mooncake_transfer_engine() @@ -384,8 +388,8 @@ class MMEncoder: hostname=self.local_ip, gpu_id=self.gpu_id, ib_device=( - get_disagg().disaggregation_ib_device - or get_exec().moe.mooncake_ib_device + self.server_args.disaggregation_ib_device + or self.server_args.mooncake_ib_device ), ) @@ -394,7 +398,7 @@ class MMEncoder: self.encode_dispatch_lock = asyncio.Lock() # Async mooncake state: track background VIT forward completion - if get_disagg().encoder_transfer_backend == "mooncake": + if self.server_args.encoder_transfer_backend == "mooncake": self._forward_ready_events: Dict[str, asyncio.Event] = {} self._forward_results: Dict[str, dict] = {} # when multiple decoder TP ranks call @@ -408,12 +412,12 @@ class MMEncoder: # Bind unified encode entry point based on backend and cache config if self.mm_global_cache is not None: - if get_disagg().encoder_transfer_backend == "mooncake": + if self.server_args.encoder_transfer_backend == "mooncake": self._encode_fn = self.encode_with_global_cache_mooncake else: self._encode_fn = self.encode_with_global_cache else: - if get_disagg().encoder_transfer_backend == "mooncake": + if self.server_args.encoder_transfer_backend == "mooncake": self._encode_fn = self.encode_with_mooncake else: self._encode_fn = self.encode @@ -1683,7 +1687,7 @@ class MMEncoder: mm_item.set(k, _convert(v)) cache_hit = False - use_mm_cache = get_mm().enable_prefix_mm_cache and log_metrics + use_mm_cache = self.server_args.enable_prefix_mm_cache and log_metrics if use_mm_cache: mm_item.set_pad_value() mm_hash = MultiModalStaticCache.combine_hashes([mm_item.hash]) @@ -1779,7 +1783,7 @@ class MMEncoder: embedding_port=None, url=None, ): - if get_disagg().encoder_transfer_backend == "mooncake": + if self.server_args.encoder_transfer_backend == "mooncake": # Wait for async VIT forward completion if needed req_id = mm_data.req_id if req_id in self._forward_ready_events: @@ -1850,7 +1854,7 @@ class MMEncoder: logger.info(f"{endpoint = }") # Serialize data - if get_disagg().encoder_transfer_backend == "mooncake": + if self.server_args.encoder_transfer_backend == "mooncake": # Mooncake already pushed the embedding via RDMA; new_mm_data = mm_data.copy_without_embedding() serialized_data = pickle.dumps(new_mm_data) @@ -1882,11 +1886,11 @@ class MMEncoder: await asyncio.get_event_loop().run_in_executor(self.executor, send_with_socket) if ( encoder_metrics_collector is not None - and get_disagg().encoder_transfer_backend != "mooncake" + and self.server_args.encoder_transfer_backend != "mooncake" ): encoder_metrics_collector.observe_transfer( time.perf_counter() - _zmq_xfer_start, - backend=get_disagg().encoder_transfer_backend, + backend=self.server_args.encoder_transfer_backend, ) async def encode( diff --git a/python/sglang/srt/disaggregation/mooncake/conn.py b/python/sglang/srt/disaggregation/mooncake/conn.py index 8226bf887..7d306a4d0 100644 --- a/python/sglang/srt/disaggregation/mooncake/conn.py +++ b/python/sglang/srt/disaggregation/mooncake/conn.py @@ -55,7 +55,6 @@ from sglang.srt.observability.trace import ( TraceReqContext, trace_set_thread_info, ) -from sglang.srt.runtime_context import get_parallel, get_schedule from sglang.srt.server_args import ServerArgs from sglang.srt.utils.network import NetworkAddress @@ -315,7 +314,9 @@ class MooncakeKVManager(CommonKVManager): self.kv_buffer_tensors = None def _handle_staging_req(self, msg): - from sglang.srt.disaggregation.common.staging_handler import handle_staging_req + from sglang.srt.disaggregation.common.staging_handler import ( + handle_staging_req, + ) room = int(msg[1].decode("ascii")) session_id = msg[4].decode("ascii") @@ -349,7 +350,9 @@ class MooncakeKVManager(CommonKVManager): def _is_watermark_ready( self, session_id: str, alloc_round: int, alloc_end: int ) -> bool: - from sglang.srt.disaggregation.common.staging_handler import is_watermark_ready + from sglang.srt.disaggregation.common.staging_handler import ( + is_watermark_ready, + ) return is_watermark_ready(self._staging_ctx, session_id, alloc_round, alloc_end) @@ -466,7 +469,7 @@ class MooncakeKVManager(CommonKVManager): room, self.transfer_infos, self.kv_buffer_tensors, - get_schedule().chunked_prefill_size, + self.server_args.chunked_prefill_size, self._staging_ctx.prefetch_requested, self._staging_ctx.prefetch_sockets, ) @@ -948,7 +951,7 @@ class MooncakeKVManager(CommonKVManager): if ( self.attn_cp_size > 1 and self.attn_cp_rank != 0 - and not get_parallel().enable_dsa_cache_layer_split + and not self.server_args.enable_dsa_cache_layer_split ): skip_state = True diff --git a/python/sglang/srt/disaggregation/nixl/conn.py b/python/sglang/srt/disaggregation/nixl/conn.py index c1f8b31ce..f60a8766a 100644 --- a/python/sglang/srt/disaggregation/nixl/conn.py +++ b/python/sglang/srt/disaggregation/nixl/conn.py @@ -13,8 +13,6 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple import numpy as np import numpy.typing as npt -from sglang.srt.runtime_context import get_schedule - if TYPE_CHECKING: from sglang.srt.disaggregation.common.staging_handler import StagingTransferInfo @@ -537,7 +535,9 @@ class NixlKVManager(CommonKVManager): def _is_watermark_ready( self, agent_name: str, alloc_round: int, alloc_end: int ) -> bool: - from sglang.srt.disaggregation.common.staging_handler import is_watermark_ready + from sglang.srt.disaggregation.common.staging_handler import ( + is_watermark_ready, + ) return is_watermark_ready(self._staging_ctx, agent_name, alloc_round, alloc_end) @@ -558,7 +558,9 @@ class NixlKVManager(CommonKVManager): threading.Thread(target=decode_staging_thread, daemon=True).start() def _handle_staging_req(self, msg): - from sglang.srt.disaggregation.common.staging_handler import handle_staging_req + from sglang.srt.disaggregation.common.staging_handler import ( + handle_staging_req, + ) room = int(msg[1].decode("ascii")) session_id = msg[4].decode("ascii") @@ -623,7 +625,7 @@ class NixlKVManager(CommonKVManager): room, self.transfer_infos, self.kv_buffer_tensors, - get_schedule().chunked_prefill_size, + self.server_args.chunked_prefill_size, self._staging_ctx.prefetch_requested, self._staging_ctx.prefetch_sockets, ) @@ -1737,7 +1739,9 @@ class NixlKVManager(CommonKVManager): req, page_start, num_pages, session_id=req.agent_name ) if not ready: - from sglang.srt.disaggregation.common.staging_buffer import StagingAllocator + from sglang.srt.disaggregation.common.staging_buffer import ( + StagingAllocator, + ) if c_offset == StagingAllocator.ALLOC_OVERSIZED: raise RuntimeError( diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 841e23a7f..1f4c947fa 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -64,7 +64,6 @@ from sglang.srt.mem_cache.common import ( ) from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.observability.req_time_stats import set_schedule_time_batch -from sglang.srt.runtime_context import get_disagg from sglang.srt.utils.nvtx_utils import scheduler_nvtx_method if TYPE_CHECKING: @@ -1182,7 +1181,7 @@ class SchedulerDisaggregationPrefillMixin: def optimistic_release_and_requeue(self: Scheduler, req: Req) -> None: """Release KV cache and requeue an optimistic prefill request.""" - max_attempts = get_disagg().optimistic_prefill_attempts + max_attempts = self.server_args.optimistic_prefill_attempts maybe_cache_unfinished_req(req, self.tree_cache) release_kv_cache(req, self.tree_cache) req.reset_for_retract() diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py index 95d9411e4..261e8d6cd 100644 --- a/python/sglang/srt/distributed/device_communicators/pymscclpp.py +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -14,7 +14,7 @@ from sglang.srt.compilation.compile_phase import ( from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( is_in_tc_piecewise_cuda_graph, ) -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args logger = logging.getLogger(__name__) @@ -25,7 +25,7 @@ class PyMscclppCommunicator: def _is_symm_mem_enabled(self) -> bool: try: - return get_exec().comm.enable_symm_mem + return get_server_args().enable_symm_mem except ValueError: return False diff --git a/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py b/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py index 5ce034f5b..3e833824e 100644 --- a/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py +++ b/python/sglang/srt/distributed/device_communicators/pynccl_allocator.py @@ -15,7 +15,7 @@ from torch.cuda.memory import ( from sglang.srt.distributed.parallel_state import GroupCoordinator from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils.common import torch_release after_2_8_0 = torch_release >= (2, 8) @@ -159,7 +159,7 @@ _register_func = None def is_symmetric_memory_enabled(): try: - return get_exec().comm.enable_symm_mem + return get_server_args().enable_symm_mem except ValueError: return False diff --git a/python/sglang/srt/distributed/device_communicators/torch_symm_mem.py b/python/sglang/srt/distributed/device_communicators/torch_symm_mem.py index 9c81599f2..3ba756c7d 100644 --- a/python/sglang/srt/distributed/device_communicators/torch_symm_mem.py +++ b/python/sglang/srt/distributed/device_communicators/torch_symm_mem.py @@ -12,7 +12,6 @@ from sglang.srt.distributed.device_communicators.all_reduce_utils import ( TORCH_SYMM_MEM_ALL_REDUCE_MAX_SIZES, ) from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_exec from sglang.srt.utils import is_cuda, is_hip try: @@ -99,9 +98,10 @@ class TorchSymmMemCommunicator: # ([16384, 6144] bf16 = 192 MiB), including room for tail regions. if envs.SGLANG_OPT_USE_INKLING_CUSTOM_AR.get(): self.max_size = max(self.max_size, 256 * 1024 * 1024) + from sglang.srt.runtime_context import get_server_args if ( - get_exec().comm.enable_scattered_sconv + get_server_args().enable_scattered_sconv or envs.SGLANG_OPT_USE_INKLING_FUSED_AR_SCONV.get() ): # Fused extend kernels are out-of-place, so OUT must hold the diff --git a/python/sglang/srt/distributed/device_communicators/triton_symm_mem_ag.py b/python/sglang/srt/distributed/device_communicators/triton_symm_mem_ag.py index 88ae03ba7..758e9c965 100644 --- a/python/sglang/srt/distributed/device_communicators/triton_symm_mem_ag.py +++ b/python/sglang/srt/distributed/device_communicators/triton_symm_mem_ag.py @@ -16,8 +16,6 @@ import torch.distributed._symmetric_memory as symm_mem import triton import triton.language as tl -from sglang.srt.runtime_context import get_parallel - logger = logging.getLogger(__name__) # Each thread moves _NUMEL_PER_THREAD bf16 via one 128-bit multimem op; the @@ -468,6 +466,7 @@ class MultimemAllGatherer: # Lazy import avoids a module-load dependency on the distributed facade. from sglang.srt.distributed import get_tp_group from sglang.srt.distributed.parallel_state import in_the_same_node_as + from sglang.srt.runtime_context import get_server_args tp_group = get_tp_group() # Only probe node topology when the deployment can actually span @@ -478,7 +477,7 @@ class MultimemAllGatherer: # EP/mooncake setups, and keep multimem enabled. if ( tp_group.world_size > 1 - and get_parallel().nnodes > 1 + and get_server_args().nnodes > 1 and not all(in_the_same_node_as(tp_group.cpu_group, source_rank=0)) ): logger.warning( diff --git a/python/sglang/srt/dllm/mixin/scheduler.py b/python/sglang/srt/dllm/mixin/scheduler.py index abc4b3ed5..6d532531e 100644 --- a/python/sglang/srt/dllm/mixin/scheduler.py +++ b/python/sglang/srt/dllm/mixin/scheduler.py @@ -11,7 +11,6 @@ from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder from sglang.srt.mem_cache.common import release_kv_cache from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.observability.req_time_stats import set_time_batch -from sglang.srt.runtime_context import get_exec, get_schedule logger = logging.getLogger(__name__) @@ -23,7 +22,7 @@ class SchedulerDllmMixin: def init_diffusion_llm(self: Scheduler): self.dllm_config = ( DllmConfig.from_server_args(self.server_args) - if get_exec().dllm.dllm_algorithm is not None + if self.server_args.dllm_algorithm is not None else None ) self.dllm_manager = DllmManager(dllm_config=self.dllm_config) @@ -201,7 +200,7 @@ class SchedulerDllmMixin: self.chunked_prefill_size, running_bs if self.is_mixed_chunk else 0, self.priority_scheduling_preemption_threshold, - prefill_max_requests=get_schedule().prefill_max_requests, + prefill_max_requests=self.server_args.prefill_max_requests, dllm_config=self.dllm_config, ) diff --git a/python/sglang/srt/elastic_ep/elastic_ep.py b/python/sglang/srt/elastic_ep/elastic_ep.py index 11886fdc6..8ceb53a9e 100644 --- a/python/sglang/srt/elastic_ep/elastic_ep.py +++ b/python/sglang/srt/elastic_ep/elastic_ep.py @@ -442,7 +442,7 @@ def get_healthy_expert_location_src_rank( *, invoked_in_elastic_ep_rejoin_path: bool ) -> int: world_group = get_world_group() - # NOTE: do not key off `get_exec().moe.elastic_ep_rejoin` here. + # NOTE: do not key off `self.server_args.elastic_ep_rejoin` here. # A rank that was started as a rejoin rank may later act as a healthy # rank in a subsequent recovery cycle. local_rejoin_flag = bool(invoked_in_elastic_ep_rejoin_path) diff --git a/python/sglang/srt/elastic_ep/expert_backup_client.py b/python/sglang/srt/elastic_ep/expert_backup_client.py index 7a20e3057..8b77f7f07 100644 --- a/python/sglang/srt/elastic_ep/expert_backup_client.py +++ b/python/sglang/srt/elastic_ep/expert_backup_client.py @@ -7,11 +7,13 @@ from typing import Any, Callable import torch import zmq -from sglang.srt.distributed.parallel_state import get_world_group, get_world_size +from sglang.srt.distributed.parallel_state import ( + get_world_group, + get_world_size, +) from sglang.srt.environ import envs from sglang.srt.eplb.expert_location import get_global_expert_location_metadata from sglang.srt.managers.io_struct import UpdateExpertBackupReq, sock_recv, sock_send -from sglang.srt.runtime_context import get_exec from sglang.srt.server_args import ServerArgs from sglang.srt.utils.network import get_local_ip_auto @@ -109,7 +111,7 @@ class ExpertBackupClient: global_expert_location_metadata = get_global_expert_location_metadata() num_experts = ( self.model_config.hf_config.n_routed_experts - + get_exec().moe.ep_num_redundant_experts + + self.server_args.ep_num_redundant_experts ) num_local_experts = num_experts // self.moe_ep_size for i in range(self.engine_num): diff --git a/python/sglang/srt/elastic_ep/expert_backup_manager.py b/python/sglang/srt/elastic_ep/expert_backup_manager.py index 15a6536e4..3ca4a7249 100644 --- a/python/sglang/srt/elastic_ep/expert_backup_manager.py +++ b/python/sglang/srt/elastic_ep/expert_backup_manager.py @@ -20,6 +20,7 @@ from sglang.srt.model_loader.utils import set_default_torch_dtype from sglang.srt.server_args import ( PortArgs, ServerArgs, + set_global_server_args_for_scheduler, ) from sglang.srt.utils.network import get_local_ip_auto @@ -158,9 +159,7 @@ def run_expert_backup_manager_process( server_args: ServerArgs, port_args: PortArgs, ): - from sglang.srt.runtime_context import publish - - publish(server_args, role="expert_backup") + set_global_server_args_for_scheduler(server_args) from sglang.srt.distributed.device_communicators.mooncake_transfer_engine import ( init_mooncake_transfer_engine, ) diff --git a/python/sglang/srt/entrypoints/engine.py b/python/sglang/srt/entrypoints/engine.py index 528e3c4ef..5c158535e 100644 --- a/python/sglang/srt/entrypoints/engine.py +++ b/python/sglang/srt/entrypoints/engine.py @@ -93,7 +93,6 @@ from sglang.srt.observability.trace import process_tracing_init, trace_set_threa from sglang.srt.parser.template_detection import resolve_auto_parsers from sglang.srt.parser.template_manager import TemplateManager from sglang.srt.plugins import load_plugins -from sglang.srt.runtime_context import get_parallel from sglang.srt.server_args import PortArgs, ServerArgs from sglang.srt.utils import ( MultiprocessingSerializer, @@ -254,7 +253,7 @@ class Engine(EngineScoreMixin, EngineBase): # Initialize ZMQ sockets context = zmq.Context(2) - if server_args.node_rank == 0: + if self.server_args.node_rank == 0: self.send_to_rpc = get_zmq_socket( context, zmq.DEALER, self.port_args.rpc_ipc_name, True ) @@ -302,7 +301,7 @@ class Engine(EngineScoreMixin, EngineBase): routed_dp_rank = data_parallel_rank if routed_dp_rank is not None: - dp_size = get_parallel().dp_size + dp_size = self.server_args.dp_size if dp_size <= 1 and routed_dp_rank == 0: logger.debug( f"routed_dp_rank={routed_dp_rank} is ignored because dp_size={dp_size}" @@ -878,14 +877,7 @@ class Engine(EngineScoreMixin, EngineBase): server_args, port_args ) else: - # Launch multi-tokenizer router. Unlike TokenizerManager, the router - # does not publish; but it runs in this parent process and reads - # resolved config through the namespace accessors (e.g. get_parallel() - # for routed_dp_rank), so publish here. The child TokenizerWorkers - # publish independently in their own processes. - from sglang.srt.runtime_context import publish - - publish(server_args, role="tokenizer") + # Launch multi-tokenizer router tokenizer_manager = MultiTokenizerRouter(server_args, port_args) template_manager = None @@ -1005,18 +997,12 @@ class Engine(EngineScoreMixin, EngineBase): ) def get_server_info(self): - from sglang.srt.runtime_context import get_context - internal_states = self.loop.run_until_complete( self.tokenizer_manager.get_internal_state() ) return msgspec_to_builtins( { - # Overlay post-publish overrides so the report reflects current - # config (weight version, model path, runtime tunables). - **get_context().resolved_server_args_dict( - base=dataclasses.asdict(self.tokenizer_manager.server_args) - ), + **dataclasses.asdict(self.tokenizer_manager.server_args), **self._scheduler_init_result.scheduler_infos[0], "internal_states": internal_states, "version": __version__, diff --git a/python/sglang/srt/entrypoints/grpc_bridge.py b/python/sglang/srt/entrypoints/grpc_bridge.py index eca22c3b8..fa0ab61b8 100644 --- a/python/sglang/srt/entrypoints/grpc_bridge.py +++ b/python/sglang/srt/entrypoints/grpc_bridge.py @@ -15,7 +15,6 @@ from typing import Any, Awaitable, Callable, Dict, List, Optional from pydantic import ValidationError -from sglang.srt.runtime_context import get_context, get_lora, get_serving from sglang.srt.utils.msgspec_utils import msgspec_to_builtins logger = logging.getLogger(__name__) @@ -230,7 +229,9 @@ class RuntimeHandle: return self._openai_serving_classes from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat - from sglang.srt.entrypoints.openai.serving_classify import OpenAIServingClassify + from sglang.srt.entrypoints.openai.serving_classify import ( + OpenAIServingClassify, + ) from sglang.srt.entrypoints.openai.serving_completions import ( OpenAIServingCompletion, ) @@ -375,20 +376,16 @@ class RuntimeHandle: model_config = self.tokenizer_manager.model_config result = { "model_path": self.tokenizer_manager.model_path, - "tokenizer_path": get_serving().tokenizer_path, + "tokenizer_path": self.server_args.tokenizer_path, "is_generation": self.tokenizer_manager.is_generation, - "weight_version": get_serving().weight_version, + "weight_version": self.server_args.weight_version, "model_type": getattr(model_config.hf_config, "model_type", None), "architectures": getattr(model_config.hf_config, "architectures", None), } return json.dumps(result, default=str) def get_server_info(self) -> str: - # Overlay post-publish overrides (weight version, model path, runtime - # tunables) so the report reflects current config, not the startup record. - result: Dict[str, Any] = get_context().resolved_server_args_dict( - base=dataclasses.asdict(self.server_args) - ) + result: Dict[str, Any] = dataclasses.asdict(self.server_args) result.update(self.scheduler_info) return json.dumps(msgspec_to_builtins(result), default=str) @@ -427,7 +424,9 @@ class RuntimeHandle: "max_model_len": self.tokenizer_manager.model_config.context_len, } ] - if get_lora().enable_lora and hasattr(self.tokenizer_manager, "lora_registry"): + if self.server_args.enable_lora and hasattr( + self.tokenizer_manager, "lora_registry" + ): lora_registry = self.tokenizer_manager.lora_registry for _, lora_ref in lora_registry.get_all_adapters().items(): models.append( diff --git a/python/sglang/srt/entrypoints/http_server.py b/python/sglang/srt/entrypoints/http_server.py index f450ee862..51c85e630 100644 --- a/python/sglang/srt/entrypoints/http_server.py +++ b/python/sglang/srt/entrypoints/http_server.py @@ -703,19 +703,18 @@ async def get_model_info(): @app.get("/model_info") async def model_info(): """Get the model information.""" - from sglang.srt.runtime_context import get_serving - model_config = _global_state.tokenizer_manager.model_config result = { "model_path": _global_state.tokenizer_manager.model_path, "tokenizer_path": _global_state.tokenizer_manager.server_args.tokenizer_path, "is_generation": _global_state.tokenizer_manager.is_generation, "preferred_sampling_params": _global_state.tokenizer_manager.server_args.preferred_sampling_params, - "weight_version": get_serving().weight_version, + "weight_version": _global_state.tokenizer_manager.server_args.weight_version, "has_image_understanding": model_config.is_image_understandable_model, "has_audio_understanding": model_config.is_audio_understandable_model, "model_type": getattr(model_config.hf_config, "model_type", None), "architectures": getattr(model_config.hf_config, "architectures", None), + "weight_version": _global_state.tokenizer_manager.server_args.weight_version, # "hf_config": model_config.hf_config.to_dict(), } return result @@ -749,18 +748,12 @@ async def server_info(): await _global_state.tokenizer_manager.get_internal_state() ) - from sglang.srt.runtime_context import get_context - server_args = _global_state.tokenizer_manager.server_args # server_args.model_config is not serializable but should be excluded by asdict. - # Overlay post-publish overrides so runtime updates (weight version, model - # path/load format) are reported, not the startup record. return msgspec_to_builtins( { - **get_context().resolved_server_args_dict( - base=dataclasses.asdict(server_args) - ), + **dataclasses.asdict(server_args), **_global_state.scheduler_info, "internal_states": internal_states, "version": __version__, @@ -1385,9 +1378,7 @@ async def update_weight_version( # since weight_version update is a simple operation that doesn't affect model weights try: # Update the weight version in server args (the single source of truth) - from sglang.srt.runtime_context import get_context - - get_context().override( + _global_state.tokenizer_manager.server_args.override( "http.update_weight_version", weight_version=obj.new_version ) diff --git a/python/sglang/srt/entrypoints/http_server_engine.py b/python/sglang/srt/entrypoints/http_server_engine.py index 02705a847..4a4996743 100644 --- a/python/sglang/srt/entrypoints/http_server_engine.py +++ b/python/sglang/srt/entrypoints/http_server_engine.py @@ -55,8 +55,6 @@ class HttpServerEngineAdapter(EngineBase): def __init__(self, **kwargs): self.server_args = ServerArgs(**kwargs) - # Read host/port from the adapter's own args: no config is published yet - # in this process (publish happens in the child from launch_server_process). print( f"Launch HttpServerEngineAdapter at: {self.server_args.host}:{self.server_args.port}" ) diff --git a/python/sglang/srt/entrypoints/openai/realtime/session.py b/python/sglang/srt/entrypoints/openai/realtime/session.py index eb122f21a..c5951993e 100644 --- a/python/sglang/srt/entrypoints/openai/realtime/session.py +++ b/python/sglang/srt/entrypoints/openai/realtime/session.py @@ -72,7 +72,6 @@ from sglang.srt.entrypoints.openai.transcription_adapters.base import ( TranscriptionAdapter, ) from sglang.srt.managers.tokenizer_manager import TokenizerManager -from sglang.srt.runtime_context import get_serving from sglang.srt.server_args import ServerArgs from sglang.srt.utils import random_uuid @@ -339,12 +338,12 @@ class RealtimeConnection: if ( transcription is not None and transcription.model - and transcription.model != get_serving().served_model_name + and transcription.model != self.server_args.served_model_name ): await self._send_error( "not_supported", f"Model {transcription.model!r} is not served by this endpoint " - f"(serving {get_serving().served_model_name!r}); set " + f"(serving {self.server_args.served_model_name!r}); set " f"transcription.model to null or to the server's model name.", param="session.audio.input.transcription.model", ) diff --git a/python/sglang/srt/eplb/eplb_manager.py b/python/sglang/srt/eplb/eplb_manager.py index b2b1db792..360ac93b7 100644 --- a/python/sglang/srt/eplb/eplb_manager.py +++ b/python/sglang/srt/eplb/eplb_manager.py @@ -16,7 +16,7 @@ from sglang.srt.eplb.expert_location import ( get_global_expert_location_metadata, ) from sglang.srt.eplb.expert_location_updater import ExpertLocationUpdater -from sglang.srt.runtime_context import get_model +from sglang.srt.runtime_context import get_server_args if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig @@ -274,8 +274,8 @@ def update_expert_location_with_recovery( else: # Load the missing weights from disk update_weights_from_disk_callable( - get_model().model_path, - get_model().load_format, + get_server_args().model_path, + get_server_args().load_format, weight_name_filter=weight_name_filter, ) diff --git a/python/sglang/srt/eplb/expert_location_dispatch.py b/python/sglang/srt/eplb/expert_location_dispatch.py index 64cbf8045..aa144c644 100644 --- a/python/sglang/srt/eplb/expert_location_dispatch.py +++ b/python/sglang/srt/eplb/expert_location_dispatch.py @@ -18,7 +18,7 @@ from typing import Literal, Optional import torch from sglang.srt.eplb.expert_location import get_global_expert_location_metadata -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args @dataclass @@ -34,7 +34,7 @@ class ExpertLocationDispatchInfo: @classmethod def init_new(cls, layer_id: int): - ep_dispatch_algorithm = get_exec().moe.ep_dispatch_algorithm + ep_dispatch_algorithm = get_server_args().ep_dispatch_algorithm expert_location_metadata = get_global_expert_location_metadata() assert expert_location_metadata is not None diff --git a/python/sglang/srt/eplb/expert_location_updater.py b/python/sglang/srt/eplb/expert_location_updater.py index a5ba50923..7873223f0 100644 --- a/python/sglang/srt/eplb/expert_location_updater.py +++ b/python/sglang/srt/eplb/expert_location_updater.py @@ -26,7 +26,7 @@ from sglang.srt.eplb.expert_location import ( ExpertLocationMetadata, get_global_expert_location_metadata, ) -from sglang.srt.runtime_context import get_device +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import get_bool_env_var logger = logging.getLogger(__name__) @@ -107,7 +107,7 @@ def _update_expert_weights_with_canary( canary_tensor = ( _get_canary_value(old_expert_location_metadata, layer_id) .clone() - .to(device=get_device().device, non_blocking=True) + .to(device=get_server_args().device, non_blocking=True) ) routed_experts_weights_of_layer[layer_id].append(canary_tensor) diff --git a/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py b/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py index 455a8715d..84f58eaeb 100644 --- a/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py +++ b/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py @@ -16,8 +16,9 @@ from sglang.srt.hardware_backend.mlx.kv_cache.auxiliary_state import ( from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool from sglang.srt.model_executor.model_runner import ModelRunner -from sglang.srt.model_executor.model_runner_components.layer_setup import ModelLayerInfo -from sglang.srt.runtime_context import get_exec, get_memory, get_schedule +from sglang.srt.model_executor.model_runner_components.layer_setup import ( + ModelLayerInfo, +) logger = logging.getLogger(__name__) @@ -143,7 +144,7 @@ class MlxModelRunnerStub(ModelRunner): (``MlxAuxiliaryStateComponent``) raises ``NotImplementedError`` for the mode. """ - if get_memory().disable_radix_cache: + if self.server_args.disable_radix_cache: return 1 return MLX_AUX_STATE_SIZE_MAX_RUNNING_REQUESTS_RATIO @@ -164,7 +165,7 @@ class MlxModelRunnerStub(ModelRunner): Requires ``self.max_total_num_tokens`` to already be set. """ capacity_cap = self.max_total_num_tokens // 2 - requested = get_schedule().max_running_requests + requested = self.server_args.max_running_requests if requested is None: requested_per_worker = None resolved = min(capacity_cap, 4096) @@ -172,7 +173,7 @@ class MlxModelRunnerStub(ModelRunner): requested_per_worker = requested // self.dp_size resolved = min(requested_per_worker, capacity_cap) - aux_state_size = get_schedule().max_mamba_cache_size + aux_state_size = self.server_args.max_mamba_cache_size if ( mambaish_config(self.model_config) is not None and aux_state_size is not None @@ -208,7 +209,7 @@ class MlxModelRunnerStub(ModelRunner): from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter self.memory_saver_adapter = TorchMemorySaverAdapter.create( - enable=get_exec().features.enable_memory_saver + enable=self.server_args.enable_memory_saver ) # Load model (sets metadata only) @@ -240,7 +241,7 @@ class MlxModelRunnerStub(ModelRunner): # Create minimal pools if mambaish_config(self.model_config) is not None: - auxiliary_state_size = get_schedule().max_mamba_cache_size + auxiliary_state_size = self.server_args.max_mamba_cache_size if auxiliary_state_size is None: auxiliary_state_size = ( self.max_running_requests * self._aux_state_slots_per_request() @@ -254,7 +255,7 @@ class MlxModelRunnerStub(ModelRunner): # With the radix cache disabled no tree component exists to # release auxiliary slots, so the pool owns their release # (see MlxAuxiliaryStateReqToTokenPool docstring). - owns_auxiliary_state_release=get_memory().disable_radix_cache, + owns_auxiliary_state_release=self.server_args.disable_radix_cache, ) else: self.req_to_token_pool = ReqToTokenPool( diff --git a/python/sglang/srt/hardware_backend/mlx/tp_worker.py b/python/sglang/srt/hardware_backend/mlx/tp_worker.py index cc65a61b2..53f9b88c1 100644 --- a/python/sglang/srt/hardware_backend/mlx/tp_worker.py +++ b/python/sglang/srt/hardware_backend/mlx/tp_worker.py @@ -31,7 +31,6 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardBatch, PPProxyTensors, ) -from sglang.srt.runtime_context import get_memory, get_model, get_schedule logger = logging.getLogger(__name__) @@ -48,23 +47,25 @@ class MlxTpModelWorker(TpModelWorker): def _init_model_runner(self): """Create MLX runner first (auto-sizes pool), then stub with matching size.""" from sglang.srt.hardware_backend.mlx.model_runner import MlxModelRunner - from sglang.srt.hardware_backend.mlx.model_runner_stub import MlxModelRunnerStub + from sglang.srt.hardware_backend.mlx.model_runner_stub import ( + MlxModelRunnerStub, + ) logger.info("Initializing MlxModelRunner for end-to-end MLX inference") init_kwargs = dict( - model_path=get_model().model_path, - trust_remote_code=get_model().trust_remote_code, - disable_radix_cache=get_memory().disable_radix_cache, - mem_fraction_static=get_schedule().mem_fraction_static, - quantization=get_model().quantization, + model_path=self.server_args.model_path, + trust_remote_code=self.server_args.trust_remote_code, + disable_radix_cache=self.server_args.disable_radix_cache, + mem_fraction_static=self.server_args.mem_fraction_static, + quantization=self.server_args.quantization, ) - if get_schedule().max_total_tokens is not None: - init_kwargs["pool_size"] = get_schedule().max_total_tokens + if self.server_args.max_total_tokens is not None: + init_kwargs["pool_size"] = self.server_args.max_total_tokens self._mlx_runner = MlxModelRunner(**init_kwargs) self._model_runner = MlxModelRunnerStub( model_config=self.model_config, - mem_fraction_static=get_schedule().mem_fraction_static, + mem_fraction_static=self.server_args.mem_fraction_static, gpu_id=self.gpu_id, ps=self.ps, nccl_port=self.nccl_port, diff --git a/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py b/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py index c48aff4c9..6b823eaec 100644 --- a/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py +++ b/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py @@ -19,9 +19,11 @@ from sglang.srt.layers.attention.flashattention_backend import ( merge_state_v2_wrapper, ) from sglang.srt.layers.radix_attention import AttentionType -from sglang.srt.layers.utils.cp_utils import cp_allgather_and_save_kv_cache +from sglang.srt.layers.utils.cp_utils import ( + cp_allgather_and_save_kv_cache, +) from sglang.srt.mem_cache.memory_pool import KVWriteLoc -from sglang.srt.runtime_context import get_schedule +from sglang.srt.runtime_context import get_server_args if TYPE_CHECKING: from sglang.srt.layers.radix_attention import RadixAttention @@ -513,7 +515,7 @@ class MusaFlashAttentionBackend(FlashAttentionBackend): and not forward_batch.forward_mode.is_draft_extend_v2() ): if forward_batch.attn_attend_prefix_cache: - assert not get_schedule().disable_chunked_prefix_cache + assert not get_server_args().disable_chunked_prefix_cache assert forward_batch.prefix_chunk_idx is not None assert forward_batch.prefix_chunk_cu_seq_lens is not None assert forward_batch.prefix_chunk_max_seq_lens is not None diff --git a/python/sglang/srt/hardware_backend/npu/attention/ascend_dsv4_backend.py b/python/sglang/srt/hardware_backend/npu/attention/ascend_dsv4_backend.py index 971315be2..d45ec4fda 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/ascend_dsv4_backend.py +++ b/python/sglang/srt/hardware_backend/npu/attention/ascend_dsv4_backend.py @@ -12,7 +12,7 @@ from sglang.srt.layers.attention.dsv4.compressor import CompressorBackendMixin from sglang.srt.layers.attention.dsv4.indexer import C4IndexerBackendMixin from sglang.srt.model_executor.forward_batch_info import DSV4OutCacheLoc, ForwardMode from sglang.srt.model_executor.forward_context import get_attn_backend -from sglang.srt.runtime_context import get_parallel, get_spec +from sglang.srt.runtime_context import get_parallel if TYPE_CHECKING: from sglang.srt.layers.radix_attention import RadixAttention @@ -1362,8 +1362,9 @@ class DeepseekV4AscendAttnBackend( or forward_batch.forward_mode.is_draft_extend_v2() ): B = forward_batch.batch_size + from sglang.srt.runtime_context import get_server_args - n_draft = get_spec().speculative_num_draft_tokens or 1 + n_draft = get_server_args().speculative_num_draft_tokens or 1 actual_q = torch.arange( n_draft, B * n_draft + 1, n_draft, dtype=torch.int32, device=device ) @@ -1408,8 +1409,9 @@ class DeepseekV4AscendAttnBackend( forward_batch.forward_mode.is_target_verify() or forward_batch.forward_mode.is_draft_extend_v2() ): + from sglang.srt.runtime_context import get_server_args - max_seqlen_q = get_spec().speculative_num_draft_tokens or 1 + max_seqlen_q = get_server_args().speculative_num_draft_tokens or 1 else: max_seqlen_q = 1 return self._kernel_metadata_from_parts( diff --git a/python/sglang/srt/hardware_backend/npu/graph_runner/vit_npu_graph_runner.py b/python/sglang/srt/hardware_backend/npu/graph_runner/vit_npu_graph_runner.py index 46dd3a101..3738c0d36 100644 --- a/python/sglang/srt/hardware_backend/npu/graph_runner/vit_npu_graph_runner.py +++ b/python/sglang/srt/hardware_backend/npu/graph_runner/vit_npu_graph_runner.py @@ -27,7 +27,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( ) from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.multimodal.vit_cuda_graph_runner import ViTCudaGraphRunner -from sglang.srt.runtime_context import get_mm +from sglang.srt.runtime_context import get_server_args class ViTNpuGraphRunner(ViTCudaGraphRunner): @@ -70,7 +70,7 @@ class ViTNpuGraphRunner(ViTCudaGraphRunner): graph = torch_npu.npu.NPUGraph() vit = self.vit - override_backend = get_mm().mm_attention_backend + override_backend = get_server_args().mm_attention_backend with torch_npu.npu.graph(graph, pool=ViTNpuGraphRunner._graph_memory_pool): y = None deepstack_outs: List[torch.Tensor] = [] diff --git a/python/sglang/srt/hardware_backend/npu/moe/fuseep.py b/python/sglang/srt/hardware_backend/npu/moe/fuseep.py index ad4fc3e87..941c84685 100644 --- a/python/sglang/srt/hardware_backend/npu/moe/fuseep.py +++ b/python/sglang/srt/hardware_backend/npu/moe/fuseep.py @@ -17,7 +17,7 @@ from sglang.srt.environ import envs from sglang.srt.hardware_backend.npu.utils import npu_format_cast from sglang.srt.layers.moe.token_dispatcher.deepep import DeepEPBuffer from sglang.srt.layers.moe.utils import DeepEPMode -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args if TYPE_CHECKING: from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -57,7 +57,7 @@ def forward_fuseep( envs.SGLANG_DEEPEP_NUM_MAX_DISPATCH_TOKENS_PER_RANK.get() ), num_experts=layer.num_experts, - fuse_mode=get_exec().moe.fuseep_mode, + fuse_mode=get_server_args().fuseep_mode, ) return hidden_states @@ -126,7 +126,7 @@ def process_fuseep_weights(layer: torch.nn.Module, weight_prefix: str) -> None: Invoked by ``maybe_apply_fuseep_weights`` for both ``"w13"`` and ``"w2"``. """ - if get_exec().moe.fuseep_mode == 1: + if get_server_args().fuseep_mode == 1: # -- The fused MoE optimization mode "1": dispatch_gmm_combine_decode -- if weight_prefix == "w13": cpu_w13 = layer.w13_weight.data.transpose(1, 2).cpu() @@ -143,7 +143,7 @@ def process_fuseep_weights(layer: torch.nn.Module, weight_prefix: str) -> None: layer.w2_weight_scale = torch.nn.Parameter( w2_scale.to(torch.float32), requires_grad=False ) - elif get_exec().moe.fuseep_mode == 2: + elif get_server_args().fuseep_mode == 2: # -- The fused MoE optimization mode "2": dispatch_ffn_combine -- if weight_prefix == "w13": w13_weight = _release_weight_cache(layer.w13_weight) diff --git a/python/sglang/srt/layers/activation.py b/python/sglang/srt/layers/activation.py index 36fa364c3..a7db9d27a 100644 --- a/python/sglang/srt/layers/activation.py +++ b/python/sglang/srt/layers/activation.py @@ -22,7 +22,9 @@ import torch.nn as nn import torch.nn.functional as F from transformers import PretrainedConfig -from sglang.srt.distributed import divide +from sglang.srt.distributed import ( + divide, +) from sglang.srt.environ import envs from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.utils import MultiPlatformOp @@ -31,7 +33,7 @@ from sglang.srt.model_executor.cuda_graph_config import ( Phase, check_cuda_graph_backend, ) -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import ( cpu_has_amx_support, get_bool_env_var, @@ -87,7 +89,7 @@ logger = logging.getLogger(__name__) class SiluAndMul(MultiPlatformOp): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if get_exec().deterministic.rl_on_policy_target is not None: + if get_server_args().rl_on_policy_target is not None: self._forward_method = self.forward_native elif _use_aiter and envs.SGLANG_OPT_USE_AITER_SILU_MUL.get(): self._forward_method = self.forward_aiter diff --git a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py index 9d5558f8a..526c61b3e 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_indexer.py @@ -37,14 +37,10 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo get_tc_piecewise_forward_context, is_in_tc_piecewise_cuda_graph, ) -from sglang.srt.runtime_context import ( - get_device, - get_exec, - get_parallel, - get_schedule, - get_server_args, +from sglang.srt.runtime_context import get_parallel, get_server_args +from sglang.srt.state_capturer.indexer_topk import ( + maybe_capture_indexer_topk, ) -from sglang.srt.state_capturer.indexer_topk import maybe_capture_indexer_topk from sglang.srt.utils import ( add_prefix, ceil_align, @@ -109,7 +105,9 @@ if is_npu(): import torch_npu from sglang.srt.hardware_backend.npu.utils import get_indexer_weight_stream -from sglang.srt.distributed import get_attn_tp_group +from sglang.srt.distributed import ( + get_attn_tp_group, +) from sglang.srt.distributed.parallel_state import get_pp_group from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.communicator import ScatterMode @@ -460,7 +458,7 @@ class Indexer(MultiPlatformOp): base=rope_theta, # type: ignore rope_scaling=rope_scaling, is_neox_style=is_neox_style, - device=get_device().device, + device=get_server_args().device, ) self.block_size = block_size self.scale_fmt = scale_fmt @@ -471,7 +469,7 @@ class Indexer(MultiPlatformOp): self.num_local_tokens = getattr(config, "index_local_tokens", 0) self.paged_mqa_logits_backend = DSAPagedMQALogitsBackend.resolve( - get_exec().kernel.dsa_paged_mqa_logits_backend + get_server_args().dsa_paged_mqa_logits_backend ) @contextlib.contextmanager @@ -1057,7 +1055,7 @@ class Indexer(MultiPlatformOp): total_mem = torch.cuda.get_device_properties(device_index).total_memory total_mem_budget = int(total_mem * self._MQA_LOGITS_TOTAL_MEM_FRACTION) - mem_fraction_static = get_schedule().mem_fraction_static + mem_fraction_static = get_server_args().mem_fraction_static if mem_fraction_static is None: static_budget = total_mem_budget else: diff --git a/python/sglang/srt/layers/attention/dsa/utils.py b/python/sglang/srt/layers/attention/dsa/utils.py index 753005208..d5b0b72b3 100644 --- a/python/sglang/srt/layers/attention/dsa/utils.py +++ b/python/sglang/srt/layers/attention/dsa/utils.py @@ -12,7 +12,7 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( is_in_tc_piecewise_cuda_graph, ) -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import get_bool_env_var, is_cuda, is_hip from sglang.srt.utils.common import ceil_align, ceil_div @@ -76,20 +76,20 @@ def should_use_dsa_fused_topk( def is_dsa_enable_prefill_cp(): - return get_parallel().enable_dsa_prefill_context_parallel + return get_server_args().enable_dsa_prefill_context_parallel def is_dsa_prefill_cp_in_seq_split(): return ( is_dsa_enable_prefill_cp() - and get_parallel().dsa_prefill_cp_mode == "in-seq-split" + and get_server_args().dsa_prefill_cp_mode == "in-seq-split" ) def is_dsa_prefill_cp_round_robin_split(): return ( is_dsa_enable_prefill_cp() - and get_parallel().dsa_prefill_cp_mode == "round-robin-split" + and get_server_args().dsa_prefill_cp_mode == "round-robin-split" ) diff --git a/python/sglang/srt/layers/attention/dsv4/indexer.py b/python/sglang/srt/layers/attention/dsv4/indexer.py index 596953e55..c5bc40368 100644 --- a/python/sglang/srt/layers/attention/dsv4/indexer.py +++ b/python/sglang/srt/layers/attention/dsv4/indexer.py @@ -28,14 +28,16 @@ from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph.context from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( is_in_tc_piecewise_cuda_graph, ) -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.state_capturer.indexer_topk import get_global_indexer_capturer from sglang.srt.utils import add_prefix, is_cuda, is_hip, is_xpu from sglang.srt.utils.common import is_sm120_supported if TYPE_CHECKING: from sglang.srt.layers.attention.base_attn_backend import AttentionBackend - from sglang.srt.layers.attention.dsv4.compressor import CompressorBackendMixin + from sglang.srt.layers.attention.dsv4.compressor import ( + CompressorBackendMixin, + ) from sglang.srt.layers.quantization import QuantizationConfig from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -127,7 +129,9 @@ def _aiter_fp8_paged_mqa_logits( clean_logits: bool = False, ) -> torch.Tensor: """Wrapper adapting aiter's deepgemm_fp8_paged_mqa_logits to SGLang's interface.""" - from aiter.ops.triton.attention.pa_mqa_logits import deepgemm_fp8_paged_mqa_logits + from aiter.ops.triton.attention.pa_mqa_logits import ( + deepgemm_fp8_paged_mqa_logits, + ) batch_size = q_fp8.shape[0] next_n = q_fp8.shape[1] @@ -834,8 +838,9 @@ class C4Indexer(nn.Module): self.rotary_emb = rotary_emb self.freqs_cis = freqs_cis self.weight_scale: float = self.softmax_scale * self.n_heads**-0.5 + from sglang.srt.runtime_context import get_server_args - self.use_fp4_indexer = get_exec().kernel.enable_deepseek_v4_fp4_indexer + self.use_fp4_indexer = get_server_args().enable_deepseek_v4_fp4_indexer self.alt_streams = alt_streams def compute_q( diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index e47efaae0..973b2a2dd 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -13,7 +13,9 @@ from sglang.kernels.ops.attention.metadata import ( ) from sglang.kernels.ops.attention.pa_page_table import _build_pa_page_table from sglang.kernels.ops.attention.utils import assert_buffer_fits -from sglang.kernels.ops.kvcache.trtllm_mha_page_table import build_trtllm_mha_page_table +from sglang.kernels.ops.kvcache.trtllm_mha_page_table import ( + build_trtllm_mha_page_table, +) from sglang.srt.configs.model_config import AttentionArch from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy @@ -26,7 +28,7 @@ from sglang.srt.layers.utils.cp_utils import ( from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode -from sglang.srt.runtime_context import get_schedule +from sglang.srt.runtime_context import get_server_args from sglang.srt.speculative.ragged_verify import build_ragged_target_verify_geometry from sglang.srt.speculative.spec_info import SpecInput, SpeculativeAlgorithm from sglang.srt.speculative.spec_utils import resolve_num_tokens_per_req @@ -164,12 +166,9 @@ class FlashAttentionBackend(AttentionBackend): self.token_to_kv_pool = model_runner.token_to_kv_pool self.req_to_token = model_runner.req_to_token_pool.req_to_token self.kv_cache_dtype = model_runner.kv_cache_dtype + from sglang.srt.runtime_context import get_model - self.kv_cache_dtype_str = getattr( - model_runner, - "kv_cache_dtype_str", - model_runner.server_args.kv_cache_dtype, - ) + self.kv_cache_dtype_str = get_model().kv_cache_dtype self.kv_cache_is_mxfp8 = self.kv_cache_dtype_str == "mxfp8" self.page_size = model_runner.page_size # Static page-table width (upper bound). The device-side page-table build @@ -1480,7 +1479,7 @@ class FlashAttentionBackend(AttentionBackend): ): # Do multi-head attention with chunked prefix cache if forward_batch.attn_attend_prefix_cache: - assert not get_schedule().disable_chunked_prefix_cache + assert not get_server_args().disable_chunked_prefix_cache # MHA for chunked prefix kv cache when running model with MLA assert forward_batch.prefix_chunk_idx is not None assert forward_batch.prefix_chunk_cu_seq_lens is not None diff --git a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py index 27599bfca..0d68b303c 100644 --- a/python/sglang/srt/layers/attention/flashinfer_mla_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_mla_backend.py @@ -1,6 +1,6 @@ from __future__ import annotations -from sglang.srt.runtime_context import get_disagg, get_exec, get_parallel, get_schedule +from sglang.srt.runtime_context import get_parallel """ Support attention backend for flashinfer MLA. @@ -32,7 +32,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMo from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( is_in_tc_piecewise_cuda_graph, ) -from sglang.srt.runtime_context import get_buffer +from sglang.srt.runtime_context import get_buffer, get_server_args from sglang.srt.speculative.spec_info import SpecInput from sglang.srt.speculative.spec_utils import ( draft_kv_indices_buffer_width, @@ -223,9 +223,9 @@ class FlashInferMLAAttnBackend(AttentionBackend): self.token_to_kv_pool = model_runner.token_to_kv_pool self.enable_chunk_kv = ( not skip_prefill - and get_disagg().disaggregation_mode != "decode" - and not get_schedule().disable_chunked_prefix_cache - and not get_exec().kernel.flashinfer_mla_disable_ragged + and get_server_args().disaggregation_mode != "decode" + and not get_server_args().disable_chunked_prefix_cache + and not get_server_args().flashinfer_mla_disable_ragged ) self.page_size = model_runner.page_size @@ -401,7 +401,7 @@ class FlashInferMLAAttnBackend(AttentionBackend): prefix_lens = forward_batch.extend_prefix_lens extend_no_prefix = not any(forward_batch.extend_prefix_lens_cpu) use_ragged = ( - not get_exec().kernel.flashinfer_mla_disable_ragged + not get_server_args().flashinfer_mla_disable_ragged and extend_no_prefix # Piecewise cuda graph should use paged prefill to be compatible with prefix cache and not is_in_tc_piecewise_cuda_graph() diff --git a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py index 8fbeb9ff6..6e9e57adf 100644 --- a/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py +++ b/python/sglang/srt/layers/attention/hybrid_linear_attn_backend.py @@ -19,7 +19,7 @@ from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.model_executor.model_runner import ModelRunner -from sglang.srt.runtime_context import get_exec, get_memory, get_server_args +from sglang.srt.runtime_context import get_server_args from sglang.srt.speculative.eagle_info import EagleDraftInput, EagleVerifyInput from sglang.srt.speculative.spec_info import SpecInput @@ -350,7 +350,7 @@ class MambaAttnBackendBase(AttentionBackend): """Per-row (length bs) bool flush mask = the radix track's seq_lens_cpu % mamba_track_interval == 0, so force-flush and snapshot fire on the same steps (no off-by-one).""" - interval = get_exec().mamba.mamba_track_interval + interval = get_server_args().mamba_track_interval if seq_lens_cpu is None: # Should not happen for the supported config; stay safe and never flush. return torch.zeros((bs,), dtype=torch.bool) @@ -764,7 +764,7 @@ class Mamba2AttnBackend(MambaAttnBackendBase): # Page-major stores state strided; only the stride-aware Triton causal-conv # reads it (CUDA causal_conv1d garbles it). A model may also force Triton. use_triton_causal_conv = ( - use_triton_causal_conv or get_memory().enable_page_major_kv_layout + use_triton_causal_conv or get_server_args().enable_page_major_kv_layout ) layer_cache = self.req_to_token_pool.mamba2_layer_cache(layer_id) mixer_out, intermediate_states = mixer.forward( diff --git a/python/sglang/srt/layers/attention/trtllm_mla_backend.py b/python/sglang/srt/layers/attention/trtllm_mla_backend.py index 6fb30a444..d69c6c99a 100755 --- a/python/sglang/srt/layers/attention/trtllm_mla_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mla_backend.py @@ -38,11 +38,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMo from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph import ( is_in_tc_piecewise_cuda_graph, ) -from sglang.srt.runtime_context import ( - get_buffer, - get_parallel, - get_schedule, -) +from sglang.srt.runtime_context import get_buffer, get_parallel, get_server_args from sglang.srt.utils import is_flashinfer_available, is_float4_e2m1fn_x2 if is_flashinfer_available(): @@ -201,7 +197,9 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend): self.forward_prefill_metadata: Optional[TRTLLMMLAPrefillMetadata] = None self.forward_decode_metadata: Union[TRTLLMMLADecodeMetadata, None] = None - self.disable_chunked_prefix_cache = get_schedule().disable_chunked_prefix_cache + self.disable_chunked_prefix_cache = ( + get_server_args().disable_chunked_prefix_cache + ) self.num_draft_tokens = model_runner.server_args.speculative_num_draft_tokens self.cuda_graph_custom_mask = None diff --git a/python/sglang/srt/layers/attention/vision.py b/python/sglang/srt/layers/attention/vision.py index 48f07e9f5..b996632b6 100644 --- a/python/sglang/srt/layers/attention/vision.py +++ b/python/sglang/srt/layers/attention/vision.py @@ -15,7 +15,7 @@ from einops import rearrange from sglang.jit_kernel.norm import can_use_fused_inplace_qknorm as can_use_jit_qk_norm from sglang.srt.environ import envs from sglang.srt.models.utils import apply_qk_norm -from sglang.srt.runtime_context import get_exec, get_mm, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import ( cpu_has_amx_support, get_bool_env_var, @@ -69,7 +69,9 @@ if _is_npu: if _is_xpu: from sgl_kernel.flash_attn import flash_attn_varlen_func -from sglang.kernels.ops.attention.prefill_attention import context_attention_fwd +from sglang.kernels.ops.attention.prefill_attention import ( + context_attention_fwd, +) from sglang.srt.distributed import ( split_tensor_along_last_dim, tensor_model_parallel_all_gather, @@ -84,6 +86,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.quantization import QuantizationConfig from sglang.srt.layers.rotary_embedding import apply_rotary_pos_emb from sglang.srt.layers.rotary_embedding.utils import apply_rotary_pos_emb_native_eager +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import add_prefix _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip @@ -1042,7 +1045,7 @@ class VisionAttention(nn.Module): # Select attention backend via a unified method _passed_backend = qkv_backend qkv_backend = self._determine_attention_backend(_passed_backend) - if get_mm().mm_attention_backend is None and _passed_backend is None: + if get_server_args().mm_attention_backend is None and _passed_backend is None: print_info_once(f"Multimodal attention backend not set. Use {qkv_backend}.") print_info_once(f"Using {qkv_backend} as multimodal attention backend.") @@ -1121,7 +1124,7 @@ class VisionAttention(nn.Module): weight_dtype=torch.float32, cast_x_before_out_mul=True, ) - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else {} ) q_norm = RMSNorm( @@ -1149,7 +1152,7 @@ class VisionAttention(nn.Module): - CUDA (other): "triton_attn" - Non-CUDA: "sdpa" """ - override_backend = get_mm().mm_attention_backend + override_backend = get_server_args().mm_attention_backend if override_backend is not None: backend = override_backend elif passed_backend is not None: @@ -1254,7 +1257,7 @@ class VisionAttention(nn.Module): x = x.unsqueeze(0) assert x.dim() == 3, x.shape if ( - get_exec().deterministic.rl_on_policy_target is not None + get_server_args().rl_on_policy_target is not None and position_embeddings is not None ): assert isinstance(position_embeddings, tuple), ( diff --git a/python/sglang/srt/layers/attention/xpu_backend.py b/python/sglang/srt/layers/attention/xpu_backend.py index 60cc92d8c..27ad57ce5 100644 --- a/python/sglang/srt/layers/attention/xpu_backend.py +++ b/python/sglang/srt/layers/attention/xpu_backend.py @@ -15,7 +15,7 @@ from sglang.srt.layers.attention.flashattention_backend import ( from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode -from sglang.srt.runtime_context import get_schedule +from sglang.srt.runtime_context import get_server_args if TYPE_CHECKING: from sglang.srt.layers.radix_attention import RadixAttention @@ -69,12 +69,9 @@ class XPUAttentionBackend(AttentionBackend): self.token_to_kv_pool = model_runner.token_to_kv_pool self.req_to_token = model_runner.req_to_token_pool.req_to_token self.kv_cache_dtype = model_runner.kv_cache_dtype + from sglang.srt.runtime_context import get_model - self.kv_cache_dtype_str = getattr( - model_runner, - "kv_cache_dtype_str", - model_runner.server_args.kv_cache_dtype, - ) + self.kv_cache_dtype_str = get_model().kv_cache_dtype self.page_size = model_runner.page_size self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA self.skip_prefill = skip_prefill @@ -643,7 +640,7 @@ class XPUAttentionBackend(AttentionBackend): ): # Do multi-head attention with chunked prefix cache if forward_batch.attn_attend_prefix_cache: - assert not get_schedule().disable_chunked_prefix_cache + assert not get_server_args().disable_chunked_prefix_cache # MHA for chunked prefix kv cache when running model with MLA assert forward_batch.prefix_chunk_idx is not None assert forward_batch.prefix_chunk_cu_seq_lens is not None diff --git a/python/sglang/srt/layers/communicator.py b/python/sglang/srt/layers/communicator.py index ddc914c96..b0969bded 100644 --- a/python/sglang/srt/layers/communicator.py +++ b/python/sglang/srt/layers/communicator.py @@ -72,12 +72,7 @@ from sglang.srt.model_executor.cuda_graph_config import ( check_cuda_graph_backend, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch -from sglang.srt.runtime_context import ( - get_exec, - get_forward, - get_parallel, - get_spec, -) +from sglang.srt.runtime_context import get_forward, get_parallel, get_server_args from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils import ( get_bool_env_var, @@ -175,7 +170,7 @@ def apply_flashinfer_allreduce_fusion(batch_size: int): and batch_size > 0 and batch_size <= FUSE_ALLREDUCE_MAX_BATCH_SIZE and not is_dp_attention_enabled() - and get_exec().comm.flashinfer_allreduce_fusion_backend is not None + and get_server_args().flashinfer_allreduce_fusion_backend is not None and not is_flashinfer_allreduce_unavailable() ) @@ -191,7 +186,7 @@ def apply_aiter_all_reduce_fusion(input_tensor: torch.Tensor): and total_bytes <= 8 * 1024 * 8192 and get_parallel().tp_size != 6 and not is_dp_attention_enabled() - and get_exec().comm.enable_aiter_allreduce_fusion + and get_server_args().enable_aiter_allreduce_fusion ) @@ -270,7 +265,7 @@ class AttnTpContext: def init_context(self, q_lora_rank, is_dsa): self.is_dsa = is_dsa self.allow_input_scattered = ( - get_parallel().enable_attn_tp_input_scattered + get_server_args().enable_attn_tp_input_scattered and (_is_cuda or _is_npu) and q_lora_rank is not None and not is_dsa @@ -279,9 +274,9 @@ class AttnTpContext: and get_moe_a2a_backend().is_none() and not enable_moe_dense_fully_dp() and not check_cuda_graph_backend(Phase.PREFILL, Backend.TC_PIECEWISE) - and get_spec().speculative_algorithm != "EAGLE3" + and get_server_args().speculative_algorithm != "EAGLE3" ) - if get_parallel().enable_attn_tp_input_scattered: + if get_server_args().enable_attn_tp_input_scattered: if not self.allow_input_scattered: logging.info( "attn_tp_input_scattered is not enabled while other conditions are not met" @@ -412,7 +407,7 @@ class LayerScatterModes: not context.is_layer_sparse and context.is_next_layer_sparse and enable_moe_dense_fully_dp() - and get_exec().overlap.enable_two_batch_overlap + and get_server_args().enable_two_batch_overlap ) @classmethod @@ -439,11 +434,11 @@ class LayerScatterModes: def enable_moe_dense_fully_dp(): - return get_parallel().moe_dense_tp_size == 1 + return get_server_args().moe_dense_tp_size == 1 def enable_dwdp(): - return get_parallel().dwdp_size > 1 + return get_server_args().dwdp_size > 1 class LayerCommunicator: @@ -476,7 +471,7 @@ class LayerCommunicator: ) self._post_init_communicate() self._speculative_algo = SpeculativeAlgorithm.from_string( - get_spec().speculative_algorithm + get_server_args().speculative_algorithm ) def _post_init_communicate(self): @@ -845,7 +840,7 @@ class LayerCommunicator: and get_parallel().tp_size != 6 and not is_dp_attention_enabled() and get_moe_a2a_backend().is_none() - and get_exec().comm.enable_aiter_allreduce_fusion + and get_server_args().enable_aiter_allreduce_fusion ) ) and (not self.is_last_layer) @@ -1150,7 +1145,7 @@ class CommunicateWithAllReduceAndLayerNormFn: if not handled: quantize_communications = ( not forward_batch.forward_mode.is_decode_or_idle() - and get_exec().comm.enable_quant_communications + and get_server_args().enable_quant_communications ) if quantize_communications: hidden_states = attention_tensor_model_parallel_quant_all_reduce( diff --git a/python/sglang/srt/layers/cp/zigzag.py b/python/sglang/srt/layers/cp/zigzag.py index 06f3bda8b..b7fe868d6 100644 --- a/python/sglang/srt/layers/cp/zigzag.py +++ b/python/sglang/srt/layers/cp/zigzag.py @@ -48,10 +48,12 @@ from sglang.srt.layers.cp.base import ( CPAttentionBackendKind, ) from sglang.srt.layers.cp.padding import pad_local_rows -from sglang.srt.layers.dp_attention import is_allocation_symmetric +from sglang.srt.layers.dp_attention import ( + is_allocation_symmetric, +) from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.model_executor.forward_context import get_token_to_kv_pool -from sglang.srt.runtime_context import get_device, get_parallel +from sglang.srt.runtime_context import get_parallel @dataclass @@ -206,8 +208,10 @@ class ZigzagCPStrategy(ContextParallelStrategy): actual_seq_q_prev_list.append(block_sizes[cp_rank]) actual_seq_q_next_list.append(block_sizes[cp_segment_num - cp_rank - 1]) + from sglang.srt.runtime_context import get_server_args + try: - device = torch.device(get_device().device) + device = torch.device(get_server_args().device) except Exception: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") cu_prev = [0] + list(accumulate(actual_seq_q_prev_list)) diff --git a/python/sglang/srt/layers/dcp/planner.py b/python/sglang/srt/layers/dcp/planner.py index d41d23630..1a9caba76 100644 --- a/python/sglang/srt/layers/dcp/planner.py +++ b/python/sglang/srt/layers/dcp/planner.py @@ -26,7 +26,7 @@ from sglang.kernels.ops.attention.dcp_kernels import ( ) from sglang.srt.layers.dcp.layout import update_local_kv_lens_for_dcp from sglang.srt.layers.dcp.metadata import DecodeContextParallelMetadata -from sglang.srt.runtime_context import get_device, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args def prepare_decode_context_parallel_metadata( @@ -53,12 +53,12 @@ def prepare_decode_context_parallel_metadata( extend_prefix_starts = torch.zeros( len(seq_lens), dtype=torch.int32, - device=get_device().device, + device=get_server_args().device, ) extend_cu_prefix_lens = torch.zeros( len(seq_lens) + 1, dtype=torch.int32, - device=get_device().device, + device=get_server_args().device, ) extend_cu_prefix_lens[1:] = torch.cumsum(extend_prefix_lens, dim=0) extend_cu_prefix_lens = extend_cu_prefix_lens[:-1] @@ -67,7 +67,7 @@ def prepare_decode_context_parallel_metadata( dcp_prefix_kv_indices = torch.empty( sum(extend_prefix_lens_cpu), dtype=torch.int32, - device=get_device().device, + device=get_server_args().device, ) create_chunked_prefix_cache_kv_indices_fn[(len(seq_lens),)]( req_to_token, @@ -81,20 +81,20 @@ def prepare_decode_context_parallel_metadata( dcp_kv_indptr = torch.zeros( len(seq_lens) + 1, dtype=torch.int32, - device=get_device().device, + device=get_server_args().device, ) dcp_kv_indptr[1:] = seq_lens.cumsum(dim=0) dcp_kv_indptr = dcp_kv_indptr[: (len(seq_lens) + 1)] dcp_kv_indices = torch.zeros( seq_lens_sum, dtype=torch.int32, - device=get_device().device, + device=get_server_args().device, ) extend_cu_lens = torch.zeros( len(seq_lens) + 1, dtype=torch.int32, - device=get_device().device, + device=get_server_args().device, ) extend_cu_lens[1:] = torch.cumsum(extend_seq_lens, dim=0) extend_cu_lens = extend_cu_lens[:-1] diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index 8f7582a93..d57c5cf8f 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -32,7 +32,7 @@ from sglang.srt.model_executor.cuda_graph_config import ( Phase, check_cuda_graph_backend, ) -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import ( cpu_has_amx_support, get_bool_env_var, @@ -148,7 +148,9 @@ if _is_cuda: _jit_rmsnorm_hf = None from sglang.jit_kernel.norm import fused_add_rmsnorm as _jit_fused_add_rmsnorm - from sglang.jit_kernel.norm import is_supported_jit_fused_add_rmsnorm_hidden_size + from sglang.jit_kernel.norm import ( + is_supported_jit_fused_add_rmsnorm_hidden_size, + ) logger = logging.getLogger(__name__) @@ -219,7 +221,7 @@ def _forward_with_allreduce_fusion( return fused_result # For AITER route, preserve correctness when fused path is unavailable. - if _use_aiter and get_exec().comm.enable_aiter_allreduce_fusion: + if _use_aiter and get_server_args().enable_aiter_allreduce_fusion: x = tensor_model_parallel_all_reduce(x) return norm_module.forward(x, residual, None) @@ -421,7 +423,7 @@ class RMSNorm(MultiPlatformOp): if ( residual is not None or self.cast_x_before_out_mul - or get_exec().deterministic.rl_on_policy_target == "fsdp" + or get_server_args().rl_on_policy_target == "fsdp" ): return self.forward_native(x, residual, post_residual_addition) out = rms_norm_batch_invariant( @@ -528,7 +530,7 @@ class RMSNorm(MultiPlatformOp): if ( residual is not None or self.cast_x_before_out_mul - or get_exec().deterministic.rl_on_policy_target == "fsdp" + or get_server_args().rl_on_policy_target == "fsdp" or (self._fused_pad_kernel is not None and self.x_pad_to_multiple > 0) ): return self.forward_native(x, residual, post_residual_addition) @@ -589,7 +591,7 @@ class RMSNorm(MultiPlatformOp): if ( residual is not None or self.cast_x_before_out_mul - or get_exec().deterministic.rl_on_policy_target == "fsdp" + or get_server_args().rl_on_policy_target == "fsdp" ): return self.forward_native(x, residual, post_residual_addition) return rms_norm_batch_invariant( @@ -716,10 +718,7 @@ class RMSNorm(MultiPlatformOp): if self.variance_size_override is not None: return self.forward_native(x, residual, post_residual_addition) if is_batch_invariant_mode_enabled(): - if ( - residual is not None - or get_exec().deterministic.rl_on_policy_target == "fsdp" - ): + if residual is not None or get_server_args().rl_on_policy_target == "fsdp": return self.forward_native(x, residual, post_residual_addition) return rms_norm_batch_invariant( x, diff --git a/python/sglang/srt/layers/linear.py b/python/sglang/srt/layers/linear.py index 674ea2c65..57aae27ba 100644 --- a/python/sglang/srt/layers/linear.py +++ b/python/sglang/srt/layers/linear.py @@ -25,7 +25,9 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) from sglang.srt.environ import envs -from sglang.srt.layers.dp_attention import is_allocation_symmetric +from sglang.srt.layers.dp_attention import ( + is_allocation_symmetric, +) from sglang.srt.layers.moe.utils import should_skip_mlp_all_reduce from sglang.srt.layers.parameter import ( BasevLLMParameter, @@ -37,7 +39,7 @@ from sglang.srt.layers.parameter import ( _ColumnvLLMParameter, ) from sglang.srt.layers.utils import pad_or_narrow_weight -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import get_bool_env_var, is_cpu, is_hip, is_npu, set_weight_attrs if TYPE_CHECKING: @@ -757,7 +759,9 @@ class MergedColumnParallelLinear(ColumnParallelLinear): shard_offsets.append((i, current_shard_offset, output_size)) current_shard_offset += output_size if _is_cpu: - from sglang.srt.model_loader.weight_utils import pad_loaded_weight + from sglang.srt.model_loader.weight_utils import ( + pad_loaded_weight, + ) loaded_weight = pad_loaded_weight( loaded_weight, param.output_dim, output_sizes @@ -801,7 +805,9 @@ class MergedColumnParallelLinear(ColumnParallelLinear): current_block_offset += shard_block_size if _is_cpu: - from sglang.srt.model_loader.weight_utils import pad_loaded_weight + from sglang.srt.model_loader.weight_utils import ( + pad_loaded_weight, + ) loaded_weight = pad_loaded_weight( loaded_weight, param.output_dim, shard_block_sizes @@ -1590,7 +1596,7 @@ class RowParallelLinear(LinearBase): quantize_communications = ( ( not forward_batch.forward_mode.is_decode_or_idle() - and get_exec().comm.enable_quant_communications + and get_server_args().enable_quant_communications ) if forward_batch is not None else False diff --git a/python/sglang/srt/layers/logits_processor.py b/python/sglang/srt/layers/logits_processor.py index df03ae977..cf55b97cf 100644 --- a/python/sglang/srt/layers/logits_processor.py +++ b/python/sglang/srt/layers/logits_processor.py @@ -47,7 +47,7 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardBatch, ForwardMode, ) -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils.common import ( is_cpu, is_npu, @@ -345,8 +345,8 @@ class LogitsProcessor(nn.Module): self.config = config self.vocab_size = config.vocab_size self.logit_scale = logit_scale - self.use_attn_tp_group = get_parallel().enable_dp_lm_head - self.use_fp32_lm_head = get_exec().features.enable_fp32_lm_head + self.use_attn_tp_group = get_server_args().enable_dp_lm_head + self.use_fp32_lm_head = get_server_args().enable_fp32_lm_head if self.use_attn_tp_group: self.attn_tp_size = get_parallel().attn_tp_size self.do_tensor_parallel_all_gather = ( @@ -370,8 +370,8 @@ class LogitsProcessor(nn.Module): self.final_logit_softcapping = None self.return_full_logits = return_full_logits - self.enable_mis = get_exec().features.enable_mis - self.rl_on_policy_target = get_exec().deterministic.rl_on_policy_target + self.enable_mis = get_server_args().enable_mis + self.rl_on_policy_target = get_server_args().rl_on_policy_target self._logits_gatherer = triton_symm_mem_ag.MultimemAllGatherer( max_tokens=triton_symm_mem_ag.recommended_max_tokens( diff --git a/python/sglang/srt/layers/moe/hash_topk.py b/python/sglang/srt/layers/moe/hash_topk.py index 533487851..ebcf45702 100644 --- a/python/sglang/srt/layers/moe/hash_topk.py +++ b/python/sglang/srt/layers/moe/hash_topk.py @@ -7,7 +7,9 @@ import torch from torch import nn from sglang.srt.environ import envs -from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder +from sglang.srt.eplb.expert_distribution import ( + get_global_expert_distribution_recorder, +) from sglang.srt.eplb.expert_location_dispatch import ( ExpertLocationDispatchInfo, topk_ids_logical_to_physical, @@ -20,7 +22,6 @@ from sglang.srt.layers.moe.topk import ( remap_topk_for_per_rank_shared_slots, ) from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots -from sglang.srt.runtime_context import get_exec from sglang.srt.utils import is_hip, is_npu logger = logging.getLogger(__name__) @@ -43,9 +44,10 @@ class HashTopK(nn.Module): ): super().__init__() self.layer_id = layer_id + from sglang.srt.runtime_context import get_server_args self.enable_waterfill = ( - num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill + num_fused_shared_experts > 0 and get_server_args().enable_waterfill ) self.waterfill_balancer = None diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py index 4e98f9cda..9b456f122 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe.py @@ -28,7 +28,7 @@ from sglang.srt.environ import envs from sglang.srt.layers.dp_attention import is_allocation_symmetric from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig from sglang.srt.layers.moe.utils import get_moe_padding_size -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import ( cpu_has_amx_support, get_bool_env_var, @@ -506,7 +506,7 @@ def _fused_moe_kernel_sequence( out_hidden_states = torch.empty_like(hidden_states) use_fused_moe_sum_all_reduce = ( - get_exec().moe.enable_fused_moe_sum_all_reduce + get_server_args().enable_fused_moe_sum_all_reduce and (not no_combine) and (topk > 2) and (not use_int8_w8a16) diff --git a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe_triton_config.py b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe_triton_config.py index c4ad7d49d..210247f86 100644 --- a/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe_triton_config.py +++ b/python/sglang/srt/layers/moe/moe_runner/triton_utils/fused_moe_triton_config.py @@ -9,7 +9,7 @@ from typing import Any, Dict, List, Optional, Tuple import torch import triton -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import get_device_name, is_hip logger = logging.getLogger(__name__) @@ -69,7 +69,7 @@ def get_moe_configs( kernel on a given batch size bs, the closest batch size in the grid should be picked and the associated configuration chosen to invoke the kernel. """ - if get_exec().deterministic.enable_deterministic_inference: + if get_server_args().enable_deterministic_inference: logger.warning( "Deterministic inference is enabled, using default MoE kernel config." ) @@ -187,7 +187,7 @@ def get_default_config( is_marlin: bool, block_shape: Optional[List[int]] = None, ) -> Dict[str, int]: - if get_exec().deterministic.enable_deterministic_inference: + if get_server_args().enable_deterministic_inference: config = { "BLOCK_SIZE_M": 64, "BLOCK_SIZE_N": 64, diff --git a/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py b/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py index 8e08b5b81..53e1b086a 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py @@ -21,9 +21,13 @@ from sglang.srt.layers.moe.token_dispatcher import ( from sglang.srt.layers.moe.token_dispatcher.flashinfer_utils import ( TorchDistributedCommBackend, ) -from sglang.srt.layers.moe.topk import StandardTopKOutput, TopKOutput, TopKOutputChecker +from sglang.srt.layers.moe.topk import ( + StandardTopKOutput, + TopKOutput, + TopKOutputChecker, +) from sglang.srt.layers.moe.utils import get_moe_runner_backend -from sglang.srt.runtime_context import get_schedule, get_spec +from sglang.srt.runtime_context import get_server_args from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils import get_int_env_var @@ -119,7 +123,7 @@ class FlashinferDispatcher(BaseDispatcher): # max_running_requests is not yet resolved at model-construction time, # so we use 4096 as a floor to cover decode batches and _dummy_run # (which warms up at batch_size = req_to_token_pool.size). - cps = get_schedule().chunked_prefill_size + cps = get_server_args().chunked_prefill_size default_max_tokens = max(cps if cps and cps > 0 else 4096, 4096) self.max_num_tokens = get_int_env_var( "SGLANG_FLASHINFER_NUM_MAX_DISPATCH_TOKENS_PER_RANK", @@ -128,7 +132,7 @@ class FlashinferDispatcher(BaseDispatcher): # Calculate workspace size. For eagle mode, use the larger workspace size since nextn layer will be unquantized. speculative_algo = SpeculativeAlgorithm.from_string( - get_spec().speculative_algorithm + get_server_args().speculative_algorithm ) if MOE_NVFP4_DISPATCH and not speculative_algo.is_eagle(): total_dispatch_payload_size_per_token = ( diff --git a/python/sglang/srt/layers/moe/token_dispatcher/nixl.py b/python/sglang/srt/layers/moe/token_dispatcher/nixl.py index 93a72433e..090eb05fa 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/nixl.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/nixl.py @@ -23,7 +23,6 @@ from sglang.srt.layers.moe.token_dispatcher.deepep import ( ) from sglang.srt.layers.moe.topk import TopKOutput from sglang.srt.layers.moe.utils import DeepEPMode -from sglang.srt.runtime_context import get_parallel try: from nixl_ep import Buffer @@ -128,7 +127,9 @@ class NixlEPBuffer: offset = ElasticEPStateManager.get_ep_join_rank_offset() global_rank = rank + offset - max_ep_size = get_parallel().max_ep_size or world_size + from sglang.srt.runtime_context import get_server_args + + max_ep_size = get_server_args().max_ep_size or world_size nixl_max_ranks = max_ep_size num_rdma_bytes = 0 @@ -225,8 +226,9 @@ class _NixlEPDispatcherImplBase: elastic_state.active_ranks if elastic_state is not None else None ) self._active_world_size = dist.get_world_size(group) + from sglang.srt.runtime_context import get_server_args - _max_ep = get_parallel().max_ep_size or self._active_world_size + _max_ep = get_server_args().max_ep_size or self._active_world_size self._mask_buffer = ( torch.zeros(_max_ep, dtype=torch.int32, device="cuda") if self.active_ranks is not None diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index f7d0f1db6..b87276287 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -32,7 +32,7 @@ from typing import ( import torch import torch.nn.functional as F -from sglang.srt.runtime_context import get_exec, get_lora, get_parallel +from sglang.srt.runtime_context import get_parallel try: from triton_kernels.matmul_ogs import GatherIndx, RoutingData, ScatterIndx @@ -83,7 +83,9 @@ except ImportError: pass from sglang.kernels.ops.attention.dsv4 import mask_topk_ids -from sglang.srt.distributed import get_tp_group +from sglang.srt.distributed import ( + get_tp_group, +) from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) @@ -96,7 +98,9 @@ from sglang.srt.eplb.expert_location_dispatch import ( ) from sglang.srt.layers.dp_attention import is_allocation_symmetric from sglang.srt.layers.moe import get_moe_runner_backend -from sglang.srt.layers.moe.utils import has_per_rank_fused_shared_slots +from sglang.srt.layers.moe.utils import ( + has_per_rank_fused_shared_slots, +) from sglang.srt.layers.utils import MultiPlatformOp from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer from sglang.srt.utils import ( @@ -415,9 +419,10 @@ class TopK(MultiPlatformOp): assert num_expert_group is not None and topk_group is not None self.layer_id = layer_id + from sglang.srt.runtime_context import get_server_args self.enable_waterfill = ( - num_fused_shared_experts > 0 and get_exec().moe.enable_waterfill + num_fused_shared_experts > 0 and get_server_args().enable_waterfill ) self.waterfill_balancer = None @@ -491,8 +496,9 @@ class TopK(MultiPlatformOp): # ===== TO BE REFACTORED ==== elif get_moe_runner_backend().is_experimental_sgl_trtllm(): try: + from sglang.srt.runtime_context import get_server_args - use_standard_for_lora = bool(get_lora().enable_lora) + use_standard_for_lora = bool(get_server_args().enable_lora) except ValueError: use_standard_for_lora = False output_format = ( diff --git a/python/sglang/srt/layers/quantization/fp8_utils.py b/python/sglang/srt/layers/quantization/fp8_utils.py index cc75aa730..15678b941 100755 --- a/python/sglang/srt/layers/quantization/fp8_utils.py +++ b/python/sglang/srt/layers/quantization/fp8_utils.py @@ -13,7 +13,7 @@ from sglang.kernels.ops.quantization.fp8_kernel import ( ) from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.quantization.mxfp4_tensor import MXFP4QuantizeUtil -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils.common import torch_release if TYPE_CHECKING: @@ -34,6 +34,7 @@ from sglang.kernels.ops.quantization.fp8_kernel import ( w8a8_block_fp8_matmul_deepgemm, w8a8_block_fp8_matmul_triton, ) +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import ( ceil_align, ceil_div, @@ -1469,7 +1470,9 @@ def requant_block_scale_ue8m0_for_deepgemm( scales are not already UE8M0, and DeepGEMM can run the layer (bf16 output, aligned shape). Returns True when it requantizes. """ - from sglang.srt.model_loader.utils import should_deepgemm_weight_requant_ue8m0 + from sglang.srt.model_loader.utils import ( + should_deepgemm_weight_requant_ue8m0, + ) if ( not use_deepgemm_runner @@ -1791,7 +1794,7 @@ def apply_fp8_linear( if ( input_scale is not None and input_scale.numel() == 1 - and get_exec().graph.cuda_graph_config.prefill.tc_compiler == "inductor" + and get_server_args().cuda_graph_config.prefill.tc_compiler == "inductor" ): qinput = ( (input_2d * input_scale.reciprocal()) diff --git a/python/sglang/srt/layers/quantization/mxfp4.py b/python/sglang/srt/layers/quantization/mxfp4.py index 458730649..822998fc2 100644 --- a/python/sglang/srt/layers/quantization/mxfp4.py +++ b/python/sglang/srt/layers/quantization/mxfp4.py @@ -48,7 +48,7 @@ from sglang.srt.layers.quantization.base_config import ( QuantizeMethodBase, ) from sglang.srt.layers.quantization.utils import is_layer_skipped -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import ( cpu_has_amx_support, is_cpu, @@ -77,7 +77,9 @@ if is_flashinfer_available(): nvfp4_block_scale_interleave, trtllm_fp4_block_scale_moe, ) - from flashinfer.fused_moe.core import get_w2_permute_indices_with_cache + from flashinfer.fused_moe.core import ( + get_w2_permute_indices_with_cache, + ) # SM90 mixed-input helpers landed in FlashInfer #3084 (post-0.6.10). Older # versions don't ship them; gate at import so unrelated code paths still load. @@ -332,7 +334,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): self.use_flashinfer = get_moe_runner_backend().is_flashinfer_mxfp4() self.use_marlin = get_moe_runner_backend().is_marlin() self.flashinfer_mxfp4_moe_precision = ( - get_exec().moe.flashinfer_mxfp4_moe_precision + get_server_args().flashinfer_mxfp4_moe_precision ) # When `flashinfer_mxfp4` is enabled, dispatch to one of two FlashInfer # entry points depending on the GPU: diff --git a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py index 89cf50f74..36f79d628 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py @@ -14,7 +14,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( ) from sglang.srt.layers.dp_attention import is_allocation_symmetric from sglang.srt.layers.moe.utils import RoutingMethodType -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import ( is_flashinfer_available, log_info_on_rank0, @@ -51,7 +51,7 @@ class Mxfp4FlashinferTrtllmMoEMethod: self._fp8 = fp8_method self.prefix = prefix self.flashinfer_mxfp4_moe_precision = ( - get_exec().moe.flashinfer_mxfp4_moe_precision + get_server_args().flashinfer_mxfp4_moe_precision ) def create_moe_runner(self, layer, moe_runner_config): @@ -376,7 +376,9 @@ def maybe_fuse_routed_scale_and_shared_add( from sglang.srt.layers.quantization.mxfp4_flashinfer_cutlass_moe import ( Mxfp4FlashinferCutlassMoEMethod, ) - from sglang.srt.layers.quantization.mxfp4_marlin_moe import Mxfp4MarlinMoEMethod + from sglang.srt.layers.quantization.mxfp4_marlin_moe import ( + Mxfp4MarlinMoEMethod, + ) fused = isinstance( experts.quant_method, diff --git a/python/sglang/srt/layers/rotary_embedding/base.py b/python/sglang/srt/layers/rotary_embedding/base.py index cdb5b6366..8443bb71d 100644 --- a/python/sglang/srt/layers/rotary_embedding/base.py +++ b/python/sglang/srt/layers/rotary_embedding/base.py @@ -11,7 +11,7 @@ from sglang.srt.environ import envs from sglang.srt.layers.rotary_embedding.utils import apply_rotary_emb from sglang.srt.layers.utils import MultiPlatformOp from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import ( cpu_has_amx_support, get_bool_env_var, @@ -67,7 +67,9 @@ if _is_npu: ) if _is_hip: - from sglang.kernels.ops.attention.utils import fused_qk_rope_reshape_and_cache + from sglang.kernels.ops.attention.utils import ( + fused_qk_rope_reshape_and_cache, + ) if _is_xpu: from sgl_kernel import fused_qk_rope_with_cos_sin_cache_inplace @@ -127,7 +129,7 @@ class RotaryEmbedding(MultiPlatformOp): self._apply_rotary_emb_wrapped = apply_rotary_emb # XXX (MUSA): Implement sgl_kernel.rotary_embedding support for MUSA backend - if get_exec().deterministic.rl_on_policy_target is not None or _is_musa: + if get_server_args().rl_on_policy_target is not None or _is_musa: self._forward_method = self.forward_native self._apply_rotary_emb_wrapped = torch.compile( dynamic=True, @@ -151,7 +153,7 @@ class RotaryEmbedding(MultiPlatformOp): # create the cache on GPU for faster initialization. This may cause # a slight numerical difference between the HF implementation and ours. init_device = ( - "cpu" if get_exec().deterministic.rl_on_policy_target is not None else None + "cpu" if get_server_args().rl_on_policy_target is not None else None ) inv_freq = 1.0 / ( base @@ -162,7 +164,7 @@ class RotaryEmbedding(MultiPlatformOp): / self.rotary_dim ) ) - if get_exec().deterministic.rl_on_policy_target is not None: + if get_server_args().rl_on_policy_target is not None: inv_freq = inv_freq.cuda() return inv_freq diff --git a/python/sglang/srt/layers/rotary_embedding/mrope.py b/python/sglang/srt/layers/rotary_embedding/mrope.py index a7b674e0f..4fd88dcc1 100644 --- a/python/sglang/srt/layers/rotary_embedding/mrope.py +++ b/python/sglang/srt/layers/rotary_embedding/mrope.py @@ -18,7 +18,7 @@ from sglang.srt.layers.rotary_embedding.yarn import ( yarn_get_mscale_simple, yarn_linear_ramp_mask, ) -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import ( cpu_has_amx_support, is_cuda, @@ -42,6 +42,7 @@ if _is_xpu: from sgl_kernel import multimodal_rotary_embedding from sglang.kernels.ops.attention.mrope import apply_interleaved_rope_triton +from sglang.srt.runtime_context import get_server_args def apply_interleaved_rope(x: torch.Tensor, mrope_section: list) -> torch.Tensor: @@ -131,7 +132,7 @@ class MRotaryEmbedding(RotaryEmbedding): self.register_buffer("axis_map", axis_map, persistent=False) else: self.axis_map = None - if get_exec().deterministic.rl_on_policy_target is not None: + if get_server_args().rl_on_policy_target is not None: self._forward_method = self.forward_native def get_cos_sin_with_position(self, positions): @@ -143,7 +144,7 @@ class MRotaryEmbedding(RotaryEmbedding): last_dim = cos_sin.size()[-1] cos, sin = cos_sin.chunk(2, dim=-1) if self.mrope_interleaved: - if support_triton(get_exec().kernel.attention_backend): + if support_triton(get_server_args().attention_backend): cos = apply_interleaved_rope_triton(cos, self.mrope_section) sin = apply_interleaved_rope_triton(sin, self.mrope_section) else: diff --git a/python/sglang/srt/layers/sampler.py b/python/sglang/srt/layers/sampler.py index 97868104b..de587ec5f 100644 --- a/python/sglang/srt/layers/sampler.py +++ b/python/sglang/srt/layers/sampler.py @@ -8,21 +8,34 @@ from torch import nn from sglang.kernels.ops.sampling.murmur_hash import murmur_hash32 from sglang.srt.distributed import get_tp_group -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.logits_processor import LogitsProcessorOutput -from sglang.srt.layers.logprob_processor import OutputLogprobProcessor -from sglang.srt.runtime_context import get_exec, get_parallel, get_server_args +from sglang.srt.layers.logprob_processor import ( + OutputLogprobProcessor, +) +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo from sglang.srt.sampling.sampling_params import TOP_K_ALL from sglang.srt.utils.async_probe import sanitize_nan_logits -from sglang.srt.utils.common import get_bool_env_var, is_cuda, is_hip, is_musa, is_npu +from sglang.srt.utils.common import ( + get_bool_env_var, + is_cuda, + is_hip, + is_musa, + is_npu, +) if is_cuda(): from flashinfer.sampling import ( min_p_sampling_from_probs, top_k_top_p_sampling_from_probs, ) - from sgl_kernel import top_k_renorm_prob, top_p_renorm_prob + from sgl_kernel import ( + top_k_renorm_prob, + top_p_renorm_prob, + ) if is_musa(): from sgl_kernel import ( @@ -61,14 +74,12 @@ class Sampler(nn.Module): if is_dp_attention_enabled(): self.tp_sync_group = get_parallel().attn_tp_group.device_group - self.rl_on_policy_target = get_exec().deterministic.rl_on_policy_target + self.rl_on_policy_target = get_server_args().rl_on_policy_target # In RL on-policy mode, deterministic inference is automatically enabled. - self.enable_deterministic = ( - get_exec().deterministic.enable_deterministic_inference - ) + self.enable_deterministic = get_server_args().enable_deterministic_inference # In RL on-policy mode, we use log_softmax to compute logprobs to match the trainer. self.use_log_softmax_logprob = self.rl_on_policy_target is not None - self.use_ascend_backend = get_exec().kernel.sampling_backend == "ascend" + self.use_ascend_backend = get_server_args().sampling_backend == "ascend" self.output_logprob_processor = OutputLogprobProcessor() @@ -234,7 +245,7 @@ class Sampler(nn.Module): positions=positions, ) else: - backend = get_exec().kernel.sampling_backend + backend = get_server_args().sampling_backend if backend == "flashinfer": assert ( sampling_info.sampling_seed is None diff --git a/python/sglang/srt/layers/utils/cp_utils.py b/python/sglang/srt/layers/utils/cp_utils.py index 7f8e3e4f2..0f289f602 100644 --- a/python/sglang/srt/layers/utils/cp_utils.py +++ b/python/sglang/srt/layers/utils/cp_utils.py @@ -58,13 +58,13 @@ class ContextParallelMetadata: def is_prefill_context_parallel_enabled(): - return get_parallel().enable_prefill_context_parallel + return get_server_args().enable_prefill_context_parallel def is_prefill_cp_in_seq_split(): return ( is_prefill_context_parallel_enabled() - and get_parallel().prefill_cp_mode == "in-seq-split" + and get_server_args().prefill_cp_mode == "in-seq-split" ) diff --git a/python/sglang/srt/managers/data_parallel_controller.py b/python/sglang/srt/managers/data_parallel_controller.py index 3457dd509..c43a4fc1b 100644 --- a/python/sglang/srt/managers/data_parallel_controller.py +++ b/python/sglang/srt/managers/data_parallel_controller.py @@ -48,7 +48,6 @@ from sglang.srt.managers.scheduler import run_scheduler_process from sglang.srt.observability.cpu_monitor import start_cpu_monitor_thread from sglang.srt.observability.req_time_stats import DPControllerReqTimeStats from sglang.srt.observability.trace import process_tracing_init, trace_set_thread_info -from sglang.srt.runtime_context import get_exec from sglang.srt.server_args import ( DP_ATTENTION_HANDSHAKE_PORT_DELTA, PortArgs, @@ -232,7 +231,7 @@ class DataParallelController: sock_send(worker, obj) def update_active_ranks(self, ranks: ActiveRanksOutput): - if get_exec().moe.elastic_ep_backend is not None: + if self.server_args.elastic_ep_backend is not None: if len(ranks.status) != self.max_dp_size: logger.warning( "[Elastic EP][DPC] active rank status len=%d != max_dp_size=%d; " @@ -485,7 +484,7 @@ class DataParallelController: logger.debug("Worker port broadcast completed") return worker_ports finally: - if get_exec().moe.elastic_ep_backend is None: + if self.server_args.elastic_ep_backend is None: rep_socket.close() else: threading.Thread( @@ -816,12 +815,6 @@ def run_data_parallel_controller_process( kill_itself_when_parent_died() parent_process = psutil.Process().parent() - # Publish the resolved config at DP-controller process entry: this process - # reads config namespaces (e.g. get_exec().moe.*) in its own address space - # before spawning schedulers. - from sglang.srt.runtime_context import publish - - publish(server_args, role="scheduler") configure_logger(server_args) if server_args.enable_trace: process_tracing_init( diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index cdf99935f..891d8df6c 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -33,13 +33,7 @@ from sglang.srt.managers.schedule_batch import ( from sglang.srt.mem_cache.multimodal_cache import EmbeddingResult, MultiModalStaticCache from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.multimodal.evs import EVSEmbeddingResult -from sglang.srt.runtime_context import ( - get_disagg, - get_parallel, - get_schedule, - get_server_args, - get_serving, -) +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import flatten_nested_list, is_hip, is_npu, print_warning_once from sglang.srt.utils.stale_shm_cleanup import make_shm_name from sglang.utils import logger @@ -884,7 +878,7 @@ def _adjust_embedding_length( f"tokens from multimodal embeddings." ) if num_mm_tokens_in_input_ids < num_mm_tokens_in_embedding: - chunked_prefill_size = get_schedule().chunked_prefill_size + chunked_prefill_size = get_server_args().chunked_prefill_size if chunked_prefill_size != -1: logger.warning( "You may want to avoid this issue by raising `chunked_prefill_size`, or disabling chunked prefill" @@ -1293,7 +1287,7 @@ def general_mm_embed_routine( feature = getattr(mm_item, "feature", None) if isinstance(feature, torch.Tensor) and feature.is_cuda: mm_item.feature = feature.to("cpu", non_blocking=True) - if get_disagg().language_only: + if get_server_args().language_only: precomputed_embeddings = getattr( mm_item, "precomputed_embeddings", None ) @@ -1973,7 +1967,7 @@ def wrap_shm_features(obj): """ Scan the object for multimodal tensors and wrap them in SHM pointers. """ - if _get_is_default_transport() or get_serving().skip_tokenizer_init: + if _get_is_default_transport() or get_server_args().skip_tokenizer_init: return obj if obj.mm_inputs: @@ -2034,7 +2028,7 @@ def unwrap_shm_features(obj): Restore ShmPointerMMData wrappers back into standard torch.Tensors. Handles both single requests and batch requests. """ - if _get_is_default_transport() or get_serving().skip_tokenizer_init: + if _get_is_default_transport() or get_server_args().skip_tokenizer_init: return obj # Handle batch requests if isinstance(obj, BaseBatchReq): diff --git a/python/sglang/srt/managers/multi_tokenizer_mixin.py b/python/sglang/srt/managers/multi_tokenizer_mixin.py index 3381ad2ba..8a1ba1839 100644 --- a/python/sglang/srt/managers/multi_tokenizer_mixin.py +++ b/python/sglang/srt/managers/multi_tokenizer_mixin.py @@ -1,7 +1,5 @@ from __future__ import annotations -from sglang.srt.runtime_context import get_disagg - # Copyright 2023-2024 SGLang Team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -647,15 +645,15 @@ class TokenizerWorker(TokenizerManager): self.tokenizer_ipc_name = port_args.tokenizer_ipc_name # For PD disaggregtion - from sglang.srt.runtime_context import get_context - - get_context().override( + self.server_args.override( "tokenizer_worker.restore_disaggregation_mode", disaggregation_mode=disaggregation_mode, ) - self.disaggregation_mode = DisaggregationMode(get_disagg().disaggregation_mode) + self.disaggregation_mode = DisaggregationMode( + self.server_args.disaggregation_mode + ) self.disaggregation_transfer_backend = TransferBackend( - get_disagg().disaggregation_transfer_backend + self.server_args.disaggregation_transfer_backend ) # Register this worker with the router for pause/continue broadcasting diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 1f11d03ed..97a3dceb6 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -77,7 +77,10 @@ from sglang.srt.managers.embed_types import PositionalEmbeds from sglang.srt.managers.scheduler_components.new_token_ratio_tracker import ( NewTokenRatioTracker, ) -from sglang.srt.mem_cache.allocation import alloc_for_decode, alloc_for_extend +from sglang.srt.mem_cache.allocation import ( + alloc_for_decode, + alloc_for_extend, +) from sglang.srt.mem_cache.allocation_sizing import get_alloc_reserve_per_decode from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( @@ -102,12 +105,7 @@ from sglang.srt.observability.req_time_stats import ( DPControllerReqTimeStats, SchedulerReqTimeStats, ) -from sglang.srt.runtime_context import ( - get_parallel, - get_server_args, - get_serving, - get_spec, -) +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.server_args import ServerArgs @@ -1096,7 +1094,7 @@ class Req(ReqDllmMixin): """Check if this request is prefill-only (no token generation needed).""" # NOTE: when spec is enabled, prefill_only optimizations are disabled - spec_alg = get_spec().speculative_algorithm + spec_alg = get_server_args().speculative_algorithm return self.sampling_params.max_new_tokens == 0 and spec_alg is None @property @@ -1117,7 +1115,7 @@ class Req(ReqDllmMixin): def effective_kv_committed_len(self) -> int: # Report only the prompt prefix so thinking + answer fall into the # overallocated range and are reclaimed by release_kv_cache. #22373. - if get_serving().strip_thinking_cache and self.reasoning_tokens > 0: + if get_server_args().strip_thinking_cache and self.reasoning_tokens > 0: return min(self.kv_committed_len, len(self.origin_input_ids)) return self.kv_committed_len diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index 4cb347de4..fa929fd38 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -56,7 +56,7 @@ from sglang.srt.mem_cache.multi_ended_allocator import ( UnifiedMambaTokenToKVPoolAllocator, ) from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode -from sglang.srt.runtime_context import get_disagg +from sglang.srt.runtime_context import get_server_args from sglang.srt.server_args import ServerArgs if TYPE_CHECKING: @@ -193,7 +193,7 @@ class SchedulePolicy: if ( not isinstance(policy, CacheAwarePolicy) and self.tree_cache.supports_fast_match_prefix() - and get_disagg().disaggregation_mode != "decode" + and get_server_args().disaggregation_mode != "decode" ): for r in waiting_queue: match_prefix_for_req(self.tree_cache, r, include_req=True) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 07e77b7aa..94340105e 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -210,7 +210,9 @@ from sglang.srt.managers.scheduler_components.pool_stats_observer import ( from sglang.srt.managers.scheduler_components.profiler_manager import ( SchedulerProfilerManager, ) -from sglang.srt.managers.scheduler_components.recv_skipper import SchedulerRecvSkipper +from sglang.srt.managers.scheduler_components.recv_skipper import ( + SchedulerRecvSkipper, +) from sglang.srt.managers.scheduler_components.request_receiver import ( SchedulerRequestReceiver, ) @@ -239,20 +241,7 @@ from sglang.srt.observability.trace import process_tracing_init, trace_set_threa from sglang.srt.parser.reasoning_parser import ReasoningParser from sglang.srt.platforms import current_platform from sglang.srt.plugins import load_plugins -from sglang.srt.runtime_context import ( - get_context, - get_device, - get_disagg, - get_exec, - get_lora, - get_memory, - get_mm, - get_observability, - get_parallel, - get_schedule, - get_serving, - get_spec, -) +from sglang.srt.runtime_context import get_context, get_parallel from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo from sglang.srt.sampling.sampling_params import TOP_K_ALL from sglang.srt.server_args import PortArgs, ServerArgs @@ -454,9 +443,9 @@ class Scheduler( attn_tp_cpu_group=self.attn_tp_cpu_group, tp_cpu_group=self.tp_cpu_group, attn_cp_cpu_group=self.attn_cp_cpu_group, - enable_metrics=get_observability().enable_metrics, + enable_metrics=self.server_args.enable_metrics, enable_kv_cache_events=bool( - get_observability().kv_events_config + self.server_args.kv_events_config and self.ps.pp_rank == 0 and self.ps.attn_tp_rank == 0 and self.ps.attn_cp_rank == 0 @@ -482,8 +471,8 @@ class Scheduler( self.init_hisparse_coordinator() if ( - get_disagg().disaggregation_mode == "decode" - and get_disagg().disaggregation_decode_enable_offload_kvcache + self.server_args.disaggregation_mode == "decode" + and self.server_args.disaggregation_decode_enable_offload_kvcache ): self.decode_offload_manager = DecodeKVCacheOffloadManager( req_to_token_pool=self.req_to_token_pool, @@ -594,7 +583,7 @@ class Scheduler( self.dllm_config = ( # For diffusion LLM DllmConfig.from_server_args(self.server_args) - if get_exec().dllm.dllm_algorithm is not None + if self.server_args.dllm_algorithm is not None else None ) @@ -622,11 +611,11 @@ class Scheduler( self.ipc_channels = SchedulerIpcChannels.create( port_args=port_args, is_rank_zero=is_rank_zero, - skip_tokenizer_init=get_serving().skip_tokenizer_init, - metrics_enabled=get_observability().enable_metrics + skip_tokenizer_init=self.server_args.skip_tokenizer_init, + metrics_enabled=self.server_args.enable_metrics and ( self.ps.attn_tp_rank == 0 - or get_observability().enable_metrics_for_all_schedulers + or self.server_args.enable_metrics_for_all_schedulers ), enable_scripted_runtime=envs.SGLANG_TEST_SCRIPTED_RUNTIME.get(), ) @@ -642,7 +631,7 @@ class Scheduler( port_args, self.ps.dp_size, dp_rank, - publish_interval=get_observability().load_snapshot_publish_interval, + publish_interval=self.server_args.load_snapshot_publish_interval, ) except Exception as e: logger.warning("load snapshot writer init failed: %s", e) @@ -652,7 +641,7 @@ class Scheduler( self.ps.pp_rank == 0 and self.ps.attn_tp_rank == 0 and self.ps.attn_cp_rank == 0 - and get_device().sleep_on_idle + and self.server_args.sleep_on_idle ): self.idle_sleeper = IdleSleeper( sockets=[ @@ -723,9 +712,9 @@ class Scheduler( ) # Set reasoning_parser and think_end_id if --reasoning_parser is enabled - if get_serving().reasoning_parser and self.tokenizer: + if self.server_args.reasoning_parser and self.tokenizer: reasoning_parser = ReasoningParser( - model_type=get_serving().reasoning_parser, + model_type=self.server_args.reasoning_parser, stream_reasoning=False, tokenizer=self.tokenizer, ) @@ -796,7 +785,7 @@ class Scheduler( target_worker=self.tp_worker, ) - if get_spec().speculative_draft_load_format is not None: + if self.server_args.speculative_draft_load_format is not None: # Write the draft load_format onto server_args (not just the bag): # the draft worker is built from a copy of self.server_args and # build_load_config reads server_args.load_format, so a bag-only @@ -804,10 +793,10 @@ class Scheduler( # format. self.server_args.override( "scheduler.draft_load_format", - load_format=get_spec().speculative_draft_load_format, + load_format=self.server_args.speculative_draft_load_format, ) logger.info( - f"Using draft model load_format: '{get_spec().speculative_draft_load_format}'" + f"Using draft model load_format: '{self.server_args.speculative_draft_load_format}'" ) DraftWorkerClass = self.spec_algorithm.create_worker(self.server_args) @@ -898,7 +887,7 @@ class Scheduler( # --min-free-slots-delay. Built independently of the prefill delayer. self.min_free_slots_delayer: Optional[MinFreeSlotsDelayer] = None min_free_slots = resolve_min_free_slots( - get_schedule().min_free_slots_delay, + self.server_args.min_free_slots_delay, self.max_running_requests, is_dflash_family=self.spec_algorithm.is_dflash_family(), ) @@ -944,14 +933,14 @@ class Scheduler( if self.ps.tp_rank == 0: logger.info( f"max_total_num_tokens={self.max_total_num_tokens}, " - f"chunked_prefill_size={get_schedule().chunked_prefill_size}, " + f"chunked_prefill_size={self.server_args.chunked_prefill_size}, " f"max_prefill_tokens={self.max_prefill_tokens}, " f"max_running_requests={self.max_running_requests}, " f"context_len={self.model_config.context_len}, " f"{'available_cpu_mem' if self.device == 'cpu' else 'available_gpu_mem'}={avail_mem:.2f} GB" ) - if get_observability().enable_metrics: + if self.server_args.enable_metrics: self.metrics_collector.emit_constants( max_total_num_tokens=self.max_total_num_tokens, # TODO: max_running_requests_under_SLO has no setter — dead chain. @@ -998,7 +987,7 @@ class Scheduler( self._engine_paused = False def init_chunked_prefill(self): - self.chunked_prefill_size = get_schedule().chunked_prefill_size + self.chunked_prefill_size = self.server_args.chunked_prefill_size uses_transformers_backend = ( get_resolved_model_impl(self.model_config) == ModelImpl.TRANSFORMERS ) @@ -1018,12 +1007,13 @@ class Scheduler( self.chunked_req = None self._pending_chunked_abort_req = None self.is_mixed_chunk = ( - self.chunked_prefill_size is not None and get_schedule().enable_mixed_chunk + self.chunked_prefill_size is not None + and self.server_args.enable_mixed_chunk ) # Init the dynamic chunking predictor for PP self.enable_dynamic_chunking = ( - get_schedule().enable_dynamic_chunking and self.ps.pp_size > 1 + self.server_args.enable_dynamic_chunking and self.ps.pp_size > 1 ) if self.enable_dynamic_chunking: try: @@ -1059,8 +1049,8 @@ class Scheduler( ) self.prefill_delayer: Optional[PrefillDelayer] = None self.max_prefill_bs: int = 0 - if get_schedule().enable_prefill_delayer: - if get_disagg().disaggregation_mode == "decode": + if self.server_args.enable_prefill_delayer: + if self.server_args.disaggregation_mode == "decode": logger.info( "Ignoring --enable-prefill-delayer on decode engine " "(no prefill scheduling path; delayer would be a no-op)." @@ -1077,15 +1067,15 @@ class Scheduler( if self.metrics_reporter.enable_metrics else None ), - max_delay_passes=get_schedule().prefill_delayer_max_delay_passes, - token_usage_low_watermark=get_schedule().prefill_delayer_token_usage_low_watermark, + max_delay_passes=self.server_args.prefill_delayer_max_delay_passes, + token_usage_low_watermark=self.server_args.prefill_delayer_token_usage_low_watermark, device=self.tp_group.device, ) # NOTE: preemption is enabled by default for priority scheduling. self.enable_priority_preemption = ( self.enable_priority_scheduling - and not get_schedule().disable_priority_preemption + and not self.server_args.disable_priority_preemption ) self.new_token_ratio_tracker = NewTokenRatioTracker.from_server_args( @@ -1101,12 +1091,12 @@ class Scheduler( def init_watch_dog_memory_saver_input_blocker(self): # Start watchdog thread self.watchdog = create_scheduler_watchdog( - self, watchdog_timeout=get_device().watchdog_timeout + self, watchdog_timeout=self.server_args.watchdog_timeout ) # Init memory saver, profiler and metric stats self.memory_saver_adapter = TorchMemorySaverAdapter.create( - enable=get_exec().features.enable_memory_saver + enable=self.server_args.enable_memory_saver ) # Init recv skipper and input blocker @@ -1128,9 +1118,11 @@ class Scheduler( self.disagg_decode_prealloc_queue = None self.disagg_decode_transfer_queue = None - self.disaggregation_mode = DisaggregationMode(get_disagg().disaggregation_mode) + self.disaggregation_mode = DisaggregationMode( + self.server_args.disaggregation_mode + ) self.transfer_backend = TransferBackend( - get_disagg().disaggregation_transfer_backend + self.server_args.disaggregation_transfer_backend ) # todo: should we fix this when enabling mtp or it doesn't matter since we only enable mtp in decode node thus we don't transfer draft kvs between P and D? @@ -1198,12 +1190,12 @@ class Scheduler( gloo_group=self.attn_tp_cpu_group, tp_rank=self.ps.tp_rank, tp_size=self.ps.tp_size, - dp_size=get_parallel().dp_size, + dp_size=self.server_args.dp_size, gpu_id=self.ps.gpu_id, - bootstrap_port=get_disagg().disaggregation_bootstrap_port, + bootstrap_port=self.server_args.disaggregation_bootstrap_port, max_total_num_tokens=self.max_total_num_tokens, pp_rank=self.ps.pp_rank, - num_reserved_decode_tokens=get_disagg().num_reserved_decode_tokens, + num_reserved_decode_tokens=self.server_args.num_reserved_decode_tokens, transfer_backend=self.transfer_backend, ) @@ -1229,7 +1221,7 @@ class Scheduler( tp_rank=self.ps.tp_rank, tp_size=self.ps.tp_size, gpu_id=self.ps.gpu_id, - bootstrap_port=get_disagg().disaggregation_bootstrap_port, + bootstrap_port=self.server_args.disaggregation_bootstrap_port, gloo_group=self.attn_tp_cpu_group, max_total_num_tokens=self.max_total_num_tokens, scheduler=self, @@ -1243,10 +1235,11 @@ class Scheduler( self.enable_staging = envs.SGLANG_DISAGG_STAGING_BUFFER.get() # Init mm receiver for EPD disaggregation mode - if get_disagg().language_only and get_disagg().encoder_transfer_backend in [ - "zmq_to_scheduler", - "mooncake", - ]: + if ( + self.server_args.language_only + and self.server_args.encoder_transfer_backend + in ["zmq_to_scheduler", "mooncake"] + ): self.mm_receiver = create_mm_receiver( self.server_args, dtype=self.model_config.dtype, @@ -1327,7 +1320,7 @@ class Scheduler( def init_deterministic_inference_config(self): """Initialize deterministic inference configuration for different attention backends.""" - if not get_exec().deterministic.enable_deterministic_inference: + if not self.server_args.enable_deterministic_inference: self.truncation_align_size = None return @@ -1336,7 +1329,7 @@ class Scheduler( "triton": ("SGLANG_TRITON_PREFILL_TRUNCATION_ALIGN_SIZE", 4096), } env_var, default_size = backend_sizes.get( - get_exec().kernel.attention_backend, (None, None) + self.server_args.attention_backend, (None, None) ) self.truncation_align_size = ( get_int_env_var(env_var, default_size) if env_var else None @@ -1732,10 +1725,10 @@ class Scheduler( ) def init_lora_drainer(self) -> None: - if get_lora().lora_drain_wait_threshold > 0.0: + if self.server_args.lora_drain_wait_threshold > 0.0: self.lora_drainer = LoRADrainer( - get_lora().max_loras_per_batch, - get_lora().lora_drain_wait_threshold, + self.server_args.max_loras_per_batch, + self.server_args.lora_drain_wait_threshold, ) else: self.lora_drainer = None @@ -1837,7 +1830,7 @@ class Scheduler( def init_kv_events_publisher(self) -> None: self.kv_events_publisher = SchedulerKvEventsPublisher( - kv_events_config=get_observability().kv_events_config, + kv_events_config=self.server_args.kv_events_config, ps=self.ps, attn_tp_rank=self.ps.attn_tp_rank, attn_cp_rank=self.ps.attn_cp_rank, @@ -2013,7 +2006,7 @@ class Scheduler( return image_inputs def _get_multimodal_inputs(self, mm_inputs_dict): - if get_mm().enable_broadcast_mm_inputs_process: + if self.server_args.enable_broadcast_mm_inputs_process: return self._process_and_broadcast_mm_inputs(mm_inputs_dict) else: return MultimodalInputs.from_processor_output(mm_inputs_dict) @@ -2060,7 +2053,7 @@ class Scheduler( def _maybe_namespace_elastic_radix_cache(self, req: Req) -> None: if ( - get_exec().moe.elastic_ep_backend is None + self.server_args.elastic_ep_backend is None or self.disable_radix_cache or not self.tree_cache.is_tree_cache() ): @@ -2106,7 +2099,8 @@ class Scheduler( ) # Radix-native sessions use only the top-level session_id. radix_native_session = ( - recv_req.session_id is not None and get_memory().enable_session_radix_cache + recv_req.session_id is not None + and self.server_args.enable_session_radix_cache ) if session_id is None or radix_native_session: @@ -2118,7 +2112,7 @@ class Scheduler( if recv_req.bootstrap_port is None: # Use default bootstrap port - recv_req.bootstrap_port = get_disagg().disaggregation_bootstrap_port + recv_req.bootstrap_port = self.server_args.disaggregation_bootstrap_port req = Req( recv_req.rid, @@ -2271,7 +2265,7 @@ class Scheduler( self._add_request_to_queue(req) return - if req.return_sampling_mask and get_exec().kernel.sampling_backend == "ascend": + if req.return_sampling_mask and self.server_args.sampling_backend == "ascend": # The ascend backend samples from logits directly and never builds the # top-k/top-p support, so it cannot produce a sampling mask. error_msg = ( @@ -2320,7 +2314,7 @@ class Scheduler( error_msg = validate_input_length( req, self.max_req_input_len, - get_serving().allow_auto_truncate, + self.server_args.allow_auto_truncate, ) if error_msg: req.set_finish_with_abort(error_msg) @@ -2598,7 +2592,7 @@ class Scheduler( error_msg = validate_input_length( req, self.max_req_input_len, - get_serving().allow_auto_truncate, + self.server_args.allow_auto_truncate, ) if error_msg: self._add_request_to_queue(req) @@ -2810,7 +2804,7 @@ class Scheduler( if ( need_mlp_sync and not self.spec_algorithm.is_none() - and not get_spec().speculative_skip_dp_mlp_sync + and not self.server_args.speculative_skip_dp_mlp_sync ): # NOTE: This branch makes sure prefill and decode batches will not be mixed when spec and dp-attn is enabled. # Before merging the new batch into running batch: @@ -2884,7 +2878,7 @@ class Scheduler( for req in ready_grammar_requests: self._add_request_to_queue(req) - if self.enable_hierarchical_cache or get_memory().enable_flexkv: + if self.enable_hierarchical_cache or self.server_args.enable_flexkv: self.tree_cache.check_hicache_events() if self.enable_priority_preemption or self.is_hybrid_swa: @@ -2951,7 +2945,7 @@ class Scheduler( self.priority_scheduling_preemption_threshold, max_prefill_bs=self.max_prefill_bs, max_running_requests=self.max_running_requests, - prefill_max_requests=get_schedule().prefill_max_requests, + prefill_max_requests=self.server_args.prefill_max_requests, prefill_delayer_single_pass=prefill_delayer_single_pass, dllm_config=self.dllm_config, waiting_queue_len=len(self.waiting_queue), @@ -3522,7 +3516,7 @@ class Scheduler( def _maybe_report_active_ranks(self) -> None: if not ( - self.enable_dp_attention and get_exec().moe.elastic_ep_backend is not None + self.enable_dp_attention and self.server_args.elastic_ep_backend is not None ): return from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager @@ -3798,7 +3792,7 @@ class Scheduler( ok, msg = self.tree_cache.attach_storage_backend( storage_backend=recv_req.hicache_storage_backend, storage_backend_extra_config_json=recv_req.hicache_storage_backend_extra_config_json, - served_model_name=get_serving().served_model_name, + served_model_name=self.server_args.served_model_name, hicache_storage_prefetch_policy=recv_req.hicache_storage_prefetch_policy, hicache_write_policy=recv_req.hicache_write_policy, ) @@ -3918,7 +3912,7 @@ class Scheduler( } ret["effective_max_running_requests_per_dp"] = self.max_running_requests - if get_exec().moe.elastic_ep_backend is not None: + if self.server_args.elastic_ep_backend is not None: from sglang.srt.elastic_ep.elastic_ep import ElasticEPStateManager ret["is_scaling_elastic_ep"] = ElasticEPStateManager.is_scaling() @@ -4313,7 +4307,7 @@ class Scheduler( old_ep_size = ElasticEPStateManager.get_effective_ep_size() new_ep_size = recv_req.new_ep_size - max_ep_size = get_parallel().max_ep_size or old_ep_size + max_ep_size = self.server_args.max_ep_size or old_ep_size logger.debug( "[Elastic EP][scale] request received: new_ep_size=%d " @@ -4451,10 +4445,10 @@ class Scheduler( return None def close_session(self, recv_req: CloseSessionReqInput): - if get_memory().enable_session_radix_cache: + if self.server_args.enable_session_radix_cache: self.tree_cache.release_radix_session(recv_req.session_id) if recv_req.session_id in self.session_controller or not ( - get_memory().enable_session_radix_cache + self.server_args.enable_session_radix_cache ): self.session_controller.close(recv_req) @@ -4633,13 +4627,6 @@ def run_scheduler_process( display_dp_rank=display_dp_rank, display_moe_ep_rank=display_moe_ep_rank, ) - # Publish the resolved config at scheduler process entry so the config - # namespaces (get_serving()/get_device()/get_exec()/...) are available to - # Scheduler.__init__ and its init_* helpers, which read them before the - # model worker's own publish. ModelRunner re-publishes idempotently. - from sglang.srt.runtime_context import publish - - publish(server_args, role="scheduler") parent_process = psutil.Process().parent() # Set up tracing diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index d46197727..248a92939 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -2,7 +2,14 @@ from __future__ import annotations import logging from dataclasses import dataclass -from typing import TYPE_CHECKING, Callable, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Callable, + List, + Optional, + Tuple, + Union, +) import torch @@ -16,14 +23,11 @@ from sglang.srt.managers.schedule_batch import ( ScheduleBatch, mamba_lazy_spec_in_window, ) -from sglang.srt.mem_cache.common import maybe_cache_unfinished_req, release_kv_cache -from sglang.srt.runtime_context import ( - get_disagg, - get_exec, - get_memory, - get_observability, - get_server_args, +from sglang.srt.mem_cache.common import ( + maybe_cache_unfinished_req, + release_kv_cache, ) +from sglang.srt.runtime_context import get_server_args from sglang.srt.speculative.base_spec_worker import BaseSpecWorker from sglang.srt.state_capturer.indexer_topk import get_global_indexer_capturer from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer @@ -44,7 +48,10 @@ if TYPE_CHECKING: SchedulerOutputStreamer, ) from sglang.srt.managers.tp_worker import BaseTpWorker - from sglang.srt.managers.utils import EmbeddingBatchResult, GenerationBatchResult + from sglang.srt.managers.utils import ( + EmbeddingBatchResult, + GenerationBatchResult, + ) from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.srt.mem_cache.memory_pool import ReqToTokenPool @@ -77,7 +84,7 @@ class SchedulerBatchResultProcessor: def process_batch_result_prebuilt(self, batch: ScheduleBatch): assert self.disaggregation_mode == DisaggregationMode.DECODE - use_free_group = get_disagg().disaggregation_decode_enable_radix_cache + use_free_group = self.server_args.disaggregation_decode_enable_radix_cache if use_free_group: self.token_to_kv_pool_allocator.free_group_begin() for req in batch.reqs: @@ -85,7 +92,7 @@ class SchedulerBatchResultProcessor: req.update_finish_state() if req.finished(): req.time_stats.set_quick_finish_time() - if get_memory().enable_hisparse: + if self.server_args.enable_hisparse: self.hisparse_coordinator.request_finished(req) release_kv_cache(req, self.tree_cache) @@ -236,7 +243,7 @@ class SchedulerBatchResultProcessor: req.time_stats.set_completion_time() elif not batch.decoding_reqs or req not in batch.decoding_reqs: maybe_cache_unfinished_req(req, self.tree_cache) - if get_memory().enable_hisparse: + if self.server_args.enable_hisparse: self.hisparse_coordinator.admit_request_into_staging(req) self._maybe_collect_customized_info(i, req, logits_output) @@ -749,7 +756,7 @@ class SchedulerBatchResultProcessor: num_block_accept_tokens=result.num_block_accept_tokens, num_cap_tokens=result.num_cap_tokens, ) - if get_observability().enable_metrics: + if self.server_args.enable_metrics: self.metrics_collector.increment_decode_cuda_graph_pass( value=can_run_cuda_graph ) @@ -932,7 +939,7 @@ class SchedulerBatchResultProcessor: self._mamba_prefix_cache_update(req, batch, result, i) if ( - get_disagg().disaggregation_decode_enable_offload_kvcache + self.server_args.disaggregation_decode_enable_offload_kvcache and not req.finished() ): self.decode_offload_manager.offload_kv_cache(req) @@ -952,12 +959,12 @@ class SchedulerBatchResultProcessor: self._maybe_collect_routed_experts(req) self._maybe_collect_indexer_topk(req) - if get_disagg().disaggregation_decode_enable_offload_kvcache: + if self.server_args.disaggregation_decode_enable_offload_kvcache: # Asynchronously offload KV cache; release_kv_cache will be called after Device->Host transfer completes if not self.decode_offload_manager.offload_kv_cache(req): self.decode_offload_manager.finalize_release_on_finish(req) else: - if get_memory().enable_hisparse: + if self.server_args.enable_hisparse: self.hisparse_coordinator.request_finished(req) prepare_release = getattr( self.model_worker, "prepare_for_kv_cache_release", None @@ -1095,7 +1102,7 @@ class SchedulerBatchResultProcessor: For spec decode, the boundary is detected by comparing the accepted seq_len range against interval boundaries. """ - interval = get_exec().mamba.mamba_track_interval + interval = get_server_args().mamba_track_interval if batch.spec_algorithm.is_none(): if req.kv_committed_len % interval == 0: diff --git a/python/sglang/srt/managers/scheduler_components/dp_attn.py b/python/sglang/srt/managers/scheduler_components/dp_attn.py index 25b30c4f3..01a1d2adb 100644 --- a/python/sglang/srt/managers/scheduler_components/dp_attn.py +++ b/python/sglang/srt/managers/scheduler_components/dp_attn.py @@ -12,7 +12,9 @@ from sglang.srt.distributed.parallel_state_wrapper import ParallelState from sglang.srt.environ import envs from sglang.srt.layers.dp_attention import world_dp_gather_enabled from sglang.srt.managers.schedule_batch import ScheduleBatch -from sglang.srt.managers.scheduler_components.recv_skipper import SchedulerRecvSkipper +from sglang.srt.managers.scheduler_components.recv_skipper import ( + SchedulerRecvSkipper, +) from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.srt.mem_cache.memory_pool import ReqToTokenPool @@ -24,7 +26,6 @@ from sglang.srt.model_executor.cuda_graph_config import ( ) from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.observability.metrics_collector import DPCooperationInfo -from sglang.srt.runtime_context import get_parallel, get_schedule from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils.common import require_mlp_tp_gather @@ -377,14 +378,14 @@ class SchedulerDPAttnAdapter: def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch): return prepare_mlp_sync_batch_raw( local_batch, - dp_size=get_parallel().dp_size, + dp_size=self.server_args.dp_size, attn_tp_size=self.ps.attn_tp_size, attn_cp_size=self.ps.attn_cp_size, tp_group=self.tp_group, get_idle_batch=self.get_idle_batch, disable_cuda_graph=cuda_graph_fully_disabled(), require_mlp_tp_gather=require_mlp_tp_gather(self.server_args), - disable_overlap_schedule=get_schedule().disable_overlap_schedule, + disable_overlap_schedule=self.server_args.disable_overlap_schedule, offload_tags=self.offload_tags, dwdp=self.server_args.dwdp_size > 1, ) diff --git a/python/sglang/srt/managers/scheduler_components/load_inquirer.py b/python/sglang/srt/managers/scheduler_components/load_inquirer.py index 5652b740e..e8619a2f7 100644 --- a/python/sglang/srt/managers/scheduler_components/load_inquirer.py +++ b/python/sglang/srt/managers/scheduler_components/load_inquirer.py @@ -14,7 +14,6 @@ from sglang.srt.managers.load_snapshot import ( QueueMetrics, SpeculativeMetrics, ) -from sglang.srt.runtime_context import get_lora if TYPE_CHECKING: from sglang.srt.distributed.parallel_state_wrapper import ParallelState @@ -145,7 +144,7 @@ class SchedulerLoadInquirer: ) lora = None - if get_lora().enable_lora: + if self.server_args.enable_lora: lora = LoRAMetrics( slots_used=stats.lora_pool_slots_used, slots_total=stats.lora_pool_slots_total, diff --git a/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py b/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py index 85fb1ef59..d97d9ae80 100644 --- a/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py @@ -1,15 +1,20 @@ from __future__ import annotations from dataclasses import dataclass -from typing import List, Tuple +from typing import ( + List, + Tuple, +) import torch from sglang.srt.configs.model_config import ModelConfig from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.managers.schedule_batch import Req -from sglang.srt.runtime_context import get_exec -from sglang.srt.server_args import MIS_DELIMITER_TOKEN_ID, ServerArgs +from sglang.srt.server_args import ( + MIS_DELIMITER_TOKEN_ID, + ServerArgs, +) @dataclass(kw_only=True, slots=True, frozen=True) @@ -159,7 +164,7 @@ class SchedulerLogprobResultProcessor: delimiter token receive logprobs. """ return ( - get_exec().features.enable_mis + self.server_args.enable_mis and req.is_prefill_only and req.multi_item_delimiter_indices is not None ) diff --git a/python/sglang/srt/managers/scheduler_components/output_streamer.py b/python/sglang/srt/managers/scheduler_components/output_streamer.py index d0b82044c..278ccf428 100644 --- a/python/sglang/srt/managers/scheduler_components/output_streamer.py +++ b/python/sglang/srt/managers/scheduler_components/output_streamer.py @@ -2,7 +2,12 @@ from __future__ import annotations import logging from dataclasses import dataclass, field -from typing import Any, Callable, List, Optional +from typing import ( + Any, + Callable, + List, + Optional, +) import torch import zmq @@ -16,9 +21,11 @@ from sglang.srt.managers.io_struct import ( CachedTokensDetails, wrap_as_pickle, ) -from sglang.srt.managers.schedule_batch import BaseFinishReason, Req +from sglang.srt.managers.schedule_batch import ( + BaseFinishReason, + Req, +) from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache -from sglang.srt.runtime_context import get_observability, get_serving from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.spec_info import SpeculativeAlgorithm @@ -137,7 +144,7 @@ class SchedulerOutputStreamer: return_sampling_mask=return_sampling_mask, spec_algorithm=self.spec_algorithm, disaggregation_mode=self.disaggregation_mode, - default_stream_interval=get_serving().stream_interval, + default_stream_interval=self.server_args.stream_interval, default_force_stream_interval=DEFAULT_FORCE_STREAM_INTERVAL, get_cached_tokens_details=self.get_cached_tokens_details, ) @@ -164,7 +171,7 @@ class SchedulerOutputStreamer: if ( req.finished() and self.ps.attn_tp_rank == 0 - and get_observability().enable_request_time_stats_logging + and self.server_args.enable_request_time_stats_logging ): req.log_time_stats() diff --git a/python/sglang/srt/managers/scheduler_components/profiler_manager.py b/python/sglang/srt/managers/scheduler_components/profiler_manager.py index 190583aaa..f1c65a3b1 100644 --- a/python/sglang/srt/managers/scheduler_components/profiler_manager.py +++ b/python/sglang/srt/managers/scheduler_components/profiler_manager.py @@ -5,7 +5,13 @@ import os import time from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, List, Optional +from typing import ( + TYPE_CHECKING, + Any, + Callable, + List, + Optional, +) import torch @@ -13,7 +19,7 @@ from sglang.srt.environ import envs from sglang.srt.managers.io_struct import ProfileReq, ProfileReqOutput, ProfileReqType from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_device +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import is_mps, is_npu from sglang.srt.utils.profile_merger import ProfileMerger from sglang.srt.utils.profile_utils import ProfileManager @@ -249,7 +255,7 @@ class SchedulerProfilerManager: self.profile_in_progress = True if "CUDA_PROFILER" in activities: - if self.ps.gpu_id == get_device().base_gpu_id: + if self.ps.gpu_id == get_server_args().base_gpu_id: torch.cuda.cudart().cudaProfilerStart() self.profile_in_progress = True @@ -359,7 +365,7 @@ class SchedulerProfilerManager: torch.cuda.memory._record_memory_history(enabled=None) if "CUDA_PROFILER" in self.profiler_activities: - if self.ps.gpu_id == get_device().base_gpu_id: + if self.ps.gpu_id == get_server_args().base_gpu_id: torch.cuda.cudart().cudaProfilerStop() merge_message = self._merge_profile_traces() diff --git a/python/sglang/srt/managers/scheduler_components/request_receiver.py b/python/sglang/srt/managers/scheduler_components/request_receiver.py index 9dcfbaaed..fccd74c1d 100644 --- a/python/sglang/srt/managers/scheduler_components/request_receiver.py +++ b/python/sglang/srt/managers/scheduler_components/request_receiver.py @@ -2,7 +2,14 @@ from __future__ import annotations from dataclasses import dataclass from http import HTTPStatus -from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union +from typing import ( + TYPE_CHECKING, + Any, + Callable, + List, + Optional, + Union, +) import zmq from torch.distributed import barrier @@ -15,9 +22,14 @@ from sglang.srt.managers.io_struct import ( TokenizedGenerateReqInput, sock_recv, ) -from sglang.srt.managers.mm_utils import has_shm_features, unwrap_shm_features -from sglang.srt.runtime_context import get_disagg, get_parallel -from sglang.srt.utils import broadcast_pyobj, point_to_point_pyobj +from sglang.srt.managers.mm_utils import ( + has_shm_features, + unwrap_shm_features, +) +from sglang.srt.utils import ( + broadcast_pyobj, + point_to_point_pyobj, +) from sglang.srt.utils.nvtx_utils import scheduler_nvtx_method if TYPE_CHECKING: @@ -127,7 +139,7 @@ class SchedulerRequestReceiver: return recv_reqs def _broadcast_reqs_across_ranks(self, recv_reqs: Optional[List]) -> List: - if get_parallel().enable_dp_attention: + if self.server_args.enable_dp_attention: if self.ps.attn_tp_rank == 0 and self.ps.attn_cp_rank == 0: work_reqs, control_reqs = self._split_work_and_control_reqs(recv_reqs) else: @@ -156,7 +168,7 @@ class SchedulerRequestReceiver: # instead of the full tp_group. This avoids an expensive # all-ranks gloo sync. _local_ctrl = ( - get_parallel().enable_dp_attention_local_control_broadcast + self.server_args.enable_dp_attention_local_control_broadcast or self.server_args.is_ep_scale_joiner ) if _local_ctrl: @@ -208,8 +220,8 @@ class SchedulerRequestReceiver: # Process MM requests under EPD-disaggregation mode if ( self.ps.pp_rank == 0 - and get_disagg().language_only - and get_disagg().encoder_transfer_backend + and self.server_args.language_only + and self.server_args.encoder_transfer_backend in ["zmq_to_scheduler", "mooncake"] ): recv_reqs, abort_reqs = self.mm_receiver.process_waiting_requests(recv_reqs) @@ -233,7 +245,7 @@ class SchedulerRequestReceiver: # peer ranks may still be unpickling ShmPointerMMData # (-> shm_open). Synchronize the same CPU groups that carried # SHM-backed work requests before materialize() unlinks them. - if get_parallel().enable_dp_attention: + if self.server_args.enable_dp_attention: if self.ps.attn_tp_size > 1: barrier(group=self.attn_tp_cpu_group) if self.ps.attn_cp_size > 1: diff --git a/python/sglang/srt/managers/scheduler_pp_mixin.py b/python/sglang/srt/managers/scheduler_pp_mixin.py index 4524240ff..0a55601d4 100644 --- a/python/sglang/srt/managers/scheduler_pp_mixin.py +++ b/python/sglang/srt/managers/scheduler_pp_mixin.py @@ -36,7 +36,6 @@ from sglang.srt.model_executor.forward_batch_info import ( PPProxyTensors, ) from sglang.srt.observability.req_time_stats import set_time_batch -from sglang.srt.runtime_context import get_disagg, get_parallel from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.utils import DynamicGradMode, broadcast_pyobj, point_to_point_pyobj from sglang.srt.utils.common import get_device_module, is_xpu @@ -123,7 +122,7 @@ class SchedulerPPMixin: next_pp_outputs = None next_batch_result = None d2h_event = None - if get_parallel().pp_async_batch_depth > 0: + if self.server_args.pp_async_batch_depth > 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -139,7 +138,7 @@ class SchedulerPPMixin: self.mb_metadata, self.last_rank_comm_queue, ) - if get_parallel().pp_async_batch_depth == 0: + if self.server_args.pp_async_batch_depth == 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -269,7 +268,7 @@ class SchedulerPPMixin: server_is_idle = False pp_proxy_tensors = self._pp_recv_proxy_tensors() - if get_parallel().pp_async_batch_depth > 0: + if self.server_args.pp_async_batch_depth > 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -285,7 +284,7 @@ class SchedulerPPMixin: self.mb_metadata, self.last_rank_comm_queue, ) - if get_parallel().pp_async_batch_depth == 0: + if self.server_args.pp_async_batch_depth == 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -428,7 +427,7 @@ class SchedulerPPMixin: pp_proxy_tensors = self._pp_recv_proxy_tensors() # early send output if possible - if get_parallel().pp_async_batch_depth > 0: + if self.server_args.pp_async_batch_depth > 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -446,7 +445,7 @@ class SchedulerPPMixin: self.last_rank_comm_queue, ) - if get_parallel().pp_async_batch_depth == 0: + if self.server_args.pp_async_batch_depth == 0: next_pp_outputs, next_batch_result, d2h_event = ( self._pp_commit_send_output_work_and_preprocess_output_tensors( next_first_rank_mb_id, @@ -480,7 +479,7 @@ class SchedulerPPMixin: ) ) - if get_disagg().disaggregation_decode_enable_offload_kvcache: + if self.server_args.disaggregation_decode_enable_offload_kvcache: self.decode_offload_manager.check_offload_progress() if rmbs[next_mb_id] is not None: @@ -550,17 +549,17 @@ class SchedulerPPMixin: + len(self.disagg_decode_transfer_queue.queue) + len(self.disagg_decode_prealloc_queue.queue) ) - if get_disagg().disaggregation_decode_enable_offload_kvcache: + if self.server_args.disaggregation_decode_enable_offload_kvcache: queue_size += len(self.decode_offload_manager.ongoing_offload) if server_is_idle and queue_size == 0: self.on_idle() def init_pp_loop_state(self: Scheduler): - self.pp_loop_size: int = self.ps.pp_size + get_parallel().pp_async_batch_depth + self.pp_loop_size: int = self.ps.pp_size + self.server_args.pp_async_batch_depth # In CP mode, attention weights are duplicated, eliminating the need for the attention TP all-gather operation. self.require_attn_tp_allgather = ( - not get_parallel().enable_dsa_prefill_context_parallel + not self.server_args.enable_dsa_prefill_context_parallel ) self.mbs = [None] * self.pp_loop_size self.last_mbs = [None] * self.pp_loop_size diff --git a/python/sglang/srt/managers/tokenizer_control_mixin.py b/python/sglang/srt/managers/tokenizer_control_mixin.py index 47069bb76..86b7b378f 100644 --- a/python/sglang/srt/managers/tokenizer_control_mixin.py +++ b/python/sglang/srt/managers/tokenizer_control_mixin.py @@ -74,7 +74,6 @@ from sglang.srt.managers.io_struct import ( UpdateWeightsFromTensorReqOutput, ) from sglang.srt.managers.load_snapshot import LoadSnapshot -from sglang.srt.runtime_context import get_lora, get_parallel from sglang.srt.server_args import LoRARef, ServerArgs from sglang.srt.utils import ( get_bool_env_var, @@ -146,8 +145,8 @@ class TokenizerControlMixin: def update_control_communicator_fan_out(self: TokenizerManager, worker_count: int): primary_group_control = ( - get_parallel().enable_dp_attention - and not get_parallel().enable_dp_attention_local_control_broadcast + self.server_args.enable_dp_attention + and not self.server_args.enable_dp_attention_local_control_broadcast ) if primary_group_control: control_fan_out = ( @@ -397,7 +396,7 @@ class TokenizerControlMixin: ) -> Tuple[bool, str]: self.auto_create_handle_loop() assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + self.server_args.dp_size == 1 or self.server_args.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for update weights from distributed" results = await self.init_weights_update_group_communicator(obj) @@ -410,7 +409,7 @@ class TokenizerControlMixin: ) -> Tuple[bool, str]: self.auto_create_handle_loop() assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + self.server_args.dp_size == 1 or self.server_args.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for destroy parameter update group" results = await self.destroy_weights_update_group_communicator(obj) @@ -423,7 +422,7 @@ class TokenizerControlMixin: ) -> Tuple[bool, str]: self.auto_create_handle_loop() assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + self.server_args.dp_size == 1 or self.server_args.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for update weights from distributed" if obj.abort_all_requests: @@ -454,7 +453,7 @@ class TokenizerControlMixin: self.auto_create_handle_loop() # TODO: support DP assert ( - get_parallel().dp_size == 1 + self.server_args.dp_size == 1 ), "dp_size must be 1 for init_weights_send_group_for_remote_instance" result = ( await self.init_weights_send_group_for_remote_instance_communicator(obj) @@ -469,7 +468,7 @@ class TokenizerControlMixin: self.auto_create_handle_loop() # TODO: support DP assert ( - get_parallel().dp_size == 1 + self.server_args.dp_size == 1 ), "dp_size must be 1 for send_weights_to_remote_instance" result = (await self.send_weights_to_remote_instance_communicator(obj))[0] return result.success, result.message @@ -481,7 +480,7 @@ class TokenizerControlMixin: ) -> Tuple[bool, str]: self.auto_create_handle_loop() assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + self.server_args.dp_size == 1 or self.server_args.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for update weights from tensor" if obj.abort_all_requests: @@ -517,7 +516,7 @@ class TokenizerControlMixin: try: # For now, we only support single data parallel instance assert ( - get_parallel().dp_size == 1 or get_parallel().enable_dp_attention + self.server_args.dp_size == 1 or self.server_args.enable_dp_attention ), "dp_size must be 1 or dp attention must be enabled for update weights from IPC" logger.info("Starting IPC weight update") @@ -570,7 +569,7 @@ class TokenizerControlMixin: self.auto_create_handle_loop() try: - if not get_lora().enable_lora: + if not self.server_args.enable_lora: raise ValueError( "LoRA is not enabled. Please set `--enable-lora` to enable LoRA." ) @@ -578,7 +577,7 @@ class TokenizerControlMixin: # TODO (lifuhuang): Remove this after we verify that dynamic lora loading works # with dp_size > 1. assert ( - get_parallel().dp_size == 1 + self.server_args.dp_size == 1 ), "dp_size must be 1 for dynamic lora loading" logger.info( "Start load Lora adapter. Lora name=%s, path=%s", @@ -603,10 +602,10 @@ class TokenizerControlMixin: await self.lora_registry.register(new_adapter) self.lora_ref_cache[obj.lora_name] = new_adapter - if get_lora().max_loaded_loras is not None: + if self.server_args.max_loaded_loras is not None: while ( self.lora_registry.num_registered_loras - > get_lora().max_loaded_loras + > self.server_args.max_loaded_loras ): lru_lora_name = await self.lora_registry.lru_lora_name( exclude_pinned=True @@ -620,7 +619,7 @@ class TokenizerControlMixin: logger.info( f"Unloading least recently used LoRA adapter '{lru_lora_name}' " f"(current number of adapters: {self.lora_registry.num_registered_loras}, " - f"max allowed: {get_lora().max_loaded_loras})" + f"max allowed: {self.server_args.max_loaded_loras})" ) unload_result = await self._unload_lora_adapter_locked( @@ -648,13 +647,13 @@ class TokenizerControlMixin: self.auto_create_handle_loop() try: - if not get_lora().enable_lora: + if not self.server_args.enable_lora: raise ValueError( "LoRA is not enabled. Please set `--enable-lora` to enable LoRA." ) assert ( - get_parallel().dp_size == 1 + self.server_args.dp_size == 1 ), "dp_size must be 1 for dynamic lora loading" logger.info( "Start load Lora adapter from tensors. Lora name=%s", @@ -673,10 +672,10 @@ class TokenizerControlMixin: if result.success: await self.lora_registry.register(new_adapter) self.lora_ref_cache[obj.lora_name] = new_adapter - if get_lora().max_loaded_loras is not None: + if self.server_args.max_loaded_loras is not None: while ( self.lora_registry.num_registered_loras - > get_lora().max_loaded_loras + > self.server_args.max_loaded_loras ): lru_lora_name = await self.lora_registry.lru_lora_name( exclude_pinned=True @@ -690,7 +689,7 @@ class TokenizerControlMixin: logger.info( f"Unloading least recently used LoRA adapter '{lru_lora_name}' " f"(current number of adapters: {self.lora_registry.num_registered_loras}, " - f"max allowed: {get_lora().max_loaded_loras})" + f"max allowed: {self.server_args.max_loaded_loras})" ) unload_result = await self._unload_lora_adapter_locked( @@ -718,7 +717,7 @@ class TokenizerControlMixin: self.auto_create_handle_loop() try: - if not get_lora().enable_lora: + if not self.server_args.enable_lora: raise ValueError( "LoRA is not enabled. Please set `--enable-lora` to enable LoRA." ) @@ -730,7 +729,7 @@ class TokenizerControlMixin: # TODO (lifuhuang): Remove this after we verify that dynamic lora loading works # with dp_size > 1. assert ( - get_parallel().dp_size == 1 + self.server_args.dp_size == 1 ), "dp_size must be 1 for dynamic lora loading" logger.info( "Start unload Lora adapter. Lora name=%s", @@ -750,7 +749,7 @@ class TokenizerControlMixin: self.auto_create_handle_loop() results = await self.get_weights_by_name_communicator(obj) all_parameters = [r.parameter for r in results] - if get_parallel().dp_size == 1: + if self.server_args.dp_size == 1: return all_parameters[0] else: return all_parameters @@ -894,8 +893,6 @@ class TokenizerControlMixin: ) -> None: """Update weight version if provided.""" if weight_version is not None: - from sglang.srt.runtime_context import get_context - - get_context().override( + self.server_args.override( "tokenizer.weight_version", weight_version=weight_version ) diff --git a/python/sglang/srt/managers/tokenizer_manager.py b/python/sglang/srt/managers/tokenizer_manager.py index 4d12e3fe5..60b35ea02 100644 --- a/python/sglang/srt/managers/tokenizer_manager.py +++ b/python/sglang/srt/managers/tokenizer_manager.py @@ -110,15 +110,6 @@ from sglang.srt.observability.request_metrics_exporter import ( RequestMetricsExporterManager, ) from sglang.srt.observability.trace import SpanAttributes, extract_trace_headers -from sglang.srt.runtime_context import ( - get_device, - get_disagg, - get_lora, - get_model, - get_observability, - get_parallel, - get_serving, -) from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.server_args import ( PortArgs, @@ -472,10 +463,10 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # TODO: Refactor and organize the log export code. # Request logging self.request_logger = RequestLogger( - log_requests=get_observability().log_requests, - log_requests_level=get_observability().log_requests_level, - log_requests_format=get_observability().log_requests_format, - log_requests_target=get_observability().log_requests_target, + log_requests=self.server_args.log_requests, + log_requests_level=self.server_args.log_requests_level, + log_requests_format=self.server_args.log_requests_format, + log_requests_target=self.server_args.log_requests_target, ) # Dumping @@ -498,7 +489,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): def init_weight_update(self): # Initial weights status self.initial_weights_loaded = True - if get_model().checkpoint_engine_wait_weights_before_ready: + if self.server_args.checkpoint_engine_wait_weights_before_ready: self.initial_weights_loaded = False # Weight updates @@ -518,7 +509,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # The registry dynamically updates as adapters are loaded / unloaded during runtime. It # serves as the source of truth for available adapters and maps user-friendly LoRA names # to internally used unique LoRA IDs. - self.lora_registry = LoRARegistry(get_lora().lora_paths) + self.lora_registry = LoRARegistry(self.server_args.lora_paths) # Lock to serialize LoRA update operations. # Please note that, unlike `model_update_lock`, this does not block inference, allowing # LoRA updates and inference to overlap. @@ -527,13 +518,15 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # point to their latest LoRARef objects, so that they can be # dynamically loaded if needed for inference self.lora_ref_cache: Dict[str, LoRARef] = {} - if get_lora().lora_paths is not None: - for lora_ref in get_lora().lora_paths: + if self.server_args.lora_paths is not None: + for lora_ref in self.server_args.lora_paths: self.lora_ref_cache[lora_ref.lora_name] = lora_ref def init_disaggregation(self): # PD Disaggregation - self.disaggregation_mode = DisaggregationMode(get_disagg().disaggregation_mode) + self.disaggregation_mode = DisaggregationMode( + self.server_args.disaggregation_mode + ) # Keep a reference so the bootstrap server is not garbage-collected. self.bootstrap_server = start_disagg_service(self.server_args) # Single-source counter for auto-assigning fake bootstrap_room. @@ -542,16 +535,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # Encoder Disaggregation self.encoder_bootstrap_server = None if self.server_args.language_only: - from sglang.srt.disaggregation.encode_receiver import EncoderBootstrapServer + from sglang.srt.disaggregation.encode_receiver import ( + EncoderBootstrapServer, + ) # Shared mutable URL list: the bootstrap server appends / removes # entries as encoders register, the receiver reads from the same # list. Pre-populated with static --encoder-urls so the legacy # CLI flag still works (alongside dynamic registrations). - self.encoder_urls: List[str] = list(get_disagg().encoder_urls) + self.encoder_urls: List[str] = list(self.server_args.encoder_urls) self.encoder_bootstrap_server = EncoderBootstrapServer( - host=get_serving().host, - port=get_disagg().encoder_bootstrap_port, + host=self.server_args.host, + port=self.server_args.encoder_bootstrap_port, urls=self.encoder_urls, ) self.mm_receiver = create_mm_receiver( @@ -565,22 +560,20 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # Metrics if self.enable_metrics: engine_type = DisaggregationMode.to_engine_type( - get_disagg().disaggregation_mode + self.server_args.disaggregation_mode ) labels = { - "model_name": get_serving().served_model_name, + "model_name": self.server_args.served_model_name, "engine_type": engine_type, } if self.enable_priority_scheduling: labels["priority"] = "" - if get_observability().tokenizer_metrics_allowed_custom_labels: - for ( - label - ) in get_observability().tokenizer_metrics_allowed_custom_labels: + if self.server_args.tokenizer_metrics_allowed_custom_labels: + for label in self.server_args.tokenizer_metrics_allowed_custom_labels: labels[label] = "" - if get_observability().extra_metric_labels: - labels.update(get_observability().extra_metric_labels) + if self.server_args.extra_metric_labels: + labels.update(self.server_args.extra_metric_labels) tokenizer_collector_cls = resolve_collector_class( self.server_args, STAT_LOGGER_ROLE_TOKENIZER, @@ -589,18 +582,18 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): self.metrics_collector = tokenizer_collector_cls( server_args=self.server_args, labels=labels, - bucket_time_to_first_token=get_observability().bucket_time_to_first_token, - bucket_e2e_request_latency=get_observability().bucket_e2e_request_latency, - bucket_inter_token_latency=get_observability().bucket_inter_token_latency, + bucket_time_to_first_token=self.server_args.bucket_time_to_first_token, + bucket_e2e_request_latency=self.server_args.bucket_e2e_request_latency, + bucket_inter_token_latency=self.server_args.bucket_inter_token_latency, ) start_cpu_monitor_thread("tokenizer") - if get_observability().gc_warning_threshold_secs > 0.0: - configure_gc_warning(get_observability().gc_warning_threshold_secs) + if self.server_args.gc_warning_threshold_secs > 0.0: + configure_gc_warning(self.server_args.gc_warning_threshold_secs) self.soft_watchdog = Watchdog.create( debug_name="TokenizerManager", - watchdog_timeout=get_device().soft_watchdog_timeout, + watchdog_timeout=self.server_args.soft_watchdog_timeout, soft=True, test_stuck_time=envs.SGLANG_TEST_STUCK_TOKENIZER.get(), ) @@ -1366,7 +1359,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): return batch_size > 0 and ( self.server_args.enable_tokenizer_batch_encode or ( - (not get_parallel().enable_dp_attention) + (not self.server_args.enable_dp_attention) and (not self._batch_has_text(batch_size, requests)) ) ) @@ -1764,7 +1757,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): # default the load format to the server_args if obj.load_format is None: - obj.load_format = get_model().load_format + obj.load_format = self.server_args.load_format logger.info("Start update_weights. Load format=%s", obj.load_format) if obj.abort_all_requests: @@ -1790,9 +1783,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): def _update_model_path_info(self, model_path: str, load_format: str): self.served_model_name = model_path - from sglang.srt.runtime_context import get_context - - get_context().override( + self.server_args.override( "tokenizer.update_weights", model_path=model_path, load_format=load_format ) self.model_path = model_path @@ -1936,7 +1927,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): "id": rid, "finish_reason": recv_obj.finished_reasons[i], "prompt_tokens": recv_obj.prompt_tokens[i], - "weight_version": get_serving().weight_version, + "weight_version": self.server_args.weight_version, "num_retractions": recv_obj.retraction_counts[i], } @@ -2810,7 +2801,7 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin): meta_info = { "id": recv_obj.rid, "finish_reason": finish_reason, - "weight_version": get_serving().weight_version, + "weight_version": self.server_args.weight_version, "e2e_latency": state.time_stats.get_e2e_latency(), } is_stream = getattr(state.obj, "stream", False) diff --git a/python/sglang/srt/managers/tokenizer_manager_score_mixin.py b/python/sglang/srt/managers/tokenizer_manager_score_mixin.py index 4a1b11110..6d5475517 100644 --- a/python/sglang/srt/managers/tokenizer_manager_score_mixin.py +++ b/python/sglang/srt/managers/tokenizer_manager_score_mixin.py @@ -597,10 +597,7 @@ class TokenizerManagerScoreMixin: f"Token ID {token_id} is out of vocabulary (vocab size: {vocab_size})" ) - # Check if multi-item scoring is enabled. enable_mis is a static startup - # feature flag (never overridden post-publish), and score_request is also - # exercised on a bare mixin without a published context, so read it off - # server_args rather than the resolved-config bag. + # Check if multi-item scoring is enabled use_multi_item_scoring = self.server_args.enable_mis input_ids = None diff --git a/python/sglang/srt/managers/tp_worker.py b/python/sglang/srt/managers/tp_worker.py index 4234ab41c..6d09e8133 100644 --- a/python/sglang/srt/managers/tp_worker.py +++ b/python/sglang/srt/managers/tp_worker.py @@ -47,7 +47,6 @@ from sglang.srt.model_executor.forward_batch_info import ( PPProxyTensors, ) from sglang.srt.model_executor.pool_configurator import MemoryPoolConfig -from sglang.srt.runtime_context import get_exec, get_model, get_schedule, get_spec from sglang.srt.server_args import ServerArgs from sglang.srt.utils import MultiprocessingSerializer, broadcast_pyobj, set_random_seed from sglang.srt.utils.hf_transformers_utils import ( @@ -406,14 +405,14 @@ class TpModelWorker(BaseTpWorker): self.model_config = ModelConfig.from_server_args( self.server_args, model_path=( - get_model().model_path + self.server_args.model_path if not self.is_draft_worker - else get_spec().speculative_draft_model_path + else self.server_args.speculative_draft_model_path ), model_revision=( - get_model().revision + self.server_args.revision if not self.is_draft_worker - else get_spec().speculative_draft_model_revision + else self.server_args.speculative_draft_model_revision ), is_draft_model=self.is_draft_worker, context_length=self.context_length, @@ -424,7 +423,7 @@ class TpModelWorker(BaseTpWorker): self._model_runner = ModelRunner( model_config=self.model_config, - mem_fraction_static=get_schedule().mem_fraction_static, + mem_fraction_static=self.server_args.mem_fraction_static, gpu_id=self.gpu_id, ps=self.ps, nccl_port=self.nccl_port, @@ -440,11 +439,11 @@ class TpModelWorker(BaseTpWorker): from sglang.srt.model_executor.model_runner import ModelRunner self.model_runner_list.append(self.model_runner) - for i in range(1, get_spec().speculative_num_steps): + for i in range(1, self.server_args.speculative_num_steps): self.model_runner_list.append( ModelRunner( model_config=self.model_config, - mem_fraction_static=get_schedule().mem_fraction_static, + mem_fraction_static=self.server_args.mem_fraction_static, gpu_id=self.gpu_id, ps=self.ps, nccl_port=self.nccl_port, @@ -460,7 +459,7 @@ class TpModelWorker(BaseTpWorker): def _init_dllm_algorithm(self): from sglang.srt.dllm.algorithm.base import DllmAlgorithm - if get_exec().dllm.dllm_algorithm is not None: + if self.server_args.dllm_algorithm is not None: self.dllm_algorithm = DllmAlgorithm.from_server_args(self.server_args) else: self.dllm_algorithm = None @@ -486,9 +485,9 @@ class TpModelWorker(BaseTpWorker): ) return ( self.model_runner.max_total_num_tokens, - get_schedule().max_prefill_tokens, + self.server_args.max_prefill_tokens, self.model_runner.max_running_requests, - get_schedule().max_queued_requests, + self.server_args.max_queued_requests, max_req_len, max_req_len - 5, self.random_seed, diff --git a/python/sglang/srt/mem_cache/allocation.py b/python/sglang/srt/mem_cache/allocation.py index 99a8e1f02..3d6c8cf86 100644 --- a/python/sglang/srt/mem_cache/allocation.py +++ b/python/sglang/srt/mem_cache/allocation.py @@ -26,7 +26,7 @@ from sglang.srt.mem_cache.common import ( evict_from_tree_cache, ) from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool -from sglang.srt.runtime_context import get_exec, get_server_args +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import ( is_cpu, is_cuda, @@ -65,7 +65,7 @@ def write_cache_indices( prefix_tensors: list[torch.Tensor], req_to_token_pool: ReqToTokenPool, ): - if support_triton(get_exec().kernel.attention_backend): + if support_triton(get_server_args().attention_backend): prefix_pointers = torch.tensor( [t.data_ptr() for t in prefix_tensors], dtype=torch.uint64, @@ -106,7 +106,7 @@ def get_last_loc( req_pool_indices_tensor: torch.Tensor, prefix_lens_tensor: torch.Tensor, ) -> torch.Tensor: - attn_backend = get_exec().kernel.attention_backend + attn_backend = get_server_args().attention_backend uses_triton_dispatch = attn_backend not in ("ascend", "torch_native") if _is_hip and uses_triton_dispatch: diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 91cb8615e..7a3e1c39d 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -16,7 +16,7 @@ from sglang.srt.hardware_backend.npu.dsv4.dsv4_common_hooks import ( from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache, EvictParams from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool -from sglang.srt.runtime_context import get_server_args, get_serving +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils.common import ceil_align if TYPE_CHECKING: @@ -183,7 +183,7 @@ def _release_overallocated_kv_indices( # strip_thinking_cache intentionally reports output tokens as overallocated # so they fall into the free path below (#22373). - if spec_algo is None and not get_serving().strip_thinking_cache: + if spec_algo is None and not global_server_args.strip_thinking_cache: assert ( start_p == end_p ), f"Unexpected overallocated KV cache, {req.kv_committed_len=}, {req.kv.kv_allocated_len=}" diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index 9382534e3..14c9e52cb 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -21,7 +21,7 @@ from sglang.srt.environ import envs from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool from sglang.srt.mem_cache.deepseek_v4_compress_state import CompressStatePool from sglang.srt.mem_cache.memory_pool import KVCache -from sglang.srt.runtime_context import get_exec, get_server_args +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import ceil_div, is_hip logger = logging.getLogger(__name__) @@ -276,7 +276,7 @@ class DeepSeekV4IndexerPool(KVCache): end_layer, ) self.index_head_dim = index_head_dim - self.use_fp4_indexer = get_exec().kernel.enable_deepseek_v4_fp4_indexer + self.use_fp4_indexer = get_server_args().enable_deepseek_v4_fp4_indexer self._create_buffer() diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 542e2c628..cd6b55ec4 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -58,15 +58,7 @@ from sglang.srt.mem_cache.memory_pool import ( ) from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import ( - get_disagg, - get_exec, - get_memory, - get_model, - get_parallel, - get_schedule, - get_spec, -) +from sglang.srt.runtime_context import get_model, get_parallel from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils.common import ( @@ -123,7 +115,9 @@ if TYPE_CHECKING: from sglang.srt.model_executor.model_runner_components.spec_aux_hidden_state import ( SpecAuxHiddenStateConfig, ) - from sglang.srt.model_executor.pool_configurator import MemoryPoolConfig + from sglang.srt.model_executor.pool_configurator import ( + MemoryPoolConfig, + ) class KVCacheConfigResult(msgspec.Struct, frozen=True, kw_only=True): @@ -314,8 +308,8 @@ class KVCacheConfigurator: # from one byte buffer, then return. Gated to the target worker # (req_to_token_pool is None); supports hybrid Mamba and hybrid SWA (not DSV4). if ( - get_memory().enable_unified_memory - and get_disagg().disaggregation_mode == "null" + self.server_args.enable_unified_memory + and self.server_args.disaggregation_mode == "null" and req_to_token_pool is None ): if self.mambaish_config is not None: @@ -364,13 +358,13 @@ class KVCacheConfigurator: # TARGET_VERIFY, so their pools skip the per-step intermediate # (SpeculativeState) buffers only the target pool consumes. req_to_token_pool = req_to_token_pool.clone_with_new_mamba( - mamba_size=get_schedule().max_mamba_cache_size, + mamba_size=self.server_args.max_mamba_cache_size, mamba_spec_state_size=sizes.max_running_requests, cache_params=self.mambaish_config.mamba2_cache_params, device=self.device, enable_mamba_extra_buffer=self.server_args.enable_mamba_extra_buffer(), draft_model_idx=self.draft_model_idx, - speculative_eagle_topk=get_spec().speculative_eagle_topk, + speculative_eagle_topk=self.server_args.speculative_eagle_topk, ) # Initialize token_to_kv_pool @@ -400,7 +394,7 @@ class KVCacheConfigurator: # unsupported pool families before allocation. Keep this guard here so # future pool-selection refactors fail at boot instead of on first use. if ( - get_schedule().prefill_only_disable_kv_cache + self.server_args.prefill_only_disable_kv_cache and not self.is_draft_worker and not isinstance(token_to_kv_pool, NoOpMHATokenToKVPool) ): @@ -438,8 +432,8 @@ class KVCacheConfigurator: assert self.page_size >= 1, f"page_size must be >= 1, got {self.page_size}" # Mirror the non-shared path's extra_max_context_len computation. extra_max_context_len = 4 - if get_spec().speculative_num_draft_tokens is not None: - extra_max_context_len += get_spec().speculative_num_draft_tokens + if self.server_args.speculative_num_draft_tokens is not None: + extra_max_context_len += self.server_args.speculative_num_draft_tokens mamba_layer_ids = [ i @@ -468,14 +462,14 @@ class KVCacheConfigurator: model_context_len=self.model_config.context_len, extra_max_context_len=extra_max_context_len, max_total_num_tokens=max_total_num_tokens, - max_mamba_cache_size=get_schedule().max_mamba_cache_size, + max_mamba_cache_size=self.server_args.max_mamba_cache_size, max_num_reqs=max_num_reqs, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, enable_mamba_extra_buffer=self.server_args.enable_mamba_extra_buffer(), - speculative_num_draft_tokens=get_spec().speculative_num_draft_tokens, - disable_overlap_schedule=get_schedule().disable_overlap_schedule, - need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"), - mamba_full_memory_ratio=get_schedule().mamba_full_memory_ratio, + speculative_num_draft_tokens=self.server_args.speculative_num_draft_tokens, + disable_overlap_schedule=self.server_args.disable_overlap_schedule, + need_sort=self.server_args.disaggregation_mode in ("decode", "prefill"), + mamba_full_memory_ratio=self.server_args.mamba_full_memory_ratio, # Overlap mode: the allocator's `free` drops a wait_stream(forward_stream) # barrier so eager compaction serializes after the in-flight forward's # v2p/KV reads. Near-no-op in normal mode. @@ -508,13 +502,13 @@ class KVCacheConfigurator: ), "unified memory pool does not support MLA-SWA hybrid yet" # Mirror the non-shared path's extra_max_context_len computation. extra_max_context_len = 4 - if get_spec().speculative_num_draft_tokens is not None: - extra_max_context_len += get_spec().speculative_num_draft_tokens + if self.server_args.speculative_num_draft_tokens is not None: + extra_max_context_len += self.server_args.speculative_num_draft_tokens req_to_token_pool = ReqToTokenPool( size=max_num_reqs, max_context_len=self.model_config.context_len + extra_max_context_len, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, ) head_num = self.model_config.get_num_kv_heads(get_parallel().attn_tp_size) @@ -564,8 +558,8 @@ class KVCacheConfigurator: full_attention_layer_ids=full_attention_layer_ids, full_max_total_num_tokens=full_max_total_num_tokens, swa_max_total_num_tokens=swa_max_total_num_tokens, - enable_memory_saver=get_exec().features.enable_memory_saver, - need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"), + enable_memory_saver=self.server_args.enable_memory_saver, + need_sort=self.server_args.disaggregation_mode in ("decode", "prefill"), # Overlap mode: same wait_stream(forward_stream) rationale as # `_init_unified_mamba_pools`. forward_stream=self.forward_stream, @@ -585,7 +579,7 @@ class KVCacheConfigurator: is_dsv4_model: bool, current_platform, ): - if not get_schedule().prefill_only_disable_kv_cache or self.is_draft_worker: + if not self.server_args.prefill_only_disable_kv_cache or self.is_draft_worker: return unsupported_pool_family = None @@ -594,7 +588,7 @@ class KVCacheConfigurator: elif current_platform.is_out_of_tree() and not self.mambaish_config: unsupported_pool_family = "out-of-tree platform KV pool" elif ( - get_exec().kernel.attention_backend == "ascend" and not self.mambaish_config + self.server_args.attention_backend == "ascend" and not self.mambaish_config ): unsupported_pool_family = "NPU/Ascend KV pool" elif self.use_mla_backend and is_dsa_model: @@ -620,9 +614,9 @@ class KVCacheConfigurator: def _build_req_to_token_pool(self, *, max_num_reqs: int) -> ReqToTokenPool: extra_max_context_len = get_req_to_token_extra_context_len(self.server_args) - if get_disagg().disaggregation_mode == "decode": + if self.server_args.disaggregation_mode == "decode": # Extra slots for pre-allocated requests - pre_alloc_size = get_disagg().disaggregation_decode_extra_slots + pre_alloc_size = self.server_args.disaggregation_decode_extra_slots if self.mambaish_config: req_to_token_pool = self._build_hybrid_mamba_decode_req_pool( max_num_reqs=max_num_reqs, @@ -654,13 +648,15 @@ class KVCacheConfigurator: extra_max_context_len: int, pre_alloc_size: int, ) -> ReqToTokenPool: - from sglang.srt.disaggregation.decode import HybridMambaDecodeReqToTokenPool + from sglang.srt.disaggregation.decode import ( + HybridMambaDecodeReqToTokenPool, + ) req_to_token_pool = HybridMambaDecodeReqToTokenPool( size=max_num_reqs, max_context_len=self.model_config.context_len + extra_max_context_len, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, cache_params=self.mambaish_config.mamba2_cache_params, mamba_layer_ids=( [ @@ -670,11 +666,11 @@ class KVCacheConfigurator: ] ), speculative_num_draft_tokens=self.server_args.max_speculative_num_draft_tokens, - speculative_eagle_topk=get_spec().speculative_eagle_topk, + speculative_eagle_topk=self.server_args.speculative_eagle_topk, enable_mamba_extra_buffer=self.server_args.enable_mamba_extra_buffer(), pre_alloc_size=pre_alloc_size, - enable_overlap_schedule=not get_schedule().disable_overlap_schedule, - mamba_size=get_schedule().max_mamba_cache_size, + enable_overlap_schedule=not self.server_args.disable_overlap_schedule, + mamba_size=self.server_args.max_mamba_cache_size, start_layer=self.layer_info.start_layer, ) return req_to_token_pool @@ -692,7 +688,7 @@ class KVCacheConfigurator: size=max_num_reqs, max_context_len=self.model_config.context_len + extra_max_context_len, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, pre_alloc_size=pre_alloc_size, ) return req_to_token_pool @@ -705,11 +701,11 @@ class KVCacheConfigurator: ) -> ReqToTokenPool: req_to_token_pool = HybridReqToTokenPool( size=max_num_reqs, - mamba_size=get_schedule().max_mamba_cache_size, + mamba_size=self.server_args.max_mamba_cache_size, mamba_spec_state_size=max_num_reqs, max_context_len=self.model_config.context_len + extra_max_context_len, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, cache_params=self.mambaish_config.mamba2_cache_params, mamba_layer_ids=( [ @@ -721,18 +717,18 @@ class KVCacheConfigurator: enable_mamba_extra_buffer=self.server_args.enable_mamba_extra_buffer(), enable_mamba_extra_buffer_lazy=self.server_args.enable_mamba_extra_buffer_lazy(), speculative_num_draft_tokens=self.server_args.max_speculative_num_draft_tokens, - speculative_eagle_topk=get_spec().speculative_eagle_topk, - enable_overlap_schedule=not get_schedule().disable_overlap_schedule, + speculative_eagle_topk=self.server_args.speculative_eagle_topk, + enable_overlap_schedule=not self.server_args.disable_overlap_schedule, start_layer=self.layer_info.start_layer, - enable_linear_replayssm=get_exec().mamba.enable_linear_replayssm, - linear_replayssm_cache_len=get_exec().mamba.linear_replayssm_cache_len, - mamba_envelope_layout=get_memory().enable_page_major_kv_layout, + enable_linear_replayssm=self.server_args.enable_linear_replayssm, + linear_replayssm_cache_len=self.server_args.linear_replayssm_cache_len, + mamba_envelope_layout=self.server_args.enable_page_major_kv_layout, # ReplaySSM spec-verify is GDN-only: activate the pool machinery # (rings + cursors + the intermediate_ssm gate) only for GDN-hybrid # models, so any other mamba-ish model (Mamba2/Nemotron, lightning, # ...) run with the flag set stays byte-identical to flag-off. enable_gdn_replayssm_spec=( - get_exec().mamba.enable_gdn_replayssm_spec + self.server_args.enable_gdn_replayssm_spec and self.hybrid_gdn_config is not None ), ) @@ -758,7 +754,7 @@ class KVCacheConfigurator: size=max_num_reqs, max_context_len=self.model_config.context_len + extra_max_context_len, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, ) return req_to_token_pool @@ -774,7 +770,7 @@ class KVCacheConfigurator: # selected by swapping in the PageMajorMHATokenToKVPool subclass. The # default keeps upstream's per-layer layout. The Mamba state pool is routed # separately via `mamba_envelope_layout` on the req-to-token pool above. - enable_page_major = get_memory().enable_page_major_kv_layout + enable_page_major = self.server_args.enable_page_major_kv_layout mha_pool_class = ( PageMajorMHATokenToKVPool if enable_page_major else MHATokenToKVPool ) @@ -806,7 +802,7 @@ class KVCacheConfigurator: max_total_num_tokens=sizes.max_total_num_tokens, ) elif ( - get_exec().kernel.attention_backend == "ascend" and not self.mambaish_config + self.server_args.attention_backend == "ascend" and not self.mambaish_config ): if self.is_hybrid_swa: token_to_kv_pool = self._build_ascend_swa_kv_pool( @@ -882,12 +878,14 @@ class KVCacheConfigurator: c128_state_dtype: Optional[torch.dtype], req_to_token_pool: ReqToTokenPool, ) -> KVCache: - swa_page_size = get_schedule().page_size + swa_page_size = self.server_args.page_size if not _is_npu: assert swa_page_size == 256, "In paged swa mode, page_size must be 256." if self.is_draft_worker: - from sglang.srt.models.deepseek_v4_nextn import COMPRESS_RATIO_NEXTN_LAYER + from sglang.srt.models.deepseek_v4_nextn import ( + COMPRESS_RATIO_NEXTN_LAYER, + ) compression_ratios = [ COMPRESS_RATIO_NEXTN_LAYER @@ -914,12 +912,12 @@ class KVCacheConfigurator: # sliding eviction in ``ScheduleBatch._evict_swa``. c4_state_pool_size = npu_state_pool_size( ratio=4, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, max_num_reqs=max_running_requests, ) c128_state_pool_size = npu_state_pool_size( ratio=128, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, max_num_reqs=max_running_requests, ) else: @@ -937,7 +935,7 @@ class KVCacheConfigurator: c128_size=c128_max_total_num_tokens, c4_state_pool_size=c4_state_pool_size, c128_state_pool_size=c128_state_pool_size, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, swa_page_size=swa_page_size, sliding_window=self.model_config.window_size, dtype=self.kv_cache_dtype, @@ -948,11 +946,11 @@ class KVCacheConfigurator: indexer_head_dim=self.model_config.index_head_dim, layer_num=self.layer_info.num_effective_layers, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, compression_ratios=compression_ratios, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, - enable_hisparse=get_memory().enable_hisparse, + enable_hisparse=self.server_args.enable_hisparse, online_mtp_max_draft_tokens=( self.server_args.max_speculative_num_draft_tokens or 0 ), @@ -963,7 +961,7 @@ class KVCacheConfigurator: PoolCls = current_platform.get_dsa_kv_pool_cls() token_to_kv_pool = PoolCls( max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, kv_lora_rank=self.model_config.kv_lora_rank, qk_rope_head_dim=self.model_config.qk_rope_head_dim, @@ -974,7 +972,7 @@ class KVCacheConfigurator: kv_cache_dtype=self.kv_cache_dtype, server_args=self.server_args, ), - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, index_head_dim=get_dsa_index_head_dim(self.model_config.hf_config), @@ -987,14 +985,14 @@ class KVCacheConfigurator: PoolCls = current_platform.get_mla_kv_pool_cls() token_to_kv_pool = PoolCls( max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, kv_lora_rank=self.model_config.kv_lora_rank, qk_rope_head_dim=self.model_config.qk_rope_head_dim, index_head_dim=(self.model_config.index_head_dim if is_dsa_model else None), layer_num=self.layer_info.num_effective_layers, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, ) @@ -1004,13 +1002,13 @@ class KVCacheConfigurator: PoolCls = current_platform.get_mha_kv_pool_cls() token_to_kv_pool = PoolCls( max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, head_num=self.model_config.get_num_kv_heads(get_parallel().attn_tp_size), head_dim=self.model_config.head_dim, layer_num=self.layer_info.num_effective_layers, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, ) @@ -1022,7 +1020,9 @@ class KVCacheConfigurator: full_max_total_num_tokens: Optional[int], swa_max_total_num_tokens: Optional[int], ) -> KVCache: - from sglang.srt.hardware_backend.npu.memory_pool_npu import NPUMHATokenToKVPool + from sglang.srt.hardware_backend.npu.memory_pool_npu import ( + NPUMHATokenToKVPool, + ) kwargs = {} if self.is_hybrid_swa_compress: @@ -1039,7 +1039,7 @@ class KVCacheConfigurator: token_to_kv_pool = SWAKVPool( size=full_max_total_num_tokens, size_swa=swa_max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, post_capture_active=self.post_capture_kv_active, head_num=self.model_config.get_num_kv_heads(get_parallel().attn_tp_size), @@ -1055,35 +1055,39 @@ class KVCacheConfigurator: def _build_ascend_mla_kv_pool( self, *, max_total_num_tokens: int, is_dsa_model: bool ) -> KVCache: - from sglang.srt.hardware_backend.npu.memory_pool_npu import NPUMLATokenToKVPool + from sglang.srt.hardware_backend.npu.memory_pool_npu import ( + NPUMLATokenToKVPool, + ) token_to_kv_pool = NPUMLATokenToKVPool( max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, kv_lora_rank=self.model_config.kv_lora_rank, qk_rope_head_dim=self.model_config.qk_rope_head_dim, index_head_dim=(self.model_config.index_head_dim if is_dsa_model else None), layer_num=self.layer_info.num_effective_layers, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, ) return token_to_kv_pool def _build_ascend_mha_kv_pool(self, *, max_total_num_tokens: int) -> KVCache: - from sglang.srt.hardware_backend.npu.memory_pool_npu import NPUMHATokenToKVPool + from sglang.srt.hardware_backend.npu.memory_pool_npu import ( + NPUMHATokenToKVPool, + ) token_to_kv_pool = NPUMHATokenToKVPool( max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, head_num=self.model_config.get_num_kv_heads(get_parallel().attn_tp_size), head_dim=self.model_config.head_dim, layer_num=self.layer_info.num_effective_layers, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, ) @@ -1097,7 +1101,7 @@ class KVCacheConfigurator: dsa_cp_layer_shard_size, ) = get_glm_dsa_cp_layer_shard_info(self) pool_kwargs = {} - if get_memory().enable_hisparse: + if self.server_args.enable_hisparse: PoolCls = HiSparseDSATokenToKVPool from sglang.srt.mem_cache.sparsity import parse_hisparse_config @@ -1117,7 +1121,7 @@ class KVCacheConfigurator: PoolCls = DSATokenToKVPool token_to_kv_pool = PoolCls( max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, kv_lora_rank=self.model_config.kv_lora_rank, qk_rope_head_dim=self.model_config.qk_rope_head_dim, @@ -1128,7 +1132,7 @@ class KVCacheConfigurator: kv_cache_dtype=self.kv_cache_dtype, server_args=self.server_args, ), - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, index_head_dim=get_dsa_index_head_dim(self.model_config.hf_config), @@ -1139,13 +1143,13 @@ class KVCacheConfigurator: def _build_mla_fp4_kv_pool(self, *, max_total_num_tokens: int) -> KVCache: token_to_kv_pool = MLATokenToKVPoolFP4( max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, kv_lora_rank=self.model_config.kv_lora_rank, qk_rope_head_dim=self.model_config.qk_rope_head_dim, layer_num=self.layer_info.num_effective_layers, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, ) @@ -1154,13 +1158,13 @@ class KVCacheConfigurator: def _build_mla_kv_pool(self, *, max_total_num_tokens: int) -> KVCache: token_to_kv_pool = MLATokenToKVPool( max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, kv_lora_rank=self.model_config.kv_lora_rank, qk_rope_head_dim=self.model_config.qk_rope_head_dim, layer_num=self.layer_info.num_effective_layers, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, ) @@ -1217,7 +1221,7 @@ class KVCacheConfigurator: token_to_kv_pool = SWAKVPool( size=full_max_total_num_tokens, size_swa=size_swa, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, post_capture_active=self.post_capture_kv_active, head_num=self.model_config.get_num_kv_heads(get_parallel().attn_tp_size), @@ -1225,7 +1229,7 @@ class KVCacheConfigurator: swa_attention_layer_ids=swa_attention_layer_ids, full_attention_layer_ids=full_attention_layer_ids, device=self.device, - enable_kv_cache_copy=(get_spec().speculative_algorithm is not None), + enable_kv_cache_copy=(self.server_args.speculative_algorithm is not None), token_to_kv_pool_class=swa_pool_class, **kwargs, ) @@ -1240,7 +1244,7 @@ class KVCacheConfigurator: ) token_to_kv_pool = MiniMaxSparseKVPool( size=max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, index_dtype=self.model_dtype, head_num=self.model_config.get_num_kv_heads(get_parallel().attn_tp_size), @@ -1250,7 +1254,7 @@ class KVCacheConfigurator: sparse_layer_ids=sparse_layer_ids, disable_value_sparse_layer_ids=disable_value_sparse_layer_ids, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, ) @@ -1289,7 +1293,7 @@ class KVCacheConfigurator: else mha_pool_class ) token_to_kv_pool = HybridLinearKVPool( - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, size=max_total_num_tokens, dtype=self.kv_cache_dtype, head_num=self.model_config.get_num_kv_heads(get_parallel().attn_tp_size), @@ -1298,8 +1302,8 @@ class KVCacheConfigurator: full_attention_layer_ids=full_attention_layer_ids, device=self.device, mamba_pool=req_to_token_pool.mamba_pool, - enable_memory_saver=get_exec().features.enable_memory_saver, - enable_kv_cache_copy=(get_spec().speculative_algorithm is not None), + enable_memory_saver=self.server_args.enable_memory_saver, + enable_kv_cache_copy=(self.server_args.speculative_algorithm is not None), use_mla=self.use_mla_backend, start_layer=self.layer_info.start_layer, full_kv_pool_class=full_pool_class, @@ -1312,18 +1316,18 @@ class KVCacheConfigurator: def _build_mha_fp4_kv_pool(self, *, max_total_num_tokens: int) -> KVCache: token_to_kv_pool = MHATokenToKVPoolFP4( max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, head_num=self.model_config.get_num_kv_heads(get_parallel().attn_tp_size), head_dim=self.model_config.head_dim, v_head_dim=self.model_config.v_head_dim, layer_num=self.layer_info.num_effective_layers, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, - enable_alt_stream=not get_disagg().enable_pdmux, - enable_kv_cache_copy=(get_spec().speculative_algorithm is not None), + enable_alt_stream=not self.server_args.enable_pdmux, + enable_kv_cache_copy=(self.server_args.speculative_algorithm is not None), ) return token_to_kv_pool @@ -1335,7 +1339,7 @@ class KVCacheConfigurator: else: pool_cls = ( NoOpMHATokenToKVPool - if get_schedule().prefill_only_disable_kv_cache + if self.server_args.prefill_only_disable_kv_cache else mha_pool_class ) pool_kwargs = {} @@ -1345,18 +1349,18 @@ class KVCacheConfigurator: pool_kwargs["post_capture_active"] = self.post_capture_kv_active token_to_kv_pool = pool_cls( max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, head_num=self.model_config.get_num_kv_heads(get_parallel().attn_tp_size), head_dim=self.model_config.head_dim, v_head_dim=self.model_config.v_head_dim, layer_num=self.layer_info.num_effective_layers, device=self.device, - enable_memory_saver=get_exec().features.enable_memory_saver, + enable_memory_saver=self.server_args.enable_memory_saver, start_layer=self.layer_info.start_layer, end_layer=self.layer_info.end_layer, - enable_alt_stream=not get_disagg().enable_pdmux, - enable_kv_cache_copy=(get_spec().speculative_algorithm is not None), + enable_alt_stream=not self.server_args.enable_pdmux, + enable_kv_cache_copy=(self.server_args.speculative_algorithm is not None), **pool_kwargs, ) return token_to_kv_pool @@ -1371,20 +1375,20 @@ class KVCacheConfigurator: token_to_kv_pool_allocator: Optional[BaseTokenToKVPoolAllocator], ) -> BaseTokenToKVPoolAllocator: # Initialize token_to_kv_pool_allocator - need_sort = get_disagg().disaggregation_mode in ("decode", "prefill") + need_sort = self.server_args.disaggregation_mode in ("decode", "prefill") if token_to_kv_pool_allocator is None: if current_platform.is_out_of_tree(): AllocatorCls = current_platform.get_paged_allocator_cls() token_to_kv_pool_allocator = AllocatorCls( sizes.max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, device=self.device, kvcache=token_to_kv_pool, need_sort=need_sort, ) elif _is_npu and ( - get_exec().kernel.attention_backend == "ascend" + self.server_args.attention_backend == "ascend" or is_dsv4_model or self.hybrid_gdn_config is not None ): @@ -1402,7 +1406,7 @@ class KVCacheConfigurator: token_to_kv_pool_allocator = swa_allocator_cls( sizes.full_max_total_num_tokens, sizes.swa_max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, device=self.device, kvcache=token_to_kv_pool, @@ -1415,7 +1419,7 @@ class KVCacheConfigurator: token_to_kv_pool_allocator = NPUPagedTokenToKVPoolAllocator( sizes.max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, device=self.device, kvcache=token_to_kv_pool, @@ -1425,7 +1429,7 @@ class KVCacheConfigurator: if self.is_hybrid_swa and sizes.full_max_total_num_tokens == 0: token_to_kv_pool_allocator = PureSWATokenToKVPoolAllocator( sizes.swa_max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, device=self.device, kvcache=token_to_kv_pool, @@ -1435,20 +1439,22 @@ class KVCacheConfigurator: token_to_kv_pool_allocator = SWATokenToKVPoolAllocator( sizes.full_max_total_num_tokens, sizes.swa_max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, device=self.device, kvcache=token_to_kv_pool, need_sort=need_sort, ) else: - if get_memory().enable_hisparse: - from sglang.srt.mem_cache.sparsity import parse_hisparse_config + if self.server_args.enable_hisparse: + from sglang.srt.mem_cache.sparsity import ( + parse_hisparse_config, + ) hisparse_cfg = parse_hisparse_config(self.server_args) token_to_kv_pool_allocator = HiSparseTokenToKVPoolAllocator( sizes.max_total_num_tokens, - page_size=get_schedule().page_size, + page_size=self.server_args.page_size, dtype=self.kv_cache_dtype, device=self.device, kvcache=token_to_kv_pool, @@ -1456,7 +1462,8 @@ class KVCacheConfigurator: host_to_device_ratio=hisparse_cfg.host_to_device_ratio, ) elif ( - get_schedule().page_size == 1 and self.server_args.dcp_size == 1 + self.server_args.page_size == 1 + and self.server_args.dcp_size == 1 ): token_to_kv_pool_allocator = TokenToKVPoolAllocator( sizes.max_total_num_tokens, @@ -1468,7 +1475,7 @@ class KVCacheConfigurator: else: token_to_kv_pool_allocator = PagedTokenToKVPoolAllocator( sizes.max_total_num_tokens * self.server_args.dcp_size, - page_size=get_schedule().page_size + page_size=self.server_args.page_size * self.server_args.dcp_size, dtype=self.kv_cache_dtype, device=self.device, @@ -1476,7 +1483,7 @@ class KVCacheConfigurator: need_sort=need_sort, ) - if get_memory().enable_hisparse and is_dsv4_model: + if self.server_args.enable_hisparse and is_dsv4_model: assert self.is_hybrid_swa, "DeepSeek V4 HiSparse requires SWA mode." token_to_kv_pool_allocator = DeepSeekV4HiSparseTokenToKVPoolAllocator( token_to_kv_pool_allocator @@ -1528,7 +1535,7 @@ class KVCacheConfigurator: cpu_group=get_world_group().cpu_group, ) - slack_gb = pre_model_load_memory * (1 - get_schedule().mem_fraction_static) + slack_gb = pre_model_load_memory * (1 - self.server_args.mem_fraction_static) if self.mambaish_config is not None and self.post_capture_kv_active: # Mamba state is a fixed pre-capture allocation, so it can't ride the ~0 post-capture slack. slack_gb = max( @@ -1552,7 +1559,7 @@ class KVCacheConfigurator: ) raise ValueError( f"Loaded weights leave no GPU memory for the KV cache under " - f"--mem-fraction-static={get_schedule().mem_fraction_static}. " + f"--mem-fraction-static={self.server_args.mem_fraction_static}. " f"Raise --mem-fraction-static above " f"{suggested_mem_fraction_static:.3f} " f"(minimum viable = 1 - available/pre = " @@ -1563,14 +1570,14 @@ class KVCacheConfigurator: return int(rest_memory * (1 << 30)) # return in bytes def _calculate_mamba_ratio(self) -> int: - if get_memory().disable_radix_cache: + if self.server_args.disable_radix_cache: return 1 additional_ratio = 0 if self.server_args.enable_mamba_extra_buffer(): # ping-pong buffer size is 2 when overlap schedule is on, 1 otherwise. # Lazy mode saves 1 slot (2 → 1) for overlap; non-overlap already uses 1. - if not get_schedule().disable_overlap_schedule: + if not self.server_args.disable_overlap_schedule: if self.server_args.enable_mamba_extra_buffer_lazy(): additional_ratio = MAMBA_CACHE_V2_ADDITIONAL_RATIO_OVERLAP_LAZY else: @@ -1589,7 +1596,7 @@ class KVCacheConfigurator: Page alignment is handled by the configurator, not here. If constraints change the value, the configurator re-runs and re-aligns. """ - user_limit = get_schedule().max_total_tokens + user_limit = self.server_args.max_total_tokens # Apply user-specified upper bound if user_limit is not None: @@ -1619,7 +1626,7 @@ class KVCacheConfigurator: estimated = int(token_capacity / self.model_config.context_len * 512) estimated = max(min(estimated, 4096), 2048) - max_num_reqs = get_schedule().max_running_requests + max_num_reqs = self.server_args.max_running_requests if max_num_reqs is not None: requested_per_worker = max_num_reqs // self.ps.attn_dp_size max_num_reqs = min(requested_per_worker, token_capacity // 2) @@ -1630,13 +1637,13 @@ class KVCacheConfigurator: if self.mambaish_config is not None: ratio = self._calculate_mamba_ratio() max_num_reqs = min( - max_num_reqs, get_schedule().max_mamba_cache_size // ratio + max_num_reqs, self.server_args.max_mamba_cache_size // ratio ) if max_num_reqs <= 0: raise RuntimeError( f"Hybrid (mamba/linear-attention) state cache is too small to serve " - f"any requests. max_mamba_cache_size={get_schedule().max_mamba_cache_size}, " + f"any requests. max_mamba_cache_size={self.server_args.max_mamba_cache_size}, " f"mamba_ratio={ratio}, resulting max_num_reqs={max_num_reqs}. " f"Try: (1) reduce --max-running-requests, " f"(2) increase --mem-fraction-static, or " @@ -1666,7 +1673,7 @@ class KVCacheConfigurator: ) configurator = create_memory_pool_configurator(self) config = configurator.finalize_with_max_running_requests(config) - config.mem_fraction_static = get_schedule().mem_fraction_static + config.mem_fraction_static = self.server_args.mem_fraction_static return config def config_from_budget( @@ -1682,20 +1689,18 @@ class KVCacheConfigurator: configurator = create_memory_pool_configurator(self) config = configurator.calculate_pool_sizes( - budget_bytes, get_schedule().page_size + budget_bytes, self.server_args.page_size ) max_tokens = self._apply_token_constraints(config.max_total_num_tokens) if cap_tokens is not None: max_tokens = min(max_tokens, cap_tokens) if max_tokens != config.max_total_num_tokens: config = configurator.calculate_pool_sizes_from_max_tokens( - max_tokens, get_schedule().page_size + max_tokens, self.server_args.page_size ) return config def _handle_max_mamba_cache(self, total_rest_memory): - from sglang.srt.runtime_context import get_context - config = self.mambaish_config server_args = self.server_args assert config is not None @@ -1705,11 +1710,11 @@ class KVCacheConfigurator: assert server_args.speculative_num_draft_tokens is not None assert server_args.max_running_requests is not None - if get_schedule().max_mamba_cache_size is not None: + if server_args.max_mamba_cache_size is not None: # Use explicitly set max_mamba_cache_size - get_context().override( + server_args.override( "mamba_pool.per_dp_shard", - max_mamba_cache_size=get_schedule().max_mamba_cache_size + max_mamba_cache_size=server_args.max_mamba_cache_size // self.ps.attn_dp_size, ) # Reserve intermediate memory based on capped max_num_reqs @@ -1717,7 +1722,7 @@ class KVCacheConfigurator: ratio = self._calculate_mamba_ratio() capped_reqs = min( server_args.max_running_requests // self.ps.attn_dp_size, - get_schedule().max_mamba_cache_size // ratio, + server_args.max_mamba_cache_size // ratio, ) intermediate_size = ( config.mamba2_cache_params.mamba_cache_per_req @@ -1730,7 +1735,7 @@ class KVCacheConfigurator: and server_args.max_running_requests is not None ): # Use explicitly set max_running_requests when radix cache is disabled - get_context().override( + server_args.override( "mamba_pool.from_max_running_requests", max_mamba_cache_size=server_args.max_running_requests // self.ps.attn_dp_size, @@ -1739,7 +1744,7 @@ class KVCacheConfigurator: if has_spec_dec: intermediate_size = ( config.mamba2_cache_params.mamba_cache_per_req - * get_schedule().max_mamba_cache_size + * server_args.max_mamba_cache_size * server_args.speculative_num_draft_tokens ) total_rest_memory = total_rest_memory - (intermediate_size / (1 << 30)) @@ -1764,7 +1769,7 @@ class KVCacheConfigurator: ratio = self._calculate_mamba_ratio() D = server_args.speculative_num_draft_tokens # Joint solve: main_state + intermediate = mamba_budget - get_context().override( + server_args.override( "mamba_pool.memory_budget_spec", max_mamba_cache_size=int( mamba_budget_bytes // (per_req * (1 + D / ratio)) @@ -1774,12 +1779,12 @@ class KVCacheConfigurator: # so the return value only has main_state subtracted from total capped_reqs = min( server_args.max_running_requests // self.ps.attn_dp_size, - get_schedule().max_mamba_cache_size // ratio, + server_args.max_mamba_cache_size // ratio, ) intermediate_size = per_req * capped_reqs * D total_rest_memory = total_rest_memory - (intermediate_size / (1 << 30)) else: - get_context().override( + server_args.override( "mamba_pool.memory_budget", max_mamba_cache_size=int(mamba_budget_bytes // per_req), ) @@ -1788,10 +1793,10 @@ class KVCacheConfigurator: # A non-positive value means GPU memory is insufficient for the requested # configuration. Fail fast with actionable advice instead of silently # producing garbled output at runtime. - if get_schedule().max_mamba_cache_size <= 0: + if server_args.max_mamba_cache_size <= 0: raise RuntimeError( f"Not enough GPU memory for hybrid (mamba/linear-attention) state cache. " - f"Computed max_mamba_cache_size={get_schedule().max_mamba_cache_size} " + f"Computed max_mamba_cache_size={server_args.max_mamba_cache_size} " f"(total_rest_memory={total_rest_memory:.2f} GB, " f"mamba_cache_per_req={config.mamba2_cache_params.mamba_cache_per_req / (1 << 20):.2f} MB). " f"Try: (1) reduce --max-running-requests, " @@ -1801,7 +1806,7 @@ class KVCacheConfigurator: ) mamba_state_memory = ( - get_schedule().max_mamba_cache_size + server_args.max_mamba_cache_size * config.mamba2_cache_params.mamba_cache_per_req / (1 << 30) ) diff --git a/python/sglang/srt/mem_cache/storage/lmcache/lmc_radix_cache.py b/python/sglang/srt/mem_cache/storage/lmcache/lmc_radix_cache.py index 8460a6d62..897a5e61b 100644 --- a/python/sglang/srt/mem_cache/storage/lmcache/lmc_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/lmcache/lmc_radix_cache.py @@ -16,7 +16,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( MatchResult, ) from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode -from sglang.srt.runtime_context import get_memory, get_server_args +from sglang.srt.runtime_context import get_server_args try: from lmcache.integration.sglang.multi_process_adapter import LMCacheMPConnector @@ -108,7 +108,7 @@ class LMCRadixCache(RadixCache): ): super().__init__(params) - cli_lmc_cfg = get_memory().lmcache_config_file or "" + cli_lmc_cfg = get_server_args().lmcache_config_file or "" kvcache = self.token_to_kv_pool_allocator.get_kvcache() connector_kwargs = dict( diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index a5f7bce03..7e7508cc5 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -51,8 +51,13 @@ from sglang.srt.layers.dp_attention import ( from sglang.srt.model_executor.forward_batch_deepseek_mha_mixin import ( ForwardBatchDeepSeekMHAMixin, ) -from sglang.srt.runtime_context import get_exec, get_parallel -from sglang.srt.utils import is_cuda, is_hip, is_npu, support_triton +from sglang.srt.runtime_context import get_parallel, get_server_args +from sglang.srt.utils import ( + is_cuda, + is_hip, + is_npu, + support_triton, +) from sglang.srt.utils.common import ceil_align, is_pin_memory_available if TYPE_CHECKING: @@ -936,7 +941,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # --enable-mis: every request must carry delimiter indices (the score # endpoint always produces MIS-structured requests; consumers index # without None-checking). - if get_exec().features.enable_mis and any( + if get_server_args().enable_mis and any( r.multi_item_delimiter_indices is not None for r in batch.reqs ): assert all( @@ -1105,7 +1110,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): # batch_size * [3 * seq_len] batch_size = self.seq_lens_cpu.shape[0] mrope_positions_list = [[]] * batch_size - rl_on_policy_target = get_exec().deterministic.rl_on_policy_target + rl_on_policy_target = get_server_args().rl_on_policy_target for batch_idx in range(batch_size): mm_input = batch.multimodal_inputs[batch_idx] if self.forward_mode.is_decode(): diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 91344dc45..fdf2e69bd 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -26,7 +26,11 @@ import torch import torch.distributed as dist from sglang.srt.configs.load_config import LoadConfig -from sglang.srt.configs.model_config import AttentionArch, ModelConfig, ModelImpl +from sglang.srt.configs.model_config import ( + AttentionArch, + ModelConfig, + ModelImpl, +) from sglang.srt.configs.update_config import adjust_config_with_unaligned_cpu_tp from sglang.srt.debug_utils.dumper import dumper from sglang.srt.distributed import bootstrap @@ -70,7 +74,9 @@ from sglang.srt.kv_canary.runner.canary_manager import context_tuple from sglang.srt.kv_canary.token_oracle.install import install_token_oracle_from_env from sglang.srt.layers import deep_gemm_wrapper, model_parallel from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp -from sglang.srt.layers.cp.utils import get_cp_strategy +from sglang.srt.layers.cp.utils import ( + get_cp_strategy, +) from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.layers.sampler import create_sampler from sglang.srt.layers.torchao_utils import apply_torchao_config_to_model @@ -80,10 +86,17 @@ from sglang.srt.lora.lora_registry import LoRARef from sglang.srt.managers.schedule_batch import sanity_check_mm_pad_shift_value from sglang.srt.mem_cache import kv_cache_dtype from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator -from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator +from sglang.srt.mem_cache.kv_cache_configurator import ( + KVCacheConfigurator, +) from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool -from sglang.srt.model_executor.cuda_graph_config import cuda_graph_fully_disabled -from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors +from sglang.srt.model_executor.cuda_graph_config import ( + cuda_graph_fully_disabled, +) +from sglang.srt.model_executor.forward_batch_info import ( + ForwardBatch, + PPProxyTensors, +) from sglang.srt.model_executor.forward_context import ( ForwardContext, forward_context, @@ -142,16 +155,14 @@ from sglang.srt.model_executor.model_runner_components.weight_updater import ( WeightUpdater, ) from sglang.srt.model_executor.pool_configurator import MemoryPoolConfig -from sglang.srt.model_executor.runner import EagerRunner, get_batch_sizes_to_capture +from sglang.srt.model_executor.runner import ( + EagerRunner, + get_batch_sizes_to_capture, +) from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import ( - get_device, - get_exec, get_global_dwdp_manager, - get_lora, - get_model, - get_parallel, - get_schedule, + get_server_args, set_global_dwdp_manager, ) from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo @@ -308,7 +319,7 @@ class ModelRunner: self.init_threads_binding() # Set float32 matmul precision - if get_exec().features.enable_tf32_matmul: + if get_server_args().enable_tf32_matmul: torch.set_float32_matmul_precision("high") # Set device early so that TransferEngine init (e.g. Ascend NPU) @@ -385,20 +396,20 @@ class ModelRunner: def _initialize_elastic_ep_joiner(self) -> None: if not ( - get_exec().moe.elastic_ep_backend is not None + self.server_args.elastic_ep_backend is not None and self.server_args.is_ep_joiner ): return - is_scale_join = get_exec().moe.ep_join_mode == "scale" + is_scale_join = self.server_args.ep_join_mode == "scale" if is_scale_join: join_effective_ep_size = ( - get_parallel().ep_join_rank_offset + self.ps.tp_size + self.server_args.ep_join_rank_offset + self.ps.tp_size ) dist.barrier(group=self.tp_group.cpu_group) if self.ps.tp_rank == 0: register_scale_cohort( - get_parallel().ep_join_rank_offset, + self.server_args.ep_join_rank_offset, join_effective_ep_size, ) join_scale_process_group() @@ -408,7 +419,7 @@ class ModelRunner: else: join_process_groups() - global_ep_rank = self.ps.tp_rank + get_parallel().ep_join_rank_offset + global_ep_rank = self.ps.tp_rank + self.server_args.ep_join_rank_offset broadcast_global_expert_location_metadata( model_config=self.model_config, moe_ep_rank=global_ep_rank, @@ -442,9 +453,9 @@ class ModelRunner: new_dp_size=join_effective_ep_size, new_dp_rank=global_ep_rank, ) - from sglang.srt.runtime_context import get_context - - get_context().override("elastic_ep.scale_join", dp_size=join_effective_ep_size) + self.server_args.override( + "elastic_ep.scale_join", dp_size=join_effective_ep_size + ) if self.eplb_manager is not None: self.eplb_manager.disable_rebalance( "EPLB rebalance is disabled after elastic EP scale-up" @@ -473,7 +484,7 @@ class ModelRunner: device=self.device, gpu_id=self.gpu_id, model_config=self.model_config, - custom_weight_loaders=get_model().custom_weight_loader, + custom_weight_loaders=self.server_args.custom_weight_loader, get_model=lambda: self.model, update_model_fields=self.update_model_fields, recapture_cuda_graph=self.init_decode_cuda_graph, @@ -550,7 +561,7 @@ class ModelRunner: def init_mindspore_runner(self): # Init the mindspore runner # for now, there is only some communication initialization work - if get_model().model_impl.lower() == ModelImpl.MINDSPORE and _is_npu: + if self.server_args.model_impl.lower() == ModelImpl.MINDSPORE and _is_npu: from sglang.srt.model_executor.mindspore_runner import init_ms_distributed init_ms_distributed( @@ -607,7 +618,7 @@ class ModelRunner: def init_memory_saver_adapter(self): self.memory_saver_adapter = TorchMemorySaverAdapter.create( - enable=get_exec().features.enable_memory_saver + enable=self.server_args.enable_memory_saver ) def maybe_init_remote_instance_transfer_engine(self): @@ -618,7 +629,7 @@ class ModelRunner: if self.is_draft_worker: return expert_rank = self.ps.moe_ep_rank + ( - get_parallel().ep_join_rank_offset + self.server_args.ep_join_rank_offset if self.server_args.is_ep_scale_joiner else 0 ) @@ -643,7 +654,7 @@ class ModelRunner: ) def maybe_init_lplb_solvers(self): - if get_exec().moe.ep_dispatch_algorithm == "lp" and not self.is_draft_worker: + if self.server_args.ep_dispatch_algorithm == "lp" and not self.is_draft_worker: init_lplb_solvers(model_config=self.model_config) def maybe_init_eplb_manager(self): @@ -657,12 +668,12 @@ class ModelRunner: get_expert_backup_client=lambda: self.expert_backup_client, get_weight_updater=lambda: self.weight_updater, ) - if get_exec().moe.enable_eplb and (not self.is_draft_worker) + if self.server_args.enable_eplb and (not self.is_draft_worker) else None ) def maybe_init_elastic_ep(self): - if get_exec().moe.elastic_ep_backend: + if self.server_args.elastic_ep_backend: ElasticEPStateManager.init(self.server_args) def init_token_oracle(self): @@ -681,8 +692,8 @@ class ModelRunner: get_model=lambda: self.model, ) if ( - get_exec().moe.enable_elastic_expert_backup - and get_exec().moe.elastic_ep_backend is not None + self.server_args.enable_elastic_expert_backup + and self.server_args.elastic_ep_backend is not None ) else None ) @@ -691,17 +702,17 @@ class ModelRunner: # In layered loading, torchao may have been applied torchao_applied = getattr(self.model, "torchao_applied", False) if not torchao_applied: - apply_torchao_config_to_model(self.model, get_exec().graph.torchao_config) + apply_torchao_config_to_model(self.model, get_server_args().torchao_config) supports_torch_tp = getattr(self.model, "supports_torch_tp", False) if self.ps.tp_size > 1 and supports_torch_tp: self.apply_torch_tp() def maybe_init_lora_manager(self): - if get_lora().enable_lora: + if self.server_args.enable_lora: self.init_lora_manager() def maybe_enable_batch_invariant_mode(self): - if get_exec().deterministic.enable_deterministic_inference: + if self.server_args.enable_deterministic_inference: from sglang.srt.batch_invariant_ops import enable_batch_invariant_mode enable_batch_invariant_mode() @@ -798,7 +809,7 @@ class ModelRunner: device=self.device, tp_group=( self.attention_tp_group.cpu_group - if get_parallel().enable_dp_attention + if self.server_args.enable_dp_attention else self.tp_group.cpu_group ), host_to_device_ratio=hisparse_cfg.host_to_device_ratio, @@ -962,7 +973,7 @@ class ModelRunner: get_offloader().post_init() # Register model for layerwise NVTX profiling if enabled - if get_exec().comm.enable_layerwise_nvtx_marker: + if self.server_args.enable_layerwise_nvtx_marker: pyt_hooks = PytHooks() pyt_hooks.register_hooks(self.model, module_prefix="model") @@ -1019,7 +1030,7 @@ class ModelRunner: ) dist_barrier_after_load( - elastic_ep_backend=get_exec().moe.elastic_ep_backend, + elastic_ep_backend=self.server_args.elastic_ep_backend, tp_rank=self.ps.tp_rank, is_ep_scale_joiner=self.server_args.is_ep_scale_joiner, ) @@ -1039,16 +1050,16 @@ class ModelRunner: self.lora_manager = LoRAManager( base_model=self.model, base_hf_config=self.model_config.hf_config, - max_loras_per_batch=get_lora().max_loras_per_batch, + max_loras_per_batch=self.server_args.max_loras_per_batch, load_config=self.load_config, dtype=self.dtype, server_args=self.server_args, - lora_backend=get_lora().lora_backend, + lora_backend=self.server_args.lora_backend, tp_size=self.ps.tp_size, tp_rank=self.ps.tp_rank, - max_lora_rank=get_lora().max_lora_rank, - target_modules=get_lora().lora_target_modules, - lora_paths=get_lora().lora_paths, + max_lora_rank=self.server_args.max_lora_rank, + target_modules=self.server_args.lora_target_modules, + lora_paths=self.server_args.lora_paths, ) if not cuda_graph_fully_disabled(): init_lora_cuda_graph_moe_buffers( @@ -1320,7 +1331,7 @@ class ModelRunner: ) output.expert_distribution_metrics = recorder_outputs.get("metrics") - no_copy_to_cpu = not get_schedule().disable_overlap_schedule + no_copy_to_cpu = not self.server_args.disable_overlap_schedule if ( not self.is_draft_worker and (experts_capturer := get_global_experts_capturer()) is not None @@ -1350,7 +1361,7 @@ class ModelRunner: self.msprobe_debugger.stop() self.msprobe_debugger.step() - if get_exec().moe.elastic_ep_backend is not None: + if self.server_args.elastic_ep_backend is not None: self.maybe_join_ep_ranks() return output @@ -1609,7 +1620,7 @@ class ModelRunner: if added <= 0: return - initial_ep_size = get_parallel().elastic_ep_initial_size + initial_ep_size = self.server_args.elastic_ep_initial_size assert initial_ep_size is not None self.server_args.override("elastic_ep.scale", ep_size=effective_size) @@ -1628,7 +1639,7 @@ class ModelRunner: set_global_expert_location_metadata(new_metadata, allow_overwrite=True) def _elastic_global_rank(self) -> int: - return self.ps.tp_rank + get_parallel().ep_join_rank_offset + return self.ps.tp_rank + self.server_args.ep_join_rank_offset def _report_elastic_scale_failure(self, error: str, effective_size: int) -> None: if self.ps.tp_rank != 0 or self.server_args.is_ep_scale_joiner: @@ -1705,9 +1716,7 @@ class ModelRunner: new_dp_size=target_size, new_dp_rank=self._elastic_global_rank(), ) - from sglang.srt.runtime_context import get_context - - get_context().override("elastic_ep.scale", dp_size=target_size) + self.server_args.override("elastic_ep.scale", dp_size=target_size) ElasticEPStateManager.mark_syncing_new_world() self._elastic_scale_ready_barrier( @@ -1756,7 +1765,7 @@ class ModelRunner: recovered = maybe_recover_ep_ranks( tp_group=self.tp_group, eplb_manager=self.eplb_manager, - random_seed=get_device().random_seed, + random_seed=self.server_args.random_seed, ) if recovered: self.forward_pass_id = 0 @@ -1765,7 +1774,7 @@ class ModelRunner: local_timeout = ( state.pending_since is not None and time.monotonic() - state.pending_since - > get_exec().moe.elastic_ep_scale_timeout + > self.server_args.elastic_ep_scale_timeout ) timeout = state.active_ranks.new_tensor(int(local_timeout)) dist.all_reduce(timeout, op=dist.ReduceOp.MAX, group=dist.group.WORLD) @@ -1833,9 +1842,7 @@ class ModelRunner: load_config: LoadConfig, ) -> None: self.model = new_model - from sglang.srt.runtime_context import get_context - - get_context().override( + self.server_args.override( "model_runner.update_model_fields", model_path=model_path, load_format=load_format, diff --git a/python/sglang/srt/model_executor/model_runner_components/misc_utils.py b/python/sglang/srt/model_executor/model_runner_components/misc_utils.py index cf82daa3d..f0261d3ae 100644 --- a/python/sglang/srt/model_executor/model_runner_components/misc_utils.py +++ b/python/sglang/srt/model_executor/model_runner_components/misc_utils.py @@ -24,12 +24,6 @@ def maybe_disable_chunked_prefix_cache( # model's (often non-MLA) config must not flip the shared setting. if is_draft_worker: return - - # This is a load-time gate that runs in ModelRunner.__init__ BEFORE the - # runner publishes its config (and direct/benchmark construction never - # publishes earlier), so read/write the supplied server_args. The runner's - # subsequent publish snapshots this into the schedule bag for get_schedule() - # readers. if ( not use_mla_backend or server_args.attention_backend diff --git a/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py b/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py index 5fbcab266..bf56487ad 100644 --- a/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py +++ b/python/sglang/srt/model_executor/model_runner_components/remote_instance_weight_transporter.py @@ -11,7 +11,6 @@ from sglang.srt.model_loader.remote_instance_weight_loader_utils import ( RemoteInstanceWeightLoaderBackend, register_memory_region, ) -from sglang.srt.runtime_context import get_model, get_parallel from sglang.srt.server_args import ServerArgs from sglang.srt.utils.network import NetworkAddress, get_local_ip_auto @@ -59,7 +58,7 @@ class RemoteInstanceWeightTransporter: # ModelExpress owns TransferEngine memory registration and metadata # publishing for backend=modelexpress. Re-registering here would # overlap the same weight buffers. - and get_model().remote_instance_weight_loader_backend + and self.server_args.remote_instance_weight_loader_backend != RemoteInstanceWeightLoaderBackend.MODELEXPRESS and self.engine is not None and self.weight_info is None @@ -76,16 +75,16 @@ class RemoteInstanceWeightTransporter: """ import requests as http_requests - if get_parallel().dist_init_addr: + if self.server_args.dist_init_addr: # Multi-node: bootstrap server is on the head node (node_rank==0). # Derive host from dist_init_addr (shared across all nodes). bootstrap_host = ( - NetworkAddress.parse(get_parallel().dist_init_addr).resolved().host + NetworkAddress.parse(self.server_args.dist_init_addr).resolved().host ) else: bootstrap_host = "127.0.0.1" - bootstrap_port = get_model().engine_info_bootstrap_port + bootstrap_port = self.server_args.engine_info_bootstrap_port bootstrap_na = NetworkAddress(bootstrap_host, bootstrap_port) url = f"{bootstrap_na.to_url()}/register_transfer_engine_info" diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index cce163b4e..13c0c07d4 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -44,7 +44,7 @@ from sglang.srt.model_loader.remote_instance_weight_loader_utils import ( get_remote_instance_transfer_engine_info_per_rank, register_memory_region, ) -from sglang.srt.runtime_context import get_exec, get_server_args +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import get_available_gpu_memory # Try to import accelerate (optional dependency) @@ -71,7 +71,9 @@ from sglang.srt.connector import ( get_connector_type, ) from sglang.srt.connector.utils import parse_model_name -from sglang.srt.distributed import model_parallel_is_initialized +from sglang.srt.distributed import ( + model_parallel_is_initialized, +) from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.model_loader.remote_instance_weight_loader_utils import ( @@ -863,8 +865,9 @@ class LayeredModelLoader(DefaultModelLoader): device_config: DeviceConfig, ) -> nn.Module: from sglang.srt.layers.torchao_utils import apply_torchao_config_to_model + from sglang.srt.runtime_context import get_server_args - torchao_config = get_exec().graph.torchao_config + torchao_config = get_server_args().torchao_config target_device = torch.device(device_config.device) quant_config = _get_quantization_config(model_config, self.load_config) diff --git a/python/sglang/srt/models/apertus.py b/python/sglang/srt/models/apertus.py index b2342e251..b9f79000f 100644 --- a/python/sglang/srt/models/apertus.py +++ b/python/sglang/srt/models/apertus.py @@ -26,7 +26,9 @@ import torch from torch import nn from transformers import ApertusConfig -from sglang.srt.distributed import get_pp_group +from sglang.srt.distributed import ( + get_pp_group, +) from sglang.srt.layers.activation import XIELU from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -50,7 +52,7 @@ from sglang.srt.model_loader.weight_utils import ( kv_cache_scales_loader, maybe_remap_kv_scale_name, ) -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, make_layers logger = logging.getLogger(__name__) @@ -440,7 +442,7 @@ class ApertusForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) diff --git a/python/sglang/srt/models/arcee.py b/python/sglang/srt/models/arcee.py index c1d085159..20d0ecc7c 100644 --- a/python/sglang/srt/models/arcee.py +++ b/python/sglang/srt/models/arcee.py @@ -20,7 +20,9 @@ import torch from torch import nn from transformers import LlamaConfig -from sglang.srt.distributed import get_pp_group +from sglang.srt.distributed import ( + get_pp_group, +) from sglang.srt.layers.activation import get_act_fn from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -44,7 +46,7 @@ from sglang.srt.model_loader.weight_utils import ( kv_cache_scales_loader, maybe_remap_kv_scale_name, ) -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, make_layers logger = logging.getLogger(__name__) @@ -403,7 +405,7 @@ class ArceeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index 3a8c8cef3..9abccdbe0 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -41,7 +41,9 @@ from sglang.srt.layers.communicator import ( LayerScatterModes, enable_moe_dense_fully_dp, ) -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -76,9 +78,9 @@ from sglang.srt.models.utils import ( enable_fused_set_kv_buffer, ) from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import add_prefix, is_cuda, is_non_idle_and_non_empty, make_layers @@ -207,7 +209,7 @@ class BailingMoESparseMoeBlock(nn.Module): self.router_dtype = torch.bfloat16 # TODO global_server_args.ep_num_redundant_experts is used for eplb, not supported now - assert get_exec().moe.ep_num_redundant_experts == 0 + assert get_server_args().ep_num_redundant_experts == 0 # check group topk self.num_expert_group = getattr(config, "n_group", 0) self.topk_group = getattr(config, "topk_group", 0) @@ -221,7 +223,9 @@ class BailingMoESparseMoeBlock(nn.Module): self.num_expert_group = self.topk_group = None self.use_grouped_topk = False - self.num_experts = config.num_experts + get_exec().moe.ep_num_redundant_experts + self.num_experts = ( + config.num_experts + get_server_args().ep_num_redundant_experts + ) self.gate = BailingMoEGate( config=config, @@ -820,7 +824,7 @@ class BailingMoEForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 424a8d311..27b265814 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -12,12 +12,17 @@ from transformers import PretrainedConfig from sglang.kernels.ops.attention.fla.layernorm_gated import RMSNorm as RMSNormGated from sglang.kernels.ops.attention.fla.layernorm_gated import layernorm_fn from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz -from sglang.srt.distributed import get_pp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed import ( + get_pp_group, + tensor_model_parallel_all_reduce, +) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -54,9 +59,9 @@ from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA, DeepseekV2MLP, _is_hip from sglang.srt.models.utils import WeightsMapper from sglang.srt.runtime_context import ( - get_device, get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import ( @@ -524,7 +529,7 @@ class BailingMoELinearAttention(nn.Module): base=self.rope_theta, rope_scaling=config.rope_scaling, is_neox_style=True, - device=get_device().device, + device=get_server_args().device, dtype=torch.float32, ) @@ -685,7 +690,7 @@ class BailingMoEAttention(nn.Module): max_position=self.max_position_embeddings, base=self.rope_theta, rope_scaling=config.rope_scaling, - device=get_device().device, + device=get_server_args().device, ) self.attn = RadixAttention( self.num_heads, @@ -1084,7 +1089,7 @@ class BailingMoELinearForCausalLM(nn.Module): config.hidden_size, params_dtype=torch.float32, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/bailing_moe_nextn.py b/python/sglang/srt/models/bailing_moe_nextn.py index dab75ef02..5741f81c4 100644 --- a/python/sglang/srt/models/bailing_moe_nextn.py +++ b/python/sglang/srt/models/bailing_moe_nextn.py @@ -42,7 +42,7 @@ from sglang.srt.models.bailing_moe_linear import ( BailingMoeV2_5ForCausalLM, ) from sglang.srt.models.utils import WeightsMapper -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import BumpAllocator, add_prefix LoraConfig = None @@ -208,7 +208,7 @@ class BailingMoeForCausalLMNextN(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) if hasattr(self.config, "model_type") and config.model_type == "bailing_hybrid": diff --git a/python/sglang/srt/models/bert.py b/python/sglang/srt/models/bert.py index 154900912..82881395f 100644 --- a/python/sglang/srt/models/bert.py +++ b/python/sglang/srt/models/bert.py @@ -16,7 +16,7 @@ from sglang.srt.layers.radix_attention import AttentionType, RadixAttention from sglang.srt.layers.vocab_parallel_embedding import VocabParallelEmbedding from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import get_model, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix BertConfig = None @@ -365,7 +365,9 @@ class BertModel(nn.Module): quant_config=quant_config, prefix=add_prefix("encoder", prefix), ) - pooling_type = PoolingType.CLS if get_model().is_embedding else PoolingType.LAST + pooling_type = ( + PoolingType.CLS if get_server_args().is_embedding else PoolingType.LAST + ) self.pooler = ( BertPooler(config) if self.use_bert_pooler diff --git a/python/sglang/srt/models/deepseek_common/attention_backend_handler.py b/python/sglang/srt/models/deepseek_common/attention_backend_handler.py index 5b3f8a95b..6e24068a1 100644 --- a/python/sglang/srt/models/deepseek_common/attention_backend_handler.py +++ b/python/sglang/srt/models/deepseek_common/attention_backend_handler.py @@ -11,7 +11,7 @@ from sglang.srt.models.deepseek_common.attention_forward_methods.forward_methods AttnForwardMethod, ) from sglang.srt.models.deepseek_common.utils import _is_hip -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import use_intel_amx_backend MHA_ONE_SHOT_SUPPORTED_BACKENDS = ["fa3", "flashinfer", "flashmla"] @@ -118,7 +118,7 @@ def handle_attention_flashinfer(attn, forward_batch): def handle_attention_fa3(attn, forward_batch): # when deterministic inference is enabled, use MLA - if get_exec().deterministic.enable_deterministic_inference: + if get_server_args().enable_deterministic_inference: return _dispatch_mla_subtype(attn, forward_batch) else: return _handle_attention_backend(attn, forward_batch, "fa3") @@ -187,7 +187,7 @@ def handle_attention_triton(attn, forward_batch): return AttnForwardMethod.MLA # when deterministic inference is enabled, use MLA - if get_exec().deterministic.enable_deterministic_inference: + if get_server_args().enable_deterministic_inference: return _dispatch_mla_subtype(attn, forward_batch) if ( diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py index 19b4805c1..c661322ac 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mha.py @@ -30,11 +30,7 @@ from sglang.srt.models.deepseek_common.utils import ( _use_aiter_bpreshuffle_gfx95, _use_aiter_gfx95, ) -from sglang.srt.runtime_context import ( - get_exec, - get_parallel, - get_schedule, -) +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import BumpAllocator, get_bool_env_var, next_power_of_2 _use_fp8_prefill_attn = ( @@ -146,7 +142,9 @@ def _forward_dsa_indexer_for_mha( class DeepseekMHAForwardMixin: def init_mha_forward(self: DeepseekV2AttentionMLA): - self.disable_chunked_prefix_cache = get_schedule().disable_chunked_prefix_cache + self.disable_chunked_prefix_cache = ( + get_server_args().disable_chunked_prefix_cache + ) # TODO: Design a finer way to determine the threshold self.chunked_prefix_cache_threshold = ( @@ -307,8 +305,8 @@ class DeepseekMHAForwardMixin: self.use_dsa and self.kv_cache_dtype == "fp8_e4m3" and ( - not get_exec().kernel.dsa_decode_backend == "trtllm" - or not get_exec().kernel.dsa_prefill_backend == "trtllm" + not get_server_args().dsa_decode_backend == "trtllm" + or not get_server_args().dsa_prefill_backend == "trtllm" ) ): # FP8 path: dequantize DSA-specific FP8 format to BF16 diff --git a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py index 2117e1593..bc26c7522 100644 --- a/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py +++ b/python/sglang/srt/models/deepseek_common/attention_forward_methods/forward_mla.py @@ -65,8 +65,10 @@ from sglang.srt.models.deepseek_common.utils import ( _use_aiter_bpreshuffle_gfx95, _use_aiter_gfx95, ) -from sglang.srt.runtime_context import get_exec, get_parallel, get_server_args -from sglang.srt.state_capturer.indexer_topk import maybe_capture_indexer_topk +from sglang.srt.runtime_context import get_parallel, get_server_args +from sglang.srt.state_capturer.indexer_topk import ( + maybe_capture_indexer_topk, +) from sglang.srt.utils import BumpAllocator from sglang.srt.utils.custom_op import register_custom_op @@ -151,7 +153,7 @@ def _should_defer_dsa_cp_kv_gather( class DeepseekMLAForwardMixin: def init_mla_forward(self: DeepseekV2AttentionMLA): self.flashinfer_mla_disable_ragged = ( - get_exec().kernel.flashinfer_mla_disable_ragged + get_server_args().flashinfer_mla_disable_ragged ) def should_run_indexer( @@ -988,8 +990,8 @@ class DeepseekMLAForwardMixin: """ if self.current_attention_backend in ("dsa", "nsa"): return ( - get_exec().kernel.dsa_decode_backend == "trtllm" - or get_exec().kernel.dsa_prefill_backend == "trtllm" + get_server_args().dsa_decode_backend == "trtllm" + or get_server_args().dsa_prefill_backend == "trtllm" ) and get_attn_backend().kv_cache_dtype == torch.float8_e4m3fn return ( diff --git a/python/sglang/srt/models/deepseek_nextn.py b/python/sglang/srt/models/deepseek_nextn.py index da3e63b86..a2246d22a 100644 --- a/python/sglang/srt/models/deepseek_nextn.py +++ b/python/sglang/srt/models/deepseek_nextn.py @@ -59,11 +59,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.deepseek_common.utils import enable_nextn_moe_bf16_cast_to_fp8 from sglang.srt.models.deepseek_v2 import DeepseekV2DecoderLayer, DeepseekV3ForCausalLM from sglang.srt.models.utils import WeightsMapper -from sglang.srt.runtime_context import ( - get_model, - get_parallel, - get_spec, -) +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import BumpAllocator, add_prefix, is_cuda, is_npu @@ -152,7 +148,7 @@ class DeepseekModelNextN(nn.Module): self.rot_weight = None if _is_npu: - rot_weight_path = get_model().model_path + "/rot.safetensors" + rot_weight_path = get_server_args().model_path + "/rot.safetensors" if os.path.isfile(rot_weight_path): self.rot_weight = load_file(rot_weight_path) self.rot_weight = self.rot_weight["rot.weight"].npu() @@ -165,7 +161,8 @@ class DeepseekModelNextN(nn.Module): layer_name = "decoder" if _is_npu and ( - get_spec().speculative_draft_model_path == get_model().model_path + get_server_args().speculative_draft_model_path + == get_server_args().model_path ): layer_name = "layers." + str(config.num_hidden_layers) @@ -204,7 +201,7 @@ class DeepseekModelNextN(nn.Module): if ( _is_npu and self.quant_config is None - and get_model().quantization is not None + and get_server_args().quantization is not None ): # ascend mtp unquant exit_stack.enter_context(envs.SGLANG_DEEPEP_BF16_DISPATCH.override(True)) @@ -380,7 +377,7 @@ class DeepseekV3ForCausalLMNextN(DeepseekV3ForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index 95db44a46..0c2745da8 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -78,7 +78,9 @@ from sglang.srt.layers.communicator_dsa_cp import ( maybe_prefetch_next_full_attention_kv, ) from sglang.srt.layers.cp.utils import is_cp_v2_active -from sglang.srt.layers.dcp.planner import prepare_decode_context_parallel_metadata +from sglang.srt.layers.dcp.planner import ( + prepare_decode_context_parallel_metadata, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -113,7 +115,9 @@ from sglang.srt.layers.moe.utils import ( ) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.fp8 import Fp8Config -from sglang.srt.layers.quantization.fp8_utils import materialize_bpreshuffle_fp8_scale +from sglang.srt.layers.quantization.fp8_utils import ( + materialize_bpreshuffle_fp8_scale, +) from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( maybe_fuse_routed_scale_and_shared_add, ) @@ -179,14 +183,11 @@ from sglang.srt.models.deepseek_common.utils import ( is_wint4afp8_or_wint4a16_config, ) from sglang.srt.runtime_context import ( - get_device, - get_exec, get_flags, get_forward, get_model, get_parallel, get_server_args, - get_spec, ) from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils import ( @@ -380,7 +381,9 @@ class DeepseekV2MLP(nn.Module): return down_output if self.use_fused_clamp_act_mul and self.swiglu_limit is not None: - from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul + from aiter.ops.triton.fusions.fused_clamp_act_mul import ( + fused_clamp_act_mul, + ) if not self._fused_clamp_fp8_checked: from sglang.srt.layers.quantization.fp8 import Fp8LinearMethod @@ -491,7 +494,7 @@ class MoEGate(nn.Module): True, # is_vnni ) - if get_exec().deterministic.enable_deterministic_inference: + if get_server_args().enable_deterministic_inference: return F.linear(hidden_states, self.weight, None) if ( @@ -557,7 +560,7 @@ class DeepseekV2MoE(nn.Module): n_shared_experts = ( 0 if config.n_shared_experts is None else int(config.n_shared_experts) ) - _fusion_disabled = get_exec().moe.disable_shared_experts_fusion + _fusion_disabled = get_server_args().disable_shared_experts_fusion # num_fused_shared_experts drives weight remapping in deepseek_weight_loader: # mlp.shared_experts → mlp.experts.256 when > 0. @@ -627,7 +630,8 @@ class DeepseekV2MoE(nn.Module): fused_shared_experts_scaling_factor = 1.0 / float(self.moe_ep_size) self.experts = get_moe_impl_class(quant_config)( - num_experts=num_experts_for_moe + get_exec().moe.ep_num_redundant_experts, + num_experts=num_experts_for_moe + + get_server_args().ep_num_redundant_experts, num_fused_shared_experts=self.num_fused_shared_experts, top_k=top_k_for_moe, hidden_size=config.hidden_size, @@ -800,7 +804,7 @@ class DeepseekV2MoE(nn.Module): # TODO: we will support tp < ep in the future self.ep_size = get_parallel().moe_ep_size self.num_experts = ( - config.n_routed_experts + get_exec().moe.ep_num_redundant_experts + config.n_routed_experts + get_server_args().ep_num_redundant_experts ) self.renormalize = config.norm_topk_prob self.topk_group = config.topk_group @@ -1714,7 +1718,7 @@ class DeepseekV2AttentionMLA( base=rope_theta, rope_scaling=rope_scaling, is_neox_style=is_neox_style, - device=get_device().device, + device=get_server_args().device, ) if rope_scaling and rope_scaling.get("apply_yarn_scaling", True): @@ -2065,7 +2069,7 @@ class DeepseekV2DecoderLayer(nn.Module): rope_scaling = config.rope_scaling max_position_embeddings = config.max_position_embeddings self.speculative_algorithm = SpeculativeAlgorithm.from_string( - get_spec().speculative_algorithm + get_server_args().speculative_algorithm ) self.dsa_enable_prefill_cp = dsa_enable_prefill_cp self.mla_enable_prefill_cp = mla_enable_prefill_cp @@ -2722,7 +2726,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: # ranks other than the last rank will have a placeholder layer @@ -2761,7 +2765,7 @@ class DeepseekV2ForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): self.num_fused_shared_experts = 0 server_args = get_server_args() - if get_exec().moe.disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 9e9896fd3..e3c11aae3 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -23,17 +23,24 @@ import torch.nn as nn import torch.nn.functional as F import sglang.srt.models.deepseek_v2 as deepseek_v2 -from sglang.kernels.ops.attention.deepseek_v4_rope import v4_rope_inplace_npu +from sglang.kernels.ops.attention.deepseek_v4_rope import ( + v4_rope_inplace_npu, +) from sglang.kernels.ops.attention.dsv4 import ( fused_norm_rope_inplace, fused_q_norm_rope, fused_rope_inplace, sglang_per_token_group_quant_fp8_dsv4_wo_a, ) -from sglang.kernels.ops.quantization.fp8_kernel import sglang_per_token_group_quant_fp8 +from sglang.kernels.ops.quantization.fp8_kernel import ( + sglang_per_token_group_quant_fp8, +) from sglang.srt.compilation.compilation_config import register_split_op from sglang.srt.configs.deepseek_v4 import DeepSeekV4Config -from sglang.srt.distributed import get_pp_group, get_tp_group +from sglang.srt.distributed import ( + get_pp_group, + get_tp_group, +) from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) @@ -129,12 +136,7 @@ from sglang.srt.models.deepseek_v2 import ( _is_npu, _is_xpu, ) -from sglang.srt.runtime_context import ( - get_device, - get_exec, - get_forward, - get_parallel, -) +from sglang.srt.runtime_context import get_forward, get_parallel, get_server_args if not _is_hip: from sglang.srt.layers.utils.cp_utils import ( @@ -309,7 +311,9 @@ def _freqs_cis_to_cos_sin( if TYPE_CHECKING: - from sglang.srt.layers.attention.deepseek_v4_backend import DeepseekV4AttnBackend + from sglang.srt.layers.attention.deepseek_v4_backend import ( + DeepseekV4AttnBackend, + ) from sglang.srt.layers.attention.deepseek_v4_backend_hip_radix import ( DeepseekV4HipRadixBackend, ) @@ -571,7 +575,7 @@ class MQALayer(MqaAttentionBase): base=self.rope_base, rope_scaling=self.rope_scaling, is_neox_style=False, - device=get_device().device, + device=get_server_args().device, ) if _is_hip: @@ -2403,7 +2407,7 @@ class DeepseekV4ForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() @@ -2454,11 +2458,11 @@ class DeepseekV4ForCausalLM(nn.Module): def determine_num_fused_shared_experts(self): self.num_fused_shared_experts = 0 - if get_exec().moe.disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None - if get_exec().moe.enforce_shared_experts_fusion: + if get_server_args().enforce_shared_experts_fusion: if self.config.n_shared_experts != 1: raise ValueError( "DeepSeek V4 shared-experts fusion expects exactly one shared " diff --git a/python/sglang/srt/models/deepseek_v4_nextn.py b/python/sglang/srt/models/deepseek_v4_nextn.py index f8d039524..1dd326c6e 100644 --- a/python/sglang/srt/models/deepseek_v4_nextn.py +++ b/python/sglang/srt/models/deepseek_v4_nextn.py @@ -38,7 +38,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.models.deepseek_v4 import DeepseekV4DecoderLayer, DeepseekV4ForCausalLM -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -233,7 +233,7 @@ class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/exaone4.py b/python/sglang/srt/models/exaone4.py index d01dad06c..648f70417 100644 --- a/python/sglang/srt/models/exaone4.py +++ b/python/sglang/srt/models/exaone4.py @@ -28,7 +28,7 @@ from sglang.srt.model_loader.weight_utils import ( default_weight_loader, maybe_remap_kv_scale_name, ) -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, make_layers from sglang.utils import get_exception_traceback, logger @@ -439,7 +439,7 @@ class Exaone4ForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index 665d52833..b90c75efe 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -24,12 +24,17 @@ import torch from torch import nn from transformers import PretrainedConfig -from sglang.srt.distributed import get_pp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed import ( + get_pp_group, + tensor_model_parallel_all_reduce, +) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -57,11 +62,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import ( - get_exec, - get_parallel, - get_stream, -) +from sglang.srt.runtime_context import get_parallel, get_server_args, get_stream from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_layers logger = logging.getLogger(__name__) @@ -164,7 +165,7 @@ class ExaoneMoESparseMoEBlock(nn.Module): ) self.experts = get_moe_impl_class(quant_config)( - num_experts=config.num_experts + get_exec().moe.ep_num_redundant_experts, + num_experts=config.num_experts + get_server_args().ep_num_redundant_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, intermediate_size=config.moe_intermediate_size, @@ -205,7 +206,7 @@ class ExaoneMoESparseMoEBlock(nn.Module): if get_moe_a2a_backend().is_deepep(): self.ep_size = get_parallel().moe_ep_size self.num_experts = ( - config.num_experts + get_exec().moe.ep_num_redundant_experts + config.num_experts + get_server_args().ep_num_redundant_experts ) self.top_k = config.num_experts_per_tok @@ -642,7 +643,7 @@ class ExaoneMoEForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # For EAGLE3 support diff --git a/python/sglang/srt/models/exaone_moe_mtp.py b/python/sglang/srt/models/exaone_moe_mtp.py index 10dea5461..439a4c354 100644 --- a/python/sglang/srt/models/exaone_moe_mtp.py +++ b/python/sglang/srt/models/exaone_moe_mtp.py @@ -30,7 +30,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.exaone_moe import ExaoneMoEForCausalLM, ExaoneMoEModel -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -63,7 +63,7 @@ class ExaoneMoEForCausalLMMTP(ExaoneMoEForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/falcon_h1.py b/python/sglang/srt/models/falcon_h1.py index 0bf3c2414..5283f2798 100644 --- a/python/sglang/srt/models/falcon_h1.py +++ b/python/sglang/srt/models/falcon_h1.py @@ -13,7 +13,9 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -34,6 +36,7 @@ from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.runtime_context import ( get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import add_prefix, is_cuda, make_layers @@ -474,7 +477,7 @@ class FalconH1ForCausalLM(nn.Module): quant_config=quant_config, org_num_embeddings=config.vocab_size, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.lm_head = self.lm_head.float() self.lm_head_multiplier = config.lm_head_multiplier diff --git a/python/sglang/srt/models/gemma4_causal.py b/python/sglang/srt/models/gemma4_causal.py index 9d4e1f5e1..f2d9a645e 100644 --- a/python/sglang/srt/models/gemma4_causal.py +++ b/python/sglang/srt/models/gemma4_causal.py @@ -18,7 +18,11 @@ from typing import Iterable, List, Optional, Set, Tuple, Union import torch from torch import nn -from transformers import Gemma4TextConfig, PretrainedConfig, PreTrainedModel +from transformers import ( + Gemma4TextConfig, + PretrainedConfig, + PreTrainedModel, +) from sglang.kernels.ops.layernorm.gemma4_fused_ops import ( gemma4_fused_routing, @@ -27,7 +31,9 @@ from sglang.kernels.ops.layernorm.gemma4_fused_ops import ( gemma_rmsnorm_residual_scalar, gemma_routing_post_topk, ) -from sglang.srt.distributed import get_pp_group +from sglang.srt.distributed import ( + get_pp_group, +) from sglang.srt.layers.layernorm import Gemma4RMSNorm, RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -49,8 +55,10 @@ from sglang.srt.model_loader.weight_utils import ( maybe_remap_kv_scale_name, ) from sglang.srt.models.gemma3_causal import Gemma3MLP, Gemma3TextScaledWordEmbedding -from sglang.srt.models.utils import create_fused_set_kv_buffer_arg -from sglang.srt.runtime_context import get_exec, get_parallel, get_server_args +from sglang.srt.models.utils import ( + create_fused_set_kv_buffer_arg, +) +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, make_layers logger = logging.getLogger(__name__) @@ -246,7 +254,7 @@ class Gemma4MoE(nn.Module): experts_type = get_moe_impl_class(quant_config) self.experts = experts_type( - num_experts=config.num_experts + get_exec().moe.ep_num_redundant_experts, + num_experts=config.num_experts + get_server_args().ep_num_redundant_experts, hidden_size=config.hidden_size, intermediate_size=config.moe_intermediate_size, layer_id=layer_id, diff --git a/python/sglang/srt/models/gemma4_vision.py b/python/sglang/srt/models/gemma4_vision.py index 63fa2d064..7e440555c 100644 --- a/python/sglang/srt/models/gemma4_vision.py +++ b/python/sglang/srt/models/gemma4_vision.py @@ -29,7 +29,7 @@ from sglang.srt.layers.clippable_linear import ( ) from sglang.srt.layers.layernorm import Gemma4RMSNorm from sglang.srt.layers.quantization.base_config import QuantizationConfig -from sglang.srt.runtime_context import get_mm, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import add_prefix, get_device_capability, is_cuda, is_hip # --------------------------------------------------------------------------- @@ -181,8 +181,9 @@ class Gemma4VisionAttention(nn.Module): @staticmethod def _select_backend() -> str: """Mirror VisionAttention._determine_attention_backend for consistency.""" + from sglang.srt.runtime_context import get_server_args - override = get_mm().mm_attention_backend + override = get_server_args().mm_attention_backend if override is not None: return override if is_cuda(): diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index 9db2034e0..758aeaa52 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -84,9 +84,9 @@ from sglang.srt.models.deepseek_nextn import DeepseekV3ForCausalLMNextN from sglang.srt.models.deepseek_v2 import DeepseekV2ForCausalLM from sglang.srt.models.utils import WeightsMapper, apply_qk_norm from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import ( @@ -406,7 +406,7 @@ class Glm4MoeSparseMoeBlock(nn.Module): self.n_shared_experts = config.n_shared_experts self.num_fused_shared_experts = ( 0 - if get_exec().moe.disable_shared_experts_fusion + if get_server_args().disable_shared_experts_fusion else config.n_shared_experts ) @@ -526,7 +526,7 @@ class Glm4MoeSparseMoeBlock(nn.Module): # TODO: we will support tp < ep in the future self.ep_size = get_parallel().moe_ep_size self.num_experts = ( - config.n_routed_experts + get_exec().moe.ep_num_redundant_experts + config.n_routed_experts + get_server_args().ep_num_redundant_experts ) self.renormalize = config.norm_topk_prob self.topk_group = config.topk_group @@ -1170,7 +1170,7 @@ class Glm4MoeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) @@ -1178,7 +1178,7 @@ class Glm4MoeForCausalLM(nn.Module): self.capture_aux_hidden_states = False def determine_num_fused_shared_experts(self): - if get_exec().moe.disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index 0df3664b9..c650585a5 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -75,9 +75,9 @@ from sglang.srt.models.deepseek_common.deepseek_weight_loader import ( from sglang.srt.models.deepseek_common.utils import _is_cuda, _use_aiter from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import ( @@ -189,7 +189,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): self.n_shared_experts = config.n_shared_experts self.num_fused_shared_experts = ( 0 - if get_exec().moe.disable_shared_experts_fusion + if get_server_args().disable_shared_experts_fusion else config.n_shared_experts ) self.config = config @@ -216,7 +216,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): self.experts = get_moe_impl_class(quant_config)( num_experts=config.n_routed_experts + self.num_fused_shared_experts - + get_exec().moe.ep_num_redundant_experts, + + get_server_args().ep_num_redundant_experts, num_fused_shared_experts=self.num_fused_shared_experts, top_k=config.num_experts_per_tok + self.num_fused_shared_experts, hidden_size=config.hidden_size, @@ -284,7 +284,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): # TODO: we will support tp < ep in the future self.ep_size = get_parallel().moe_ep_size self.num_experts = ( - config.n_routed_experts + get_exec().moe.ep_num_redundant_experts + config.n_routed_experts + get_server_args().ep_num_redundant_experts ) self.renormalize = config.norm_topk_prob self.topk_group = config.topk_group @@ -907,7 +907,7 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) @@ -928,7 +928,7 @@ class Glm4MoeLiteForCausalLM(nn.Module, DeepseekV2WeightLoaderMixin): self, architecture: str = "Glm4MoeLiteForCausalLM" ): self.num_fused_shared_experts = 0 - if get_exec().moe.disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None diff --git a/python/sglang/srt/models/glm4_moe_lite_nextn.py b/python/sglang/srt/models/glm4_moe_lite_nextn.py index d3dbb95be..a8ad68b18 100644 --- a/python/sglang/srt/models/glm4_moe_lite_nextn.py +++ b/python/sglang/srt/models/glm4_moe_lite_nextn.py @@ -35,7 +35,7 @@ from sglang.srt.models.glm4_moe_lite import ( Glm4MoeLiteDecoderLayer, Glm4MoeLiteForCausalLM, ) -from sglang.srt.runtime_context import get_exec, get_parallel, get_spec +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import BumpAllocator, add_prefix, is_npu logger = logging.getLogger(__name__) @@ -139,7 +139,7 @@ class Glm4MoeLiteForCausalLMNextN(Glm4MoeLiteForCausalLM): nn.Module.__init__(self) self.config = config self.tp_size = get_parallel().tp_size - if is_npu() and get_spec().speculative_draft_model_quantization is None: + if is_npu() and get_server_args().speculative_draft_model_quantization is None: quant_config = None self.quant_config = quant_config @@ -151,12 +151,12 @@ class Glm4MoeLiteForCausalLMNextN(Glm4MoeLiteForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.num_fused_shared_experts = ( - 0 if get_exec().moe.disable_shared_experts_fusion else 1 + 0 if get_server_args().disable_shared_experts_fusion else 1 ) @torch.no_grad() diff --git a/python/sglang/srt/models/glm4_moe_nextn.py b/python/sglang/srt/models/glm4_moe_nextn.py index c836ae19e..3126fd026 100644 --- a/python/sglang/srt/models/glm4_moe_nextn.py +++ b/python/sglang/srt/models/glm4_moe_nextn.py @@ -32,7 +32,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.glm4_moe import Glm4MoeDecoderLayer, Glm4MoeForCausalLM -from sglang.srt.runtime_context import get_exec, get_parallel, get_spec +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_npu logger = logging.getLogger(__name__) @@ -125,7 +125,7 @@ class Glm4MoeForCausalLMNextN(Glm4MoeForCausalLM): nn.Module.__init__(self) self.config = config self.tp_size = get_parallel().tp_size - if is_npu() and get_spec().speculative_draft_model_quantization is None: + if is_npu() and get_server_args().speculative_draft_model_quantization is None: quant_config = None self.quant_config = quant_config @@ -137,12 +137,12 @@ class Glm4MoeForCausalLMNextN(Glm4MoeForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.num_fused_shared_experts = ( - 0 if get_exec().moe.disable_shared_experts_fusion else 1 + 0 if get_server_args().disable_shared_experts_fusion else 1 ) @torch.no_grad() diff --git a/python/sglang/srt/models/glm4v.py b/python/sglang/srt/models/glm4v.py index 43a74f4ad..598c6a22f 100644 --- a/python/sglang/srt/models/glm4v.py +++ b/python/sglang/srt/models/glm4v.py @@ -57,7 +57,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTe from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.glm4 import Glm4Model from sglang.srt.multimodal.mm_utils import run_dp_sharded_mrope_vision_model -from sglang.srt.runtime_context import get_mm, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_npu from sglang.srt.utils.hf_transformers_utils import get_processor @@ -558,7 +558,7 @@ class Glm4vForConditionalGeneration(nn.Module): self.pp_group = get_pp_group() self.config = config - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder vision_utils.update_vit_attn_dummy_heads_config(self.config) self.visual = Glm4vVisionModel( config.vision_config, diff --git a/python/sglang/srt/models/glm4v_moe.py b/python/sglang/srt/models/glm4v_moe.py index 0b5e21f58..c69899003 100644 --- a/python/sglang/srt/models/glm4v_moe.py +++ b/python/sglang/srt/models/glm4v_moe.py @@ -18,7 +18,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.glm4_moe import Glm4MoeModel from sglang.srt.models.glm4v import Glm4vForConditionalGeneration, Glm4vVisionModel -from sglang.srt.runtime_context import get_exec, get_mm, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, get_device_sm, is_cuda, log_info_on_rank0 from sglang.srt.utils.hf_transformers_utils import get_processor @@ -41,7 +41,7 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): self.pp_group = get_pp_group() self.config = config - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder vision_utils.update_vit_attn_dummy_heads_config(self.config) self.tp_size = get_parallel().tp_size self.quant_config = quant_config @@ -69,7 +69,7 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: # ranks other than the last rank will have a placeholder layer @@ -83,7 +83,7 @@ class Glm4vMoeForConditionalGeneration(Glm4vForConditionalGeneration): self.capture_aux_hidden_states = False def determine_num_fused_shared_experts(self): - if get_exec().moe.disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None diff --git a/python/sglang/srt/models/glm_image_vl.py b/python/sglang/srt/models/glm_image_vl.py index 4c3c96024..7402aa80b 100644 --- a/python/sglang/srt/models/glm_image_vl.py +++ b/python/sglang/srt/models/glm_image_vl.py @@ -34,7 +34,10 @@ from sglang.srt.layers.attention.vision import ( ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm -from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear +from sglang.srt.layers.linear import ( + QKVParallelLinear, + RowParallelLinear, +) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention @@ -54,7 +57,7 @@ from sglang.srt.models.qwen2 import Qwen2MLP as GlmImageTextMLP from sglang.srt.models.qwen3_vl import Qwen3_VisionMLP as GlmImageVisionMLP from sglang.srt.models.utils import compute_cu_seqlens_from_grid_numpy from sglang.srt.multimodal.mm_utils import run_dp_sharded_mrope_vision_model -from sglang.srt.runtime_context import get_mm, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_npu logger = logging.getLogger(__name__) @@ -1015,7 +1018,7 @@ class GlmImageForConditionalGeneration(nn.Module): self.vision_config = config.vision_config self.vq_config = config.vq_config self.text_config = config.text_config - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder # Bridge rope_parameters -> rope_scaling so Glm4Model can pick it up if hasattr(self.text_config, "rope_parameters") and not getattr( diff --git a/python/sglang/srt/models/glm_ocr.py b/python/sglang/srt/models/glm_ocr.py index dfd00f31b..e696b7c01 100644 --- a/python/sglang/srt/models/glm_ocr.py +++ b/python/sglang/srt/models/glm_ocr.py @@ -54,7 +54,7 @@ from sglang.srt.models.glm4v import ( Glm4vVisionModel, Glm4vVisionPatchEmbed, ) -from sglang.srt.runtime_context import get_mm +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import add_prefix from sglang.srt.utils.hf_transformers_utils import get_processor @@ -282,7 +282,7 @@ class GlmOcrForConditionalGeneration(Glm4vForConditionalGeneration): self.pp_group = get_pp_group() self.config = config - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder self.visual = GlmOcrVisionModel( vision_config=config.vision_config, text_config=config.text_config, diff --git a/python/sglang/srt/models/glm_ocr_nextn.py b/python/sglang/srt/models/glm_ocr_nextn.py index cdd9ca18c..07a2bb245 100644 --- a/python/sglang/srt/models/glm_ocr_nextn.py +++ b/python/sglang/srt/models/glm_ocr_nextn.py @@ -33,7 +33,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.glm4 import Glm4DecoderLayer from sglang.srt.models.glm_ocr import GlmOcrForConditionalGeneration -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -134,12 +134,12 @@ class GlmOcrForConditionalGenerationNextN(GlmOcrForConditionalGeneration): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.num_fused_shared_experts = ( - 0 if get_exec().moe.disable_shared_experts_fusion else 1 + 0 if get_server_args().disable_shared_experts_fusion else 1 ) @torch.no_grad() diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index c53f54592..cd5731671 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -34,7 +34,9 @@ from sglang.srt.distributed import ( from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -67,9 +69,9 @@ from sglang.srt.models.utils import ( enable_fused_set_kv_buffer, ) from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, + get_server_args, ) from sglang.srt.utils import ( LazyValue, @@ -228,7 +230,7 @@ class GptOssSparseMoeBlock(nn.Module): self.experts = experts_type( num_experts=config.num_local_experts - + get_exec().moe.ep_num_redundant_experts, + + get_server_args().ep_num_redundant_experts, top_k=config.num_experts_per_tok, layer_id=layer_id, hidden_size=config.hidden_size, @@ -256,7 +258,7 @@ class GptOssSparseMoeBlock(nn.Module): hidden_states: torch.Tensor, forward_batch: Optional[ForwardBatch] = None, ) -> torch.Tensor: - if get_parallel().dwdp_size > 1: + if get_server_args().dwdp_size > 1: return self.forward_dwdp(hidden_states) if not get_moe_a2a_backend().is_deepep(): @@ -418,7 +420,7 @@ class GptOssAttention(nn.Module): # Choose dtype of sinks based on attention backend: trtllm_mha requires float32, # others can use bfloat16 - attn_backend = get_exec().kernel.attention_backend + attn_backend = get_server_args().attention_backend sinks_dtype = torch.float32 if attn_backend == "trtllm_mha" else torch.bfloat16 self.sinks = nn.Parameter( torch.empty(self.num_heads, dtype=sinks_dtype), requires_grad=False @@ -774,7 +776,7 @@ class GptOssForCausalLM(nn.Module): config.hidden_size, # quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/inkling.py b/python/sglang/srt/models/inkling.py index 34cc1cf65..c2e10f487 100644 --- a/python/sglang/srt/models/inkling.py +++ b/python/sglang/srt/models/inkling.py @@ -14,7 +14,9 @@ from sglang.srt.configs.inkling import ( InklingModelConfig, InklingVisionConfig, ) -from sglang.srt.distributed import get_tensor_model_parallel_group +from sglang.srt.distributed import ( + get_tensor_model_parallel_group, +) from sglang.srt.environ import envs from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor @@ -70,12 +72,7 @@ from sglang.srt.models.inkling_common.util import ( trtllm_bf16_weight_prep_enabled, use_inkling_shared_fused_moe, ) -from sglang.srt.runtime_context import ( - get_exec, - get_model, - get_parallel, - get_server_args, -) +from sglang.srt.runtime_context import get_model, get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_cuda, make_layers logger = logging.getLogger(__name__) @@ -221,7 +218,7 @@ class InklingDecoderLayer(nn.Module): # cache (configs/inkling.py stream_dim) shard with them. The layer # all-gathers back to [T, H] after each sconv, before the residual add. self.attn_tp_group = get_parallel().attn_tp_group - self.scattered_sconv = get_exec().comm.enable_scattered_sconv + self.scattered_sconv = get_server_args().enable_scattered_sconv sconv_hidden = config.hidden_size if self.scattered_sconv: assert config.use_sconv, "--enable-scattered-sconv requires use_sconv" @@ -1281,7 +1278,9 @@ class InklingForConditionalGeneration(nn.Module): and not self.text_config.inference_moe_w13_interleaved and weight_loader is not default_weight_loader ): - from sglang.srt.layers.quantization.modelopt_quant import deinterleave_w13 + from sglang.srt.layers.quantization.modelopt_quant import ( + deinterleave_w13, + ) loaded_weight = deinterleave_w13(loaded_weight) if ( diff --git a/python/sglang/srt/models/inkling_common/attn.py b/python/sglang/srt/models/inkling_common/attn.py index cd2ab443c..14b12e576 100644 --- a/python/sglang/srt/models/inkling_common/attn.py +++ b/python/sglang/srt/models/inkling_common/attn.py @@ -29,7 +29,7 @@ from sglang.srt.models.inkling_common.kernels.comm import ( from sglang.srt.models.inkling_common.norm import RMSNorm from sglang.srt.models.inkling_common.sconv import SconvType, ShortConvolution from sglang.srt.models.utils import apply_qk_norm -from sglang.srt.runtime_context import get_exec, get_parallel, get_server_args +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, get_current_device_stream_fast try: @@ -296,7 +296,7 @@ class InklingAttention(nn.Module): ) # --enable-scattered-sconv: the output reduction becomes a hidden-dim # reduce-scatter (the consumer attn_sconv runs on the [T, H/P] shard). - self.scattered_sconv = get_exec().comm.enable_scattered_sconv + self.scattered_sconv = get_server_args().enable_scattered_sconv if is_local: self.rel_extent = local_extent diff --git a/python/sglang/srt/models/inkling_common/dense_mlp.py b/python/sglang/srt/models/inkling_common/dense_mlp.py index d4b6d357a..0509cb29b 100644 --- a/python/sglang/srt/models/inkling_common/dense_mlp.py +++ b/python/sglang/srt/models/inkling_common/dense_mlp.py @@ -18,7 +18,7 @@ from sglang.srt.models.inkling_common.util import ( lora_compatible_layout_enabled, ) from sglang.srt.models.llama import LlamaMLP -from sglang.srt.runtime_context import get_exec, get_model +from sglang.srt.runtime_context import get_server_args logger = logging.getLogger(__name__) @@ -123,7 +123,7 @@ class InklingDenseMLP(LlamaMLP): fused = fused and not lora_compatible_layout_enabled() self.layer_id = layer_id self.act_fn = InklingSwiglu(interleaved=fused) - self.scattered_sconv = get_exec().comm.enable_scattered_sconv + self.scattered_sconv = get_server_args().enable_scattered_sconv def forward( self, @@ -484,7 +484,7 @@ class InklingBatchDenseMLP(nn.Module, FusedMoELoadingMixin): # All shared experts must share one global weight scale (reshard with # single_global_scale=True). ModelOpt's input_scale = amax / (6 * 448). flat2 = scale2.reshape(-1).float() - if get_model().load_format == "dummy" and not bool( + if get_server_args().load_format == "dummy" and not bool( torch.all(flat2 == flat2[0]) ): # Dummy loading uses per-element noise; replace it with a valid scale. diff --git a/python/sglang/srt/models/inkling_common/kernels/comm.py b/python/sglang/srt/models/inkling_common/kernels/comm.py index 61787c32a..02f979b81 100644 --- a/python/sglang/srt/models/inkling_common/kernels/comm.py +++ b/python/sglang/srt/models/inkling_common/kernels/comm.py @@ -7,7 +7,7 @@ import msgspec import torch from sglang.srt.environ import envs -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import is_cuda if TYPE_CHECKING: @@ -251,7 +251,7 @@ def ar_sconv_norm_fusable( and envs.SGLANG_OPT_USE_INKLING_FUSED_AR_SCONV_NORM.get() ): return False - if get_exec().comm.enable_scattered_sconv: + if get_server_args().enable_scattered_sconv: # The decode {AR -> sconv -> norm} fusion is full-width; under scattered # sconv the output sconvs are hidden-sharded, so it does not apply. return False @@ -396,7 +396,7 @@ def get_ar_buffer( envs.SGLANG_OPT_USE_INKLING_CUSTOM_AR.get() # Scattered sconv replaces the AR with reduce_scatter_hidden, which # stages from comm.buffer[:n] -- never hand out the v4 region there. - and not get_exec().comm.enable_scattered_sconv + and not get_server_args().enable_scattered_sconv ): res = _get_inkling_ar_resources(comm) if ( @@ -696,7 +696,7 @@ def scattered_ar_sconv_fusable( if not is_cuda(): return False if not ( - get_exec().comm.enable_scattered_sconv + get_server_args().enable_scattered_sconv and envs.SGLANG_OPT_USE_INKLING_CUSTOM_AR.get() and envs.SGLANG_OPT_USE_INKLING_FUSED_AR_SCONV.get() ): @@ -1033,7 +1033,7 @@ def fullwidth_ar_sconv_fusable( if not is_cuda(): return False if not ( - not get_exec().comm.enable_scattered_sconv + not get_server_args().enable_scattered_sconv and envs.SGLANG_OPT_USE_INKLING_CUSTOM_AR.get() and envs.SGLANG_OPT_USE_INKLING_FUSED_AR_SCONV.get() ): diff --git a/python/sglang/srt/models/inkling_common/moe.py b/python/sglang/srt/models/inkling_common/moe.py index faf01817d..e6f7bf918 100644 --- a/python/sglang/srt/models/inkling_common/moe.py +++ b/python/sglang/srt/models/inkling_common/moe.py @@ -16,7 +16,9 @@ from sglang.kernels.ops.model.inkling.inkling_gate_topk_renorm import ( inkling_gate_gemv_fused, ) from sglang.srt.configs.inkling import InklingModelConfig -from sglang.srt.distributed import get_tensor_model_parallel_group +from sglang.srt.distributed import ( + get_tensor_model_parallel_group, +) from sglang.srt.environ import GateGemvMode, envs from sglang.srt.layers.moe import get_moe_runner_backend from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -57,7 +59,7 @@ from sglang.srt.models.inkling_common.util import ( lora_compatible_layout_enabled, use_inkling_shared_fused_moe, ) -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer from sglang.srt.utils import add_prefix, is_cuda, is_hip @@ -889,8 +891,9 @@ class InklingMoE(nn.Module): ) # --enable-scattered-sconv: the output reduction becomes a hidden-dim # reduce-scatter (the consumer mlp_sconv runs on the [T, H/P] shard). + from sglang.srt.runtime_context import get_server_args - self.scattered_sconv = get_exec().comm.enable_scattered_sconv + self.scattered_sconv = get_server_args().enable_scattered_sconv # Fold the shared-expert partials into the custom AR kernels (or their # stage-in copies) instead of a separate torch.add per MoE layer. self._fused_ar_shared = envs.SGLANG_OPT_USE_INKLING_FUSED_AR_SHARED.get() diff --git a/python/sglang/srt/models/inkling_common/sconv.py b/python/sglang/srt/models/inkling_common/sconv.py index 5a793941e..4b78e807f 100644 --- a/python/sglang/srt/models/inkling_common/sconv.py +++ b/python/sglang/srt/models/inkling_common/sconv.py @@ -26,7 +26,7 @@ from sglang.srt.models.inkling_common.kernels.sconv import ( save_intermediate_conv_windows, update_sconv_cache, ) -from sglang.srt.runtime_context import get_exec, get_parallel, get_server_args +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.speculative.eagle_info import EagleDraftExtendInput from sglang.srt.utils import is_cuda, set_weight_attrs @@ -482,7 +482,7 @@ class ShortConvolution(nn.Module): crossed = track_step = None if do_tracking: - mamba_track_interval = get_exec().mamba.mamba_track_interval + mamba_track_interval = get_server_args().mamba_track_interval pre_seqlen = forward_batch.seq_lens[:batch_size] - draft_token_num post_seqlen = pre_seqlen + num_accept_tokens crossed = (pre_seqlen // mamba_track_interval) != ( diff --git a/python/sglang/srt/models/inkling_common/util.py b/python/sglang/srt/models/inkling_common/util.py index 19c27da0e..1d8bbea68 100644 --- a/python/sglang/srt/models/inkling_common/util.py +++ b/python/sglang/srt/models/inkling_common/util.py @@ -9,12 +9,12 @@ from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod -from sglang.srt.runtime_context import get_lora +from sglang.srt.runtime_context import get_server_args def lora_compatible_layout_enabled() -> bool: """Use the contiguous ``[gate || up]`` layout required by LoRA slicing.""" - return get_lora().enable_lora + return get_server_args().enable_lora def use_inkling_shared_fused_moe( diff --git a/python/sglang/srt/models/internvl.py b/python/sglang/srt/models/internvl.py index e04616689..952d21b39 100644 --- a/python/sglang/srt/models/internvl.py +++ b/python/sglang/srt/models/internvl.py @@ -46,7 +46,7 @@ from sglang.srt.multimodal.internvl_vit_cuda_graph_runner import ( InternViTCudaGraphRunner, ) from sglang.srt.multimodal.mm_utils import run_dp_sharded_vision_model -from sglang.srt.runtime_context import get_mm, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import is_cuda from sglang.utils import logger @@ -520,7 +520,7 @@ class InternVLChatModel(nn.Module): ) -> None: super().__init__() self.config = config - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder self.quant_config = quant_config vision_utils.update_vit_attn_dummy_heads_config(self.config) image_size = config.force_image_size or config.vision_config.image_size diff --git a/python/sglang/srt/models/kimi_k25.py b/python/sglang/srt/models/kimi_k25.py index 1e920ace0..47e49c8d3 100644 --- a/python/sglang/srt/models/kimi_k25.py +++ b/python/sglang/srt/models/kimi_k25.py @@ -39,7 +39,7 @@ from sglang.srt.multimodal.mm_utils import ( materialize_multimodal_features, run_dp_sharded_mrope_vision_model, ) -from sglang.srt.runtime_context import get_mm, get_parallel, get_server_args +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_npu logger = logging.getLogger(__name__) @@ -659,7 +659,7 @@ class KimiK25ForConditionalGeneration(nn.Module): super().__init__() self.config = config self.quant_config = quant_config - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder # Create vision tower self.vision_tower = MoonViT3dPretrainedModel( config.vision_config, diff --git a/python/sglang/srt/models/kimi_vl.py b/python/sglang/srt/models/kimi_vl.py index 529f7a469..27d1e1845 100644 --- a/python/sglang/srt/models/kimi_vl.py +++ b/python/sglang/srt/models/kimi_vl.py @@ -74,7 +74,7 @@ from sglang.srt.model_loader.weight_utils import ( from sglang.srt.models.deepseek_v2 import DeepseekV2ForCausalLM from sglang.srt.models.kimi_vl_moonvit import MoonVitPretrainedModel from sglang.srt.multimodal.mm_utils import run_dp_sharded_mrope_vision_model -from sglang.srt.runtime_context import get_mm +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -126,7 +126,7 @@ class KimiVLForConditionalGeneration(nn.Module): self.config = config assert isinstance(config.vision_config, MoonViTConfig) - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder self.vision_tower = MoonVitPretrainedModel( config.vision_config, prefix=add_prefix("vision_tower", prefix), diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 1c942fe61..0373fd0c1 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -17,11 +17,19 @@ import torch.nn.functional as F from torch import nn from sglang.srt.configs.laguna import LagunaConfig, normalize_gating -from sglang.srt.distributed import get_pp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed import ( + get_pp_group, + tensor_model_parallel_all_reduce, +) from sglang.srt.environ import envs from sglang.srt.layers.activation import SiluAndMul -from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.communicator import ( + LayerCommunicator, + LayerScatterModes, +) +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -45,11 +53,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.utils import apply_qk_norm -from sglang.srt.runtime_context import ( - get_exec, - get_forward, - get_parallel, -) +from sglang.srt.runtime_context import get_forward, get_parallel, get_server_args from sglang.srt.utils import LazyValue, add_prefix, make_layers logger = logging.getLogger(__name__) @@ -151,7 +155,7 @@ class LagunaMoE(nn.Module): self.gate = LagunaMoEGate(config, prefix=add_prefix("gate", prefix)) self.experts = get_moe_impl_class(quant_config)( - num_experts=config.num_experts + get_exec().moe.ep_num_redundant_experts, + num_experts=config.num_experts + get_server_args().ep_num_redundant_experts, top_k=config.num_experts_per_tok, layer_id=layer_id, hidden_size=config.hidden_size, @@ -636,7 +640,7 @@ class LagunaForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index e9eab5137..a6471de6b 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -41,7 +41,9 @@ from sglang.srt.layers.communicator import ( LayerScatterModes, enable_moe_dense_fully_dp, ) -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -75,9 +77,9 @@ from sglang.srt.models.utils import ( enable_fused_set_kv_buffer, ) from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import ( @@ -229,7 +231,7 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): self.router_dtype = torch.bfloat16 # TODO global_server_args.ep_num_redundant_experts is used for eplb, not supported now - assert get_exec().moe.ep_num_redundant_experts == 0 + assert get_server_args().ep_num_redundant_experts == 0 # check group topk self.num_expert_group = getattr(config, "n_group", 0) self.topk_group = getattr(config, "topk_group", 0) @@ -243,7 +245,9 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): self.num_expert_group = self.topk_group = None self.use_grouped_topk = False - self.num_experts = config.num_experts + get_exec().moe.ep_num_redundant_experts + self.num_experts = ( + config.num_experts + get_server_args().ep_num_redundant_experts + ) self.gate = LLaDA2MoeGate( config=config, @@ -827,7 +831,7 @@ class LLaDA2MoeModelLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config, return_full_logits=True) diff --git a/python/sglang/srt/models/llama.py b/python/sglang/srt/models/llama.py index faa6bb37c..6771c6136 100644 --- a/python/sglang/srt/models/llama.py +++ b/python/sglang/srt/models/llama.py @@ -25,7 +25,10 @@ import torch from torch import nn from transformers import LlamaConfig -from sglang.srt.distributed import get_pp_group, get_pp_indices +from sglang.srt.distributed import ( + get_pp_group, + get_pp_indices, +) from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -50,7 +53,7 @@ from sglang.srt.model_loader.weight_utils import ( maybe_remap_kv_scale_name, ) from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_cuda, is_npu, is_xpu, make_layers from sglang.utils import get_exception_traceback @@ -527,7 +530,7 @@ class LlamaForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) diff --git a/python/sglang/srt/models/llama_eagle3.py b/python/sglang/srt/models/llama_eagle3.py index 995cc2161..294710d11 100644 --- a/python/sglang/srt/models/llama_eagle3.py +++ b/python/sglang/srt/models/llama_eagle3.py @@ -38,7 +38,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.llama import LlamaDecoderLayer, LlamaForCausalLM, LlamaMLP -from sglang.srt.runtime_context import get_spec +from sglang.srt.runtime_context import get_server_args class LlamaDecoderLayer(LlamaDecoderLayer): @@ -258,7 +258,7 @@ class LlamaForCausalLMEagle3(LlamaForCausalLM): # Cache draft SWA size from server args once; consumed both by the post-init # attention patch below and by `get_attention_sliding_window_size` later. self._draft_window_size: Optional[int] = ( - get_spec().speculative_draft_window_size + get_server_args().speculative_draft_window_size ) self.model = LlamaModel( diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 27b42d4dc..7b972bbf0 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -41,13 +41,17 @@ from sglang.kernels.ops.attention.dsv4 import linear_bf16_fp32 from sglang.kernels.ops.moe.ep_moe_kernels import zero_experts_compute_triton from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz from sglang.srt.configs import LongcatFlashConfig -from sglang.srt.distributed import tensor_model_parallel_all_reduce +from sglang.srt.distributed import ( + tensor_model_parallel_all_reduce, +) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers import deep_gemm_wrapper from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -83,7 +87,7 @@ from sglang.srt.model_loader.utils import ( ) from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.deepseek_v2 import DeepseekV2AttentionMLA -from sglang.srt.runtime_context import get_parallel, get_stream +from sglang.srt.runtime_context import get_parallel, get_server_args, get_stream from sglang.srt.utils import ( BumpAllocator, add_prefix, @@ -710,7 +714,7 @@ class LongcatFlashForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/mellum.py b/python/sglang/srt/models/mellum.py index 625bef1cf..ff62c558d 100644 --- a/python/sglang/srt/models/mellum.py +++ b/python/sglang/srt/models/mellum.py @@ -51,7 +51,7 @@ from sglang.srt.models.utils import ( create_fused_set_kv_buffer_arg, enable_fused_set_kv_buffer, ) -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_cuda _is_cuda = is_cuda() @@ -231,7 +231,7 @@ class MellumAttention(Qwen3MoeAttention): _yarn_factor = self._yarn_params["factor"] self.use_fused_qk_norm_rope = ( - get_exec().kernel.enable_fused_qk_norm_rope + get_server_args().enable_fused_qk_norm_rope and self.compatible_with_fused_qk_norm_rope and _is_cuda and can_use_fused_qk_norm_rope( @@ -520,7 +520,7 @@ class MellumForCausalLM(Qwen3MoeForCausalLM): cfg.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(cfg) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/mimo_audio.py b/python/sglang/srt/models/mimo_audio.py index 6e738b3ad..650c3309f 100644 --- a/python/sglang/srt/models/mimo_audio.py +++ b/python/sglang/srt/models/mimo_audio.py @@ -22,7 +22,7 @@ from transformers.models.qwen2.modeling_qwen2 import Qwen2Model from sglang.srt.layers.attention.vision import VisionAttention from sglang.srt.layers.quantization.base_config import QuantizationConfig -from sglang.srt.runtime_context import get_model +from sglang.srt.runtime_context import get_server_args logger = logging.getLogger(__name__) @@ -1255,7 +1255,7 @@ class AudioEncoderMixin: else: raise ValueError(f"Invalid projection layers: {config.projection_layers}") - model_path = get_model().model_path + model_path = get_server_args().model_path if not os.path.isdir(model_path): from huggingface_hub import snapshot_download diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index a3f4bb3c9..781f3093e 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -23,7 +23,10 @@ from torch import nn from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size -from sglang.srt.distributed import get_pp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed import ( + get_pp_group, + tensor_model_parallel_all_reduce, +) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -34,7 +37,9 @@ from sglang.srt.layers.communicator import ( ScatterMode, enable_moe_dense_fully_dp, ) -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -74,9 +79,9 @@ from sglang.srt.model_loader.weight_utils import ( from sglang.srt.models.mimo_audio import AudioEncoderMixin, MiMoAudioEncoderConfig from sglang.srt.models.mimo_vl import MiMoVisionTransformer, MiMoVLVisionConfig from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, + get_server_args, ) from sglang.srt.utils import ( LazyValue, @@ -408,7 +413,7 @@ class MiMoV2MoE(nn.Module): experts_type = get_moe_impl_class(quant_config) self.experts = experts_type( num_experts=config.n_routed_experts - + get_exec().moe.ep_num_redundant_experts, + + get_server_args().ep_num_redundant_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, intermediate_size=config.moe_intermediate_size, @@ -443,7 +448,7 @@ class MiMoV2MoE(nn.Module): # TODO: we will support tp < ep in the future self.ep_size = get_parallel().moe_ep_size self.num_experts = ( - config.n_routed_experts + get_exec().moe.ep_num_redundant_experts + config.n_routed_experts + get_server_args().ep_num_redundant_experts ) self.renormalize = config.norm_topk_prob self.topk_group = config.topk_group @@ -1186,7 +1191,7 @@ class MiMoV2ForCausalLM(nn.Module, AudioEncoderMixin): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() diff --git a/python/sglang/srt/models/mimo_v2_nextn.py b/python/sglang/srt/models/mimo_v2_nextn.py index 9207938d6..49d32b66f 100644 --- a/python/sglang/srt/models/mimo_v2_nextn.py +++ b/python/sglang/srt/models/mimo_v2_nextn.py @@ -26,7 +26,9 @@ from sglang.srt.layers.communicator import ( LayerScatterModes, enable_moe_dense_fully_dp, ) -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -42,7 +44,7 @@ from sglang.srt.models.mimo_v2 import ( MiMoV2MLP, load_mimo_v2_qkv_proj_weight, ) -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix MiMoV2Config = None @@ -257,7 +259,7 @@ class MiMoV2MTP(MiMoV2ForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/mimo_vl.py b/python/sglang/srt/models/mimo_vl.py index 3f4c7267b..b36bc9616 100644 --- a/python/sglang/srt/models/mimo_vl.py +++ b/python/sglang/srt/models/mimo_vl.py @@ -22,7 +22,7 @@ from sglang.srt.layers.attention.vision import ( from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.quantization import QuantizationConfig from sglang.srt.models.qwen2_5_vl import Qwen2_5_VisionPatchMerger, Qwen2_5_VLMLP -from sglang.srt.runtime_context import get_mm, get_server_args +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import add_prefix @@ -258,7 +258,7 @@ class MiMoVisionTransformer(nn.Module): self.fullatt_block_indexes = vision_config.fullatt_block_indexes self.window_size = vision_config.window_size self.patch_size = vision_config.patch_size - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = self.server_args.mm_enable_dp_encoder mlp_hidden_size: int = vision_config.intermediate_size self.patch_embed = MiMoVisionPatchEmbed( patch_size=patch_size, diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index abd228bfe..ea4a6b5be 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -32,7 +32,10 @@ from sglang.kernels.ops.communication.all_reduce import ( get_fused_parallel_qknorm_max_occupancy, ) from sglang.srt.batch_overlap.two_batch_overlap import model_forward_maybe_tbo -from sglang.srt.distributed import get_pp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed import ( + get_pp_group, + tensor_model_parallel_all_reduce, +) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.communicator import ( @@ -40,7 +43,10 @@ from sglang.srt.layers.communicator import ( LayerScatterModes, ScatterMode, ) -from sglang.srt.layers.dp_attention import attn_tp_all_reduce, is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + attn_tp_all_reduce, + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -74,12 +80,7 @@ from sglang.srt.model_loader.weight_utils import ( maybe_remap_kv_scale_name, narrow_padded_param_and_loaded_weight, ) -from sglang.srt.runtime_context import ( - get_exec, - get_forward, - get_parallel, - get_server_args, -) +from sglang.srt.runtime_context import get_forward, get_parallel, get_server_args # get_bool_env_var is defined in sglang.srt.utils.common, not sglang.srt.distributed. # Importing from the wrong module causes this file to fail import, which prevents the @@ -512,7 +513,7 @@ class MiniMaxM2MoE(nn.Module): self.experts = get_moe_impl_class(quant_config)( num_experts=config.num_local_experts - + get_exec().moe.ep_num_redundant_experts, + + get_server_args().ep_num_redundant_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, intermediate_size=config.intermediate_size, diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 73d0db535..3ccafeeb4 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -28,7 +28,10 @@ from sglang.srt.configs.model_config import ( get_minimax_sparse_disable_value_layer_ids, get_minimax_sparse_layer_ids, ) -from sglang.srt.distributed import get_pp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed import ( + get_pp_group, + tensor_model_parallel_all_reduce, +) from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -76,7 +79,7 @@ from sglang.srt.model_loader.weight_utils import ( maybe_remap_kv_scale_name, ) from sglang.srt.models.minimax_m2 import MiniMaxM2RMSNormTP -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import ( add_prefix, get_device_sm, @@ -284,7 +287,7 @@ class MiniMaxM3MoE(nn.Module): self.n_shared_experts = getattr(config, "n_shared_experts", None) self.num_fused_shared_experts = ( 0 - if get_exec().moe.disable_shared_experts_fusion + if get_server_args().disable_shared_experts_fusion else config.n_shared_experts ) @@ -308,7 +311,7 @@ class MiniMaxM3MoE(nn.Module): self.experts = get_moe_impl_class(quant_config)( num_experts=config.num_local_experts + self.num_fused_shared_experts - + get_exec().moe.ep_num_redundant_experts, + + get_server_args().ep_num_redundant_experts, num_fused_shared_experts=self.num_fused_shared_experts, top_k=config.num_experts_per_tok + self.num_fused_shared_experts, hidden_size=config.hidden_size, @@ -1440,7 +1443,7 @@ class MiniMaxM3SparseForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) @@ -1453,7 +1456,7 @@ class MiniMaxM3SparseForCausalLM(nn.Module): return self.model.get_input_embeddings() def determine_num_fused_shared_experts(self): - if get_exec().moe.disable_shared_experts_fusion: + if get_server_args().disable_shared_experts_fusion: return disable_reason = None diff --git a/python/sglang/srt/models/minimax_m3_vl.py b/python/sglang/srt/models/minimax_m3_vl.py index 3b0fe7b0b..c3d563962 100644 --- a/python/sglang/srt/models/minimax_m3_vl.py +++ b/python/sglang/srt/models/minimax_m3_vl.py @@ -6,7 +6,9 @@ from typing import Iterable, List, Optional, Tuple import torch import torch.nn as nn -from sglang.srt.distributed import get_pp_group +from sglang.srt.distributed import ( + get_pp_group, +) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.utils import get_moe_a2a_backend from sglang.srt.layers.quantization.base_config import QuantizationConfig @@ -17,7 +19,10 @@ from sglang.srt.managers.mm_utils import ( MultiModalityDataPaddingPatternMultimodalTokens, general_mm_embed_routine, ) -from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs +from sglang.srt.managers.schedule_batch import ( + MultimodalDataItem, + MultimodalInputs, +) from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import ( default_weight_loader, @@ -37,7 +42,7 @@ from sglang.srt.models.minimax_vl_common import ( load_vision_weight, merge_vit_qkv_weights, ) -from sglang.srt.runtime_context import get_mm, get_parallel, get_server_args +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, get_device_sm, is_cuda, log_info_on_rank0 from sglang.srt.utils.hf_transformers_utils import get_rope_config @@ -60,7 +65,7 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module): self.quant_config = quant_config self.pp_group = get_pp_group() - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder self.num_fused_shared_experts = 0 self._determine_num_fused_shared_experts() @@ -105,7 +110,7 @@ class MiniMaxM3SparseForConditionalGeneration(nn.Module): text_config.hidden_size, quant_config=quant_config, prefix=add_prefix("language_model.lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) else: self.lm_head = PPMissingLayer() diff --git a/python/sglang/srt/models/minimax_vl_common.py b/python/sglang/srt/models/minimax_vl_common.py index 68f811578..0987165b7 100644 --- a/python/sglang/srt/models/minimax_vl_common.py +++ b/python/sglang/srt/models/minimax_vl_common.py @@ -18,13 +18,16 @@ from sglang.srt.layers.attention.vision import ( prepare_vision_attention_metadata, ) from sglang.srt.layers.dp_attention import is_dp_attention_enabled -from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear +from sglang.srt.layers.linear import ( + ColumnParallelLinear, + RowParallelLinear, +) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.rotary_embedding.utils import rotate_half from sglang.srt.managers.schedule_batch import MultimodalDataItem from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.multimodal.mm_utils import run_dp_sharded_mrope_vision_model -from sglang.srt.runtime_context import get_mm, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, get_compiler_backend, round_up logger = logging.getLogger(__name__) @@ -410,7 +413,7 @@ class MiniMaxVLVisionTransformer(nn.Module): workspace_buffer: Optional[torch.Tensor] = None if ( - get_mm().mm_attention_backend == "flashinfer_cudnn" + get_server_args().mm_attention_backend == "flashinfer_cudnn" and torch.cuda.is_available() ): workspace_buffer = torch.empty( @@ -676,7 +679,7 @@ class MiniMaxVLVisionTransformer(nn.Module): max_seqlen: Optional[int] = None sequence_lengths: Optional[torch.Tensor] = None encoder_cu_seq_len = cu_seq_len - if get_mm().mm_attention_backend == "flashinfer_cudnn": + if get_server_args().mm_attention_backend == "flashinfer_cudnn": ( encoder_cu_seq_len, sequence_lengths, @@ -688,7 +691,7 @@ class MiniMaxVLVisionTransformer(nn.Module): device=hidden_states.device, packed_indptrs=( encoder_cu_seq_len - if get_mm().mm_attention_backend == "flashinfer_cudnn" + if get_server_args().mm_attention_backend == "flashinfer_cudnn" else None ), sequence_lengths=sequence_lengths, @@ -720,7 +723,7 @@ class MiniMaxVLVisionModel(nn.Module): self.config = config self.quant_config = quant_config - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder self.vision_config = config self.vision_model = MiniMaxVLVisionTransformer( diff --git a/python/sglang/srt/models/mllama4.py b/python/sglang/srt/models/mllama4.py index 28ddcdc87..68fc8f6db 100644 --- a/python/sglang/srt/models/mllama4.py +++ b/python/sglang/srt/models/mllama4.py @@ -33,7 +33,7 @@ from sglang.srt.managers.schedule_batch import ( MultimodalInputs, ) from sglang.srt.model_executor.forward_batch_info import ForwardBatch -from sglang.srt.runtime_context import get_mm +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import is_cpu _is_cpu = is_cpu() @@ -476,7 +476,9 @@ class Llama4ForConditionalGeneration(nn.Module): "Please not that this warning might be inaccurate if the weights haven't been fully downloaded" ) - self.has_vision = self.has_vision_weights and get_mm().enable_multimodal + self.has_vision = ( + self.has_vision_weights and get_server_args().enable_multimodal + ) if self.has_vision: # TODO: make this more general diff --git a/python/sglang/srt/models/moss_vl.py b/python/sglang/srt/models/moss_vl.py index 68976d6b4..9b04905f5 100644 --- a/python/sglang/srt/models/moss_vl.py +++ b/python/sglang/srt/models/moss_vl.py @@ -34,7 +34,10 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention -from sglang.srt.layers.rotary_embedding import MRotaryEmbedding, get_rope +from sglang.srt.layers.rotary_embedding import ( + MRotaryEmbedding, + get_rope, +) from sglang.srt.layers.rotary_embedding.mrope import apply_interleaved_rope from sglang.srt.layers.rotary_embedding.utils import apply_rotary_emb from sglang.srt.layers.vocab_parallel_embedding import ( @@ -45,7 +48,7 @@ from sglang.srt.managers.schedule_batch import MultimodalInputs from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -999,7 +1002,7 @@ class MossVLSelfAttentionDecoderLayer(nn.Module): override_orig_dtype=torch.float32, fp32_residual=True, ) - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else {} ) self.input_layernorm = RMSNorm( diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index 00a405d4a..f678c5c32 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -36,7 +36,10 @@ from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( Mamba2AttnBackend, ) from sglang.srt.layers.attention.mamba.mamba import MambaMixer2 -from sglang.srt.layers.dp_attention import attn_tp_all_reduce, is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + attn_tp_all_reduce, + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -86,11 +89,7 @@ from sglang.srt.models.nemotron_h_utils import ( pad_to_original_num_tokens, ) from sglang.srt.models.utils import WeightsMapper -from sglang.srt.runtime_context import ( - get_exec, - get_forward, - get_parallel, -) +from sglang.srt.runtime_context import get_forward, get_parallel, get_server_args from sglang.srt.utils import ( add_prefix, get_current_device_stream_fast, @@ -201,7 +200,7 @@ class NemotronHMoE(nn.Module): self.experts = get_moe_impl_class(quant_config)( num_experts=config.n_routed_experts - + get_exec().moe.ep_num_redundant_experts, + + get_server_args().ep_num_redundant_experts, top_k=config.num_experts_per_tok, hidden_size=self.moe_hidden_size, intermediate_size=config.moe_intermediate_size, @@ -938,7 +937,7 @@ class NemotronHForCausalLM(nn.Module): else lora_config.lora_vocab_padding_size ), quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/nemotron_h_mtp.py b/python/sglang/srt/models/nemotron_h_mtp.py index ef414c3f9..d38de9b9d 100644 --- a/python/sglang/srt/models/nemotron_h_mtp.py +++ b/python/sglang/srt/models/nemotron_h_mtp.py @@ -19,7 +19,10 @@ from torch import nn from sglang.srt.configs import NemotronHConfig from sglang.srt.distributed import get_pp_group -from sglang.srt.layers.dp_attention import attn_tp_all_reduce, is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + attn_tp_all_reduce, + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ColumnParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor @@ -35,7 +38,7 @@ from sglang.srt.models.nemotron_h import ( NemotronHMoEDecoderLayer, ) from sglang.srt.models.nemotron_h_utils import is_attn_layer -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix @@ -335,7 +338,7 @@ class NemotronHForCausalLMMTP(NemotronHForCausalLM): self.config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/qwen2.py b/python/sglang/srt/models/qwen2.py index 7e61333f2..14e2601a7 100644 --- a/python/sglang/srt/models/qwen2.py +++ b/python/sglang/srt/models/qwen2.py @@ -22,7 +22,10 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Union import torch from torch import nn -from sglang.srt.distributed import get_pp_group, get_pp_indices +from sglang.srt.distributed import ( + get_pp_group, + get_pp_indices, +) from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.dp_attention import is_dp_attention_enabled from sglang.srt.layers.layernorm import RMSNorm @@ -47,7 +50,7 @@ from sglang.srt.model_loader.weight_utils import ( kv_cache_scales_loader, ) from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, make_layers from sglang.srt.utils.hf_transformers_utils import get_rope_config @@ -93,7 +96,7 @@ class Qwen2MLP(nn.Module): x: torch.Tensor, forward_batch: ForwardBatch = None, ) -> torch.Tensor: - if get_exec().deterministic.rl_on_policy_target is not None: + if get_server_args().rl_on_policy_target is not None: x = x.bfloat16() gate_up, _ = self.gate_up_proj(x) @@ -327,7 +330,7 @@ class Qwen2Model(nn.Module): prefix=add_prefix("embed_tokens", prefix), params_dtype=( torch.float32 - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else None ), ) @@ -363,7 +366,7 @@ class Qwen2Model(nn.Module): override_orig_dtype=torch.float32, fp32_residual=True, ) - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else {} ) self.norm = RMSNorm( diff --git a/python/sglang/srt/models/qwen2_5_vl.py b/python/sglang/srt/models/qwen2_5_vl.py index 34fca18ce..bba029f69 100644 --- a/python/sglang/srt/models/qwen2_5_vl.py +++ b/python/sglang/srt/models/qwen2_5_vl.py @@ -76,7 +76,7 @@ from sglang.srt.models.qwen2 import Qwen2Model from sglang.srt.models.utils import RotaryPosMixin, WeightsMapper, permute_inv from sglang.srt.multimodal.mm_utils import run_dp_sharded_mrope_vision_model from sglang.srt.multimodal.vit_cuda_graph_runner import ViTCudaGraphRunner -from sglang.srt.runtime_context import get_mm, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_cpu, is_cuda, is_npu _is_cuda = is_cuda() @@ -619,7 +619,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module): self.pp_group = get_pp_group() self.config = config - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder if not self.config.encoder_only: self.model = Qwen2Model( diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index a56553a50..4c35be1f7 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -46,7 +46,9 @@ from sglang.srt.layers.communicator import ( ScatterMode, ) from sglang.srt.layers.cp.utils import is_cp_v2_active -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -89,11 +91,7 @@ from sglang.srt.model_executor.cuda_graph_config import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import ( - get_exec, - get_forward, - get_parallel, -) +from sglang.srt.runtime_context import get_forward, get_parallel, get_server_args from sglang.srt.utils import ( add_prefix, cpu_has_amx_support, @@ -148,7 +146,7 @@ def can_fuse_shared_expert( Caller must still gate on the model/backend support flag. """ if ( - get_exec().moe.disable_shared_experts_fusion is True + get_server_args().disable_shared_experts_fusion is True or getattr(config, "shared_expert_intermediate_size", 0) <= 0 or config.shared_expert_intermediate_size != config.moe_intermediate_size or get_moe_a2a_backend().is_deepep() @@ -273,10 +271,10 @@ class Qwen2MoeSparseMoeBlock(nn.Module): else config.num_experts_per_tok + self.num_fused_shared_experts ), num_experts=( - config.num_experts + get_exec().moe.ep_num_redundant_experts + config.num_experts + get_server_args().ep_num_redundant_experts if not self.enable_shared_expert_fusion else config.num_experts - + get_exec().moe.ep_num_redundant_experts + + get_server_args().ep_num_redundant_experts + self.num_fused_shared_experts ), hidden_size=config.hidden_size, @@ -335,7 +333,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): # TODO: we will support tp < ep in the future self.ep_size = get_parallel().moe_ep_size self.num_experts = ( - config.num_experts + get_exec().moe.ep_num_redundant_experts + config.num_experts + get_server_args().ep_num_redundant_experts ) self.top_k = config.num_experts_per_tok self.is_nextn = is_nextn @@ -1010,7 +1008,7 @@ class Qwen2MoeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # For EAGLE3 support diff --git a/python/sglang/srt/models/qwen3.py b/python/sglang/srt/models/qwen3.py index 7c05fda69..daaca2276 100644 --- a/python/sglang/srt/models/qwen3.py +++ b/python/sglang/srt/models/qwen3.py @@ -5,7 +5,9 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple import torch from torch import nn -from sglang.srt.distributed import get_pp_group +from sglang.srt.distributed import ( + get_pp_group, +) from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import QKVParallelLinear, RowParallelLinear @@ -31,11 +33,7 @@ from sglang.srt.model_loader.weight_utils import ( from sglang.srt.models.qwen2 import Qwen2MLP as Qwen3MLP from sglang.srt.models.qwen2 import Qwen2Model from sglang.srt.models.utils import apply_qk_norm -from sglang.srt.runtime_context import ( - get_exec, - get_parallel, - get_stream, -) +from sglang.srt.runtime_context import get_parallel, get_server_args, get_stream from sglang.srt.utils import add_prefix, get_bool_env_var, is_cuda, is_hip, is_npu Qwen3Config = None @@ -113,7 +111,7 @@ class Qwen3Attention(nn.Module): weight_dtype=torch.float32, cast_x_before_out_mul=True, ) - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else {} ) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps, **norm_kwargs) @@ -273,14 +271,14 @@ class Qwen3Attention(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - if get_exec().deterministic.rl_on_policy_target is not None: + if get_server_args().rl_on_policy_target is not None: hidden_states = hidden_states.bfloat16() save_kv_cache = True use_aiter_fused = ( self.use_fused_qk_norm_mrope and forward_batch.forward_mode.is_decode() - and get_exec().deterministic.rl_on_policy_target is None + and get_server_args().rl_on_policy_target is None ) if use_aiter_fused: @@ -300,7 +298,7 @@ class Qwen3Attention(nn.Module): forward_batch=forward_batch, ) - if get_exec().deterministic.rl_on_policy_target is not None: + if get_server_args().rl_on_policy_target is not None: q = q.to(torch.bfloat16) k = k.to(torch.bfloat16) @@ -364,7 +362,7 @@ class Qwen3DecoderLayer(nn.Module): override_orig_dtype=torch.float32, fp32_residual=True, ) - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else {} ) self.input_layernorm = RMSNorm( @@ -494,7 +492,7 @@ class Qwen3ForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 687bf77a9..ecbb4c239 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -42,7 +42,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) # Layers - Others from sglang.srt.layers.layernorm import GemmaRMSNorm @@ -89,7 +91,6 @@ from sglang.srt.models.utils import ( fused_qk_gemma_rmsnorm_with_gate, ) from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, get_server_args, @@ -137,7 +138,7 @@ cached_get_processor = lru_cache(get_processor) def _disable_shared_experts_fusion() -> bool: # Resolved lazily: the global server args is not set at module import time # (e.g. when this module is imported by unit tests). - return get_exec().moe.disable_shared_experts_fusion + return get_server_args().disable_shared_experts_fusion if _is_cuda: @@ -1310,7 +1311,7 @@ class Qwen3_5ForCausalLM(nn.Module): # so the model still gets the #25885 multi-streaming path. ROCm-only. if ( config.model_type == "qwen3_5_moe_text" - and not get_exec().moe.disable_shared_experts_fusion + and not get_server_args().disable_shared_experts_fusion and not can_fuse_shared_expert(config, quant_config) ): from sglang.srt.arg_groups.overrides import declare_load_time_override diff --git a/python/sglang/srt/models/qwen3_5_mtp.py b/python/sglang/srt/models/qwen3_5_mtp.py index ded5ef3b8..ebda7517f 100644 --- a/python/sglang/srt/models/qwen3_5_mtp.py +++ b/python/sglang/srt/models/qwen3_5_mtp.py @@ -34,11 +34,7 @@ from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.qwen3_5 import Qwen3_5ForCausalLM -from sglang.srt.runtime_context import ( - get_model, - get_parallel, - get_spec, -) +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_npu logger = logging.getLogger(__name__) @@ -67,7 +63,7 @@ class Qwen3_5ForCausalLMMTP(nn.Module): "modelopt_mixed", ): quant_config = None - if is_npu() and get_spec().speculative_draft_model_quantization is None: + if is_npu() and get_server_args().speculative_draft_model_quantization is None: quant_config = None # Quark-quantized Qwen3.5 MXFP4 checkpoints ship the MTP module in @@ -157,7 +153,7 @@ class Qwen3_5ForCausalLMMTP(nn.Module): if ( is_npu() and self.quant_config is None - and get_model().quantization is not None + and get_server_args().quantization is not None ): # ascend mtp unquant exit_stack.enter_context(envs.SGLANG_DEEPEP_BF16_DISPATCH.override(True)) diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 0123b702a..bb8a4d660 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -73,9 +73,9 @@ from sglang.srt.models.utils import ( enable_fused_set_kv_buffer, ) from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import ( @@ -261,7 +261,7 @@ class Qwen3MoeSparseMoeBlock(nn.Module): ) self.experts = get_moe_impl_class(quant_config)( - num_experts=config.num_experts + get_exec().moe.ep_num_redundant_experts, + num_experts=config.num_experts + get_server_args().ep_num_redundant_experts, top_k=config.num_experts_per_tok, layer_id=layer_id, hidden_size=config.hidden_size, @@ -283,7 +283,7 @@ class Qwen3MoeSparseMoeBlock(nn.Module): # TODO: we will support tp < ep in the future self.ep_size = get_parallel().moe_ep_size self.num_experts = ( - config.num_experts + get_exec().moe.ep_num_redundant_experts + config.num_experts + get_server_args().ep_num_redundant_experts ) self.top_k = config.num_experts_per_tok @@ -514,7 +514,7 @@ class Qwen3MoeAttention(nn.Module): ) and self.head_dim in (64, 128, 256) _yarn_factor, _, _, _ = compute_yarn_parameters(config) self.use_fused_qk_norm_rope = ( - get_exec().kernel.enable_fused_qk_norm_rope + get_server_args().enable_fused_qk_norm_rope and self.compatible_with_fused_qk_norm_rope and _is_cuda and can_use_fused_qk_norm_rope( @@ -955,7 +955,7 @@ class Qwen3MoeForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) self.capture_aux_hidden_states = False diff --git a/python/sglang/srt/models/qwen3_moe_mtp.py b/python/sglang/srt/models/qwen3_moe_mtp.py index e351fb4d7..6f6ec6091 100644 --- a/python/sglang/srt/models/qwen3_moe_mtp.py +++ b/python/sglang/srt/models/qwen3_moe_mtp.py @@ -30,7 +30,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.qwen3_moe import Qwen3MoeForCausalLM, Qwen3MoeModel -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix logger = logging.getLogger(__name__) @@ -63,7 +63,7 @@ class Qwen3MoeForCausalLMMTP(Qwen3MoeForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/qwen3_next.py b/python/sglang/srt/models/qwen3_next.py index 74772592d..5022107d7 100644 --- a/python/sglang/srt/models/qwen3_next.py +++ b/python/sglang/srt/models/qwen3_next.py @@ -17,7 +17,9 @@ from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_r from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.attention.mamba.mamba import mamba_v2_sharded_weight_loader from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -50,6 +52,7 @@ from sglang.srt.models.qwen2_moe import Qwen2MoeMLP, Qwen2MoeSparseMoeBlock from sglang.srt.runtime_context import ( get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import ( @@ -1029,7 +1032,7 @@ class Qwen3NextForCausalLM(nn.Module): quant_config=quant_config, org_num_embeddings=config.vocab_size, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # For EAGLE3 support diff --git a/python/sglang/srt/models/qwen3_next_mtp.py b/python/sglang/srt/models/qwen3_next_mtp.py index ce3149661..ef33eb9a1 100644 --- a/python/sglang/srt/models/qwen3_next_mtp.py +++ b/python/sglang/srt/models/qwen3_next_mtp.py @@ -32,11 +32,7 @@ from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.vocab_parallel_embedding import ParallelLMHead from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.qwen3_next import Qwen3NextForCausalLM, Qwen3NextModel -from sglang.srt.runtime_context import ( - get_model, - get_parallel, - get_spec, -) +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import add_prefix, is_npu logger = logging.getLogger(__name__) @@ -55,7 +51,7 @@ class Qwen3NextForCausalLMMTP(Qwen3NextForCausalLM): config = copy.deepcopy(config) self.config = config self.tp_size = get_parallel().tp_size - if is_npu() and get_spec().speculative_draft_model_quantization is None: + if is_npu() and get_server_args().speculative_draft_model_quantization is None: quant_config = None self.quant_config = quant_config # if not set, model load will be broken in Qwen3NextForCausalLM load_weights() @@ -84,7 +80,7 @@ class Qwen3NextForCausalLMMTP(Qwen3NextForCausalLM): config.hidden_size, quant_config=quant_config, prefix=add_prefix("model.shared_head.head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) # Mirror Qwen3NextForCausalLM.__init__'s shared-expert fusion setup so @@ -114,7 +110,7 @@ class Qwen3NextForCausalLMMTP(Qwen3NextForCausalLM): if ( is_npu() and self.quant_config is None - and get_model().quantization is not None + and get_server_args().quantization is not None ): # ascend mtp unquant exit_stack.enter_context(envs.SGLANG_DEEPEP_BF16_DISPATCH.override(True)) diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 7454d7aad..7036cd717 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -38,7 +38,9 @@ from sglang.srt.layers.attention.vision import ( prepare_vision_attention_metadata, ) from sglang.srt.layers.conv import Conv3dLayer -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.linear import ColumnParallelLinear, RowParallelLinear from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.pooler import Pooler, PoolingType @@ -68,8 +70,14 @@ from sglang.srt.models.utils import ( ) from sglang.srt.multimodal.mm_utils import run_dp_sharded_mrope_vision_model from sglang.srt.multimodal.vit_cuda_graph_runner import ViTCudaGraphRunner -from sglang.srt.runtime_context import get_exec, get_mm, get_parallel -from sglang.srt.utils import add_prefix, cpu_has_amx_support, is_cpu, is_npu, round_up +from sglang.srt.runtime_context import get_parallel, get_server_args +from sglang.srt.utils import ( + add_prefix, + cpu_has_amx_support, + is_cpu, + is_npu, + round_up, +) from sglang.srt.utils.hf_transformers_utils import get_processor _is_npu = is_npu() @@ -318,7 +326,7 @@ class Qwen3VLMoeVisionModel(nn.Module, RotaryPosMixin): self.num_position_embeddings = vision_config.num_position_embeddings self.num_grid_per_side = int(self.num_position_embeddings**0.5) self.num_grid = self.num_grid_per_side * self.num_grid_per_side - self.align_corners = get_exec().kernel.enable_precise_embedding_interpolation + self.align_corners = get_server_args().enable_precise_embedding_interpolation self.patch_size = vision_config.patch_size self.spatial_merge_size = vision_config.spatial_merge_size self.spatial_merge_unit = self.spatial_merge_size**2 @@ -361,7 +369,7 @@ class Qwen3VLMoeVisionModel(nn.Module, RotaryPosMixin): ) workspace_buffer = None - if get_mm().mm_attention_backend == "flashinfer_cudnn": + if get_server_args().mm_attention_backend == "flashinfer_cudnn": if torch.cuda.is_available() and (not _is_npu): ws_device = torch.device("cuda", torch.cuda.current_device()) else: @@ -909,7 +917,7 @@ class Qwen3VLMoeVisionModel(nn.Module, RotaryPosMixin): flashinfer_sequence_lengths = None flashinfer_max_seqlen = 0 - if get_mm().mm_attention_backend == "flashinfer_cudnn": + if get_server_args().mm_attention_backend == "flashinfer_cudnn": # real token lens (B,) real_seq_lens = token_cu_seqlens[1:] - token_cu_seqlens[:-1] flashinfer_max_seqlen = self.bucket_flashinfer_max_seqlen( @@ -1225,7 +1233,7 @@ class Qwen3VLForConditionalGeneration(nn.Module): self.pp_group = get_pp_group() self.quant_config = quant_config - self.use_data_parallel = get_mm().mm_enable_dp_encoder + self.use_data_parallel = get_server_args().mm_enable_dp_encoder self.visual = Qwen3VLMoeVisionModel( config.vision_config, @@ -1269,7 +1277,7 @@ class Qwen3VLForConditionalGeneration(nn.Module): self.config.vocab_size, self.config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index 674c859cf..11ad6468c 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -13,7 +13,10 @@ from torch import nn from transformers import PretrainedConfig from sglang.kernels.ops.attention.utils import concat_and_cast_mha_k_triton -from sglang.srt.distributed import get_pp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed import ( + get_pp_group, + tensor_model_parallel_all_reduce, +) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.layers.activation import SiluAndMul @@ -22,7 +25,9 @@ from sglang.srt.layers.communicator import ( LayerScatterModes, enable_moe_dense_fully_dp, ) -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -56,7 +61,6 @@ from sglang.srt.models.deepseek_common.attention_forward_methods.forward_mha imp DeepseekMHAForwardMixin, ) from sglang.srt.runtime_context import ( - get_exec, get_forward, get_model, get_parallel, @@ -270,7 +274,7 @@ class SarvamMoESparseMoeBlock(nn.Module): ) self.experts = get_moe_impl_class(quant_config)( - num_experts=config.num_experts + get_exec().moe.ep_num_redundant_experts, + num_experts=config.num_experts + get_server_args().ep_num_redundant_experts, top_k=config.num_experts_per_tok, hidden_size=config.hidden_size, intermediate_size=config.moe_intermediate_size, @@ -1226,7 +1230,7 @@ class SarvamMLAForCausalLM(nn.Module): config.hidden_size, quant_config=quant_config, prefix=add_prefix("lm_head", prefix), - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, ) self.logits_processor = LogitsProcessor(config) diff --git a/python/sglang/srt/models/sdar.py b/python/sglang/srt/models/sdar.py index 7c5d22a56..91946db54 100644 --- a/python/sglang/srt/models/sdar.py +++ b/python/sglang/srt/models/sdar.py @@ -13,7 +13,9 @@ from transformers import PretrainedConfig from sglang.srt.distributed import get_pp_group from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( MergedColumnParallelLinear, @@ -40,9 +42,9 @@ from sglang.srt.models.utils import ( enable_fused_set_kv_buffer, ) from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import add_prefix, is_cuda, make_layers @@ -205,7 +207,7 @@ class SDARAttention(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, ): - if get_exec().deterministic.rl_on_policy_target is not None: + if get_server_args().rl_on_policy_target is not None: hidden_states = hidden_states.bfloat16() qkv, _ = self.qkv_proj(hidden_states) @@ -233,7 +235,7 @@ class SDARAttention(nn.Module): ), ) - if get_exec().deterministic.rl_on_policy_target is not None: + if get_server_args().rl_on_policy_target is not None: q = q.to(torch.bfloat16) k = k.to(torch.bfloat16) @@ -268,7 +270,7 @@ class SDARBlock(nn.Module): override_orig_dtype=torch.float32, fp32_residual=True, ) - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else {} ) self.input_layernorm = RMSNorm( @@ -393,7 +395,7 @@ class SDARModel(nn.Module): override_orig_dtype=torch.float32, fp32_residual=True, ) - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else {} ) self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps, **norm_kwargs) @@ -472,7 +474,7 @@ class SDARForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index 436634f1f..b91db7be3 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -10,12 +10,17 @@ import torch from torch import nn from transformers import PretrainedConfig -from sglang.srt.distributed import get_pp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed import ( + get_pp_group, + tensor_model_parallel_all_reduce, +) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( QKVParallelLinear, @@ -53,9 +58,9 @@ from sglang.srt.models.utils import ( enable_fused_set_kv_buffer, ) from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import LazyValue, add_prefix, is_cuda, make_layers @@ -96,7 +101,7 @@ class SDARMoeSparseMoeBlock(nn.Module): ) self.experts = get_moe_impl_class(quant_config)( - num_experts=config.num_experts + get_exec().moe.ep_num_redundant_experts, + num_experts=config.num_experts + get_server_args().ep_num_redundant_experts, top_k=config.num_experts_per_tok, layer_id=layer_id, hidden_size=config.hidden_size, @@ -118,7 +123,7 @@ class SDARMoeSparseMoeBlock(nn.Module): if get_moe_a2a_backend().is_deepep(): self.ep_size = get_parallel().moe_ep_size self.num_experts = ( - config.num_experts + get_exec().moe.ep_num_redundant_experts + config.num_experts + get_server_args().ep_num_redundant_experts ) self.top_k = config.num_experts_per_tok @@ -269,7 +274,7 @@ class SDARMoeAttention(nn.Module): hidden_states: torch.Tensor, forward_batch: ForwardBatch, ) -> torch.Tensor: - if get_exec().deterministic.rl_on_policy_target is not None: + if get_server_args().rl_on_policy_target is not None: hidden_states = hidden_states.bfloat16() qkv, _ = self.qkv_proj(hidden_states) @@ -297,7 +302,7 @@ class SDARMoeAttention(nn.Module): ), ) - if get_exec().deterministic.rl_on_policy_target is not None: + if get_server_args().rl_on_policy_target is not None: q = q.to(torch.bfloat16) k = k.to(torch.bfloat16) @@ -333,7 +338,7 @@ class SDARMoeBlock(nn.Module): override_orig_dtype=torch.float32, fp32_residual=True, ) - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else {} ) self.input_layernorm = RMSNorm( @@ -473,7 +478,7 @@ class SDARMoeModel(nn.Module): override_orig_dtype=torch.float32, fp32_residual=True, ) - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else {} ) self.norm = RMSNorm(self.embed_dim, eps=config.rms_norm_eps, **norm_kwargs) @@ -556,7 +561,7 @@ class SDARMoeForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 103d0bf78..117b795bd 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -4,13 +4,18 @@ import torch import torch.nn.functional as F from torch import nn -from sglang.srt.distributed import get_pp_group, tensor_model_parallel_all_reduce +from sglang.srt.distributed import ( + get_pp_group, + tensor_model_parallel_all_reduce, +) from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo from sglang.srt.layers.activation import SiluAndMul from sglang.srt.layers.communicator import LayerCommunicator, LayerScatterModes -from sglang.srt.layers.dp_attention import is_dp_attention_enabled +from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, +) from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.linear import ( ColumnParallelLinear, @@ -42,9 +47,9 @@ from sglang.srt.layers.vocab_parallel_embedding import ( from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.runtime_context import ( - get_exec, get_forward, get_parallel, + get_server_args, get_stream, ) from sglang.srt.utils import add_prefix, is_cuda, is_non_idle_and_non_empty, make_layers @@ -148,7 +153,7 @@ class Step3p5MoEMLP(nn.Module): self.experts = get_moe_impl_class(quant_config)( num_experts=config.moe_num_experts - + get_exec().moe.ep_num_redundant_experts, + + get_server_args().ep_num_redundant_experts, top_k=config.moe_top_k, layer_id=layer_id, hidden_size=config.hidden_size, @@ -171,7 +176,7 @@ class Step3p5MoEMLP(nn.Module): # TODO: we will support tp < ep in the future self.ep_size = get_parallel().moe_ep_size self.moe_num_experts = ( - config.moe_num_experts + get_exec().moe.ep_num_redundant_experts + config.moe_num_experts + get_server_args().ep_num_redundant_experts ) self.top_k = config.moe_top_k @@ -671,7 +676,7 @@ class Step3p5Model(nn.Module): prefix=add_prefix("embed_tokens", prefix), params_dtype=( torch.float32 - if get_exec().deterministic.rl_on_policy_target is not None + if get_server_args().rl_on_policy_target is not None else None ), ) @@ -816,7 +821,7 @@ class Step3p5ForCausalLM(nn.Module): config.vocab_size, config.hidden_size, quant_config=quant_config, - use_attn_tp_group=get_parallel().enable_dp_lm_head, + use_attn_tp_group=get_server_args().enable_dp_lm_head, prefix=add_prefix("lm_head", prefix), ) else: diff --git a/python/sglang/srt/models/transformers.py b/python/sglang/srt/models/transformers.py index 3ee32ba40..e8e801477 100644 --- a/python/sglang/srt/models/transformers.py +++ b/python/sglang/srt/models/transformers.py @@ -58,12 +58,14 @@ from sglang.srt.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from sglang.srt.managers.mm_utils import MultiModalityDataPaddingPatternMultimodalTokens +from sglang.srt.managers.mm_utils import ( + MultiModalityDataPaddingPatternMultimodalTokens, +) from sglang.srt.managers.schedule_batch import MultimodalDataItem, MultimodalInputs from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_loader.weight_utils import default_weight_loader from sglang.srt.models.utils import AutoWeightsLoader, WeightsMapper -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel, get_server_args from sglang.srt.utils import get_device from sglang.srt.utils.common import direct_register_custom_op from sglang.srt.utils.hf_transformers_utils import get_hf_text_config @@ -350,7 +352,7 @@ class TransformersFusedMoE(nn.Module): expert_mapping: list, ) -> None: super().__init__() - num_redundant = get_exec().moe.ep_num_redundant_experts + num_redundant = get_server_args().ep_num_redundant_experts experts_cls = get_moe_impl_class(quant_config) self.experts = experts_cls( num_experts=num_experts + num_redundant, @@ -1229,7 +1231,7 @@ class MoEMixin: expert_mapping = self._get_expert_mapping(num_experts) # EPLB / EP tracking - num_redundant = get_exec().moe.ep_num_redundant_experts + num_redundant = get_server_args().ep_num_redundant_experts ep_size = get_parallel().moe_ep_size self.mlp_moe_layers: list[nn.Module] = [] diff --git a/python/sglang/srt/models/utils.py b/python/sglang/srt/models/utils.py index d62bb003b..dd767b2f5 100644 --- a/python/sglang/srt/models/utils.py +++ b/python/sglang/srt/models/utils.py @@ -34,7 +34,7 @@ from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.model_executor.runner import get_is_capture_mode from sglang.srt.model_loader.weight_utils import default_weight_loader -from sglang.srt.runtime_context import get_exec +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import get_current_device_stream_fast, is_cuda, is_hip from sglang.srt.utils.custom_op import register_custom_op @@ -444,7 +444,7 @@ def _reshape_for_qk_norm(x: torch.Tensor, head_dim: int) -> torch.Tensor: if ( _is_cuda - and get_exec().graph.cuda_graph_config.prefill.tc_compiler == "inductor" + and get_server_args().cuda_graph_config.prefill.tc_compiler == "inductor" ): return x.view(*x.shape[:-1], -1, head_dim) return x.reshape(-1, head_dim) @@ -485,7 +485,7 @@ def apply_qk_norm( and allow_inplace # TODO(dark): this can be relaxed if needed and (q_eps == k_eps) # TODO(dark): this can also be relaxed and not envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.get() - and get_exec().graph.cuda_graph_config.prefill.tc_compiler + and get_server_args().cuda_graph_config.prefill.tc_compiler != "inductor" # let inductor fuse QK norm and can_use_fused_inplace_qknorm(head_dim, q.dtype) ): diff --git a/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py b/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py index 23ce493eb..10461fe86 100644 --- a/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py +++ b/python/sglang/srt/multimodal/internvl_vit_cuda_graph_runner.py @@ -22,7 +22,7 @@ import torch import torch.nn as nn from sglang.srt.layers.attention.vision import VisionAttention -from sglang.srt.runtime_context import get_mm +from sglang.srt.runtime_context import get_server_args class InternViTCudaGraphRunner: @@ -95,7 +95,7 @@ class InternViTCudaGraphRunner: def _warmup_once(self, key: Hashable) -> None: """Run a tiny eager warmup on the preallocated buffers to trigger lazy init.""" - override_backend = get_mm().mm_attention_backend + override_backend = get_server_args().mm_attention_backend cu = self.cu[key] cu_kk = self.cu_kk[key] max_len = int(cu_kk.max().item()) if cu_kk.numel() else 0 @@ -115,7 +115,7 @@ class InternViTCudaGraphRunner: def _capture_graph(self, key: Hashable) -> None: g = torch.cuda.CUDAGraph() - override_backend = get_mm().mm_attention_backend + override_backend = get_server_args().mm_attention_backend cu = self.cu[key] cu_kk = self.cu_kk[key] diff --git a/python/sglang/srt/multimodal/processors/base_processor.py b/python/sglang/srt/multimodal/processors/base_processor.py index b264dc400..04c181aaa 100644 --- a/python/sglang/srt/multimodal/processors/base_processor.py +++ b/python/sglang/srt/multimodal/processors/base_processor.py @@ -20,7 +20,7 @@ from sglang.srt.managers.schedule_batch import ( MultimodalProcessorOutput, ) from sglang.srt.multimodal.processors.executor import MultimodalProcessorExecutor -from sglang.srt.runtime_context import get_device, get_exec, get_mm, get_serving +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import ( envs, is_cpu, @@ -205,7 +205,7 @@ class BaseMultimodalProcessor(ABC): self.disable_fast_image_processor = server_args.disable_fast_image_processor self.skip_tokenizer_init = server_args.skip_tokenizer_init - mm_process_config = get_mm().mm_process_config + mm_process_config = self.server_args.mm_process_config self.image_config = mm_process_config.get("image", {}) self.video_config = mm_process_config.get("video", {}) self.audio_config = mm_process_config.get("audio", {}) @@ -337,7 +337,7 @@ class BaseMultimodalProcessor(ABC): # SGLANG_MM_FEATURE_CACHE_MB is the total pool budget across all # tokenizer workers. Each worker gets an equal share so that adding # workers doesn't multiply the GPU-side footprint. - worker_num = get_serving().tokenizer_worker_num + worker_num = self.server_args.tokenizer_worker_num per_worker_pool_size = get_mm_feature_pool_size_per_worker( MM_FEATURE_CACHE_SIZE, worker_num ) @@ -347,7 +347,7 @@ class BaseMultimodalProcessor(ABC): "GPU %d (%.0f MiB per tokenizer worker × %d; configured " "budget %.0f MiB).", total_pool_size / (1024 * 1024), - get_device().base_gpu_id, + self.server_args.base_gpu_id, per_worker_pool_size / (1024 * 1024), worker_num, MM_FEATURE_CACHE_SIZE / (1024 * 1024), @@ -355,7 +355,7 @@ class BaseMultimodalProcessor(ABC): self.cudaipc_mmfeature_pool = MmItemMemoryPool( per_worker_pool_size, MM_ITEM_MEMORY_POOL_RECYCLE_INTERVAL, - get_device().base_gpu_id, + self.server_args.base_gpu_id, ) def compute_mrope_positions(self, input_ids, mm_items): @@ -528,12 +528,12 @@ class BaseMultimodalProcessor(ABC): and isinstance(processor.image_processor, BaseImageProcessor) and not self.disable_fast_image_processor ): - if _is_cpu or get_exec().deterministic.rl_on_policy_target is not None: + if _is_cpu or get_server_args().rl_on_policy_target is not None: kwargs["device"] = "cpu" elif _is_xpu: kwargs["device"] = "xpu" elif not _is_npu: - base_gpu_id = get_device().base_gpu_id + base_gpu_id = get_server_args().base_gpu_id kwargs["device"] = f"cuda:{base_gpu_id}" elif processor.__class__.__name__ not in { "Glm4vProcessor", diff --git a/python/sglang/srt/multimodal/processors/kimi_k25.py b/python/sglang/srt/multimodal/processors/kimi_k25.py index 8167604d6..e15935620 100644 --- a/python/sglang/srt/multimodal/processors/kimi_k25.py +++ b/python/sglang/srt/multimodal/processors/kimi_k25.py @@ -8,7 +8,9 @@ import torch import torch.nn.functional as F from PIL import Image -from sglang.srt.managers.schedule_batch import MultimodalProcessorOutput +from sglang.srt.managers.schedule_batch import ( + MultimodalProcessorOutput, +) from sglang.srt.models.kimi_k25 import KimiK25ForConditionalGeneration from sglang.srt.multimodal.processors.base_processor import ( BaseMultimodalProcessor as SGLangBaseProcessor, @@ -17,7 +19,6 @@ from sglang.srt.multimodal.processors.base_processor import ( MultimodalSpecialTokens, ) from sglang.srt.multimodal.processors.kimi_common import KimiGridMMDataMixin -from sglang.srt.runtime_context import get_mm from sglang.srt.utils.cuda_ipc_transport_utils import ( DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY, ) @@ -463,7 +464,7 @@ class KimiK2_5VLImageProcessor(KimiGridMMDataMixin, SGLangBaseProcessor): # its IPC proxy lazy until that assignment is known, avoiding a full # image copy to every rank. The scheduler only honors this marker once # the processor has already set the item's hash and pad value. - if self.use_cuda_ipc and get_mm().mm_enable_dp_encoder: + if self.use_cuda_ipc and self.server_args.mm_enable_dp_encoder: for item in mm_items: item.model_specific_data[DEFER_CUDA_IPC_FEATURE_RECONSTRUCTION_KEY] = ( True diff --git a/python/sglang/srt/multimodal/vit_cuda_graph_runner.py b/python/sglang/srt/multimodal/vit_cuda_graph_runner.py index 336a09b6a..e05171e34 100644 --- a/python/sglang/srt/multimodal/vit_cuda_graph_runner.py +++ b/python/sglang/srt/multimodal/vit_cuda_graph_runner.py @@ -25,7 +25,7 @@ import torch.nn as nn from sglang.srt.distributed.parallel_state import get_tp_group from sglang.srt.layers.attention.vision import VisionAttention -from sglang.srt.runtime_context import get_mm +from sglang.srt.runtime_context import get_server_args class ViTCudaGraphRunner: @@ -151,7 +151,7 @@ class ViTCudaGraphRunner: cu_full_kk = self.cu_full_len_kk[graph_key] max_full_len = int(cu_full_kk.max().item()) - override_backend = get_mm().mm_attention_backend + override_backend = get_server_args().mm_attention_backend if self._fullatt_block_indexes and 0 not in vit.fullatt_block_indexes: warmup_cu_ws = [cu_window, cu_window_kk, max_window_len] diff --git a/python/sglang/srt/multiplex/multiplexing_mixin.py b/python/sglang/srt/multiplex/multiplexing_mixin.py index 5a4e2ffc7..419dbe9b1 100644 --- a/python/sglang/srt/multiplex/multiplexing_mixin.py +++ b/python/sglang/srt/multiplex/multiplexing_mixin.py @@ -21,7 +21,6 @@ from sglang.srt.multiplex.pdmux_context import ( load_pdmux_config, set_current_stream_idx, ) -from sglang.srt.runtime_context import get_disagg if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import ScheduleBatch @@ -37,7 +36,7 @@ class SchedulerMultiplexMixin: self.split_prefill_batch: Optional[ScheduleBatch] = None # for pd_multiplexing, Init stream_groups, exclude normal stream for prefill only and decode only - self.pdmux_config = load_pdmux_config(get_disagg().pdmux_config_path) + self.pdmux_config = load_pdmux_config(self.server_args.pdmux_config_path) initialize_stream_groups(self.gpu_id, self.pdmux_config) self.stream_groups = get_stream_groups() self.sm_counts = get_sm_counts() diff --git a/python/sglang/srt/runtime_context.py b/python/sglang/srt/runtime_context.py index e0daefeea..5281ef432 100644 --- a/python/sglang/srt/runtime_context.py +++ b/python/sglang/srt/runtime_context.py @@ -574,27 +574,17 @@ class _ConfigBag: writers are ``get_context().override(source, ...)`` (permanent) and the scoped ``.override(**kw)`` context manager (tests). Sub-namespaces (e.g. ``exec.moe``) are nested ``_ConfigBag`` instances reached by attribute. - - Leaves and sub-bags are stored as **real instance attributes** (in - ``__dict__``), so ``bag.leaf`` / ``bag.sub`` is a plain attribute load that - ``torch.compile`` / dynamo can trace — config reads inside a compiled model - forward (e.g. ``get_exec().comm.enable_symm_mem`` in the embedding layer) - must not graph-break. ``_fields`` / ``_subs`` keep the authoritative - name→value maps used for override routing, membership, and scoped restore; - ``__getattr__`` is only a fallback for genuinely absent names. (Deliberately - no ``__slots__``: leaves are dynamic, and the ``__dict__`` is what makes the - reads traceable.) """ + __slots__ = ("_path", "_fields", "_subs") + def __init__(self, path: str): object.__setattr__(self, "_path", path) object.__setattr__(self, "_fields", {}) # {leaf: value} object.__setattr__(self, "_subs", {}) # {subname: _ConfigBag} def __getattr__(self, name: str) -> Any: - # Fallback only: real leaves/sub-bags resolve via __dict__ before this - # runs. Uses object.__getattribute__ (not self._fields) to stay safe if - # invoked before __init__ populates the bookkeeping dicts. + # Reached only when ``name`` is not a real attribute (slot). fields = object.__getattribute__(self, "_fields") if name in fields: return fields[name] @@ -611,16 +601,8 @@ class _ConfigBag: ) def _set(self, name: str, value: Any) -> None: - """Internal write (publish + override) that bypasses the read-only guard. - Updates both the bookkeeping map and the real attribute (traceable read).""" + """Internal write (publish + override) that bypasses the read-only guard.""" object.__getattribute__(self, "_fields")[name] = value - object.__setattr__(self, name, value) - - def _set_sub(self, name: str, sub: _ConfigBag) -> None: - """Register a nested bag as both a bookkeeping entry and a real - attribute (so ``bag.sub`` is a plain, traceable attribute load).""" - object.__getattribute__(self, "_subs")[name] = sub - object.__setattr__(self, name, sub) def __contains__(self, name: str) -> bool: return name in object.__getattribute__(self, "_fields") @@ -635,13 +617,11 @@ class _ConfigBag: path = object.__getattribute__(self, "_path") raise ValueError(f"unknown config leaf for {path!r}: {sorted(unknown)}") saved = {name: fields[name] for name in kwargs} - for name, value in kwargs.items(): - self._set(name, value) + fields.update(kwargs) try: yield self finally: - for name, value in saved.items(): - self._set(name, value) + fields.update(saved) def _build_config_bags(server_args: Any) -> dict: @@ -680,8 +660,7 @@ def _build_config_bags(server_args: Any) -> dict: subs = object.__getattribute__(bag, "_subs") child = subs.get(name) if child is None: - child = _ConfigBag(".".join(parts[: depth + 1])) - bag._set_sub(name, child) + child = subs[name] = _ConfigBag(".".join(parts[: depth + 1])) bag = child if field in object.__getattribute__(bag, "_subs"): raise ValueError( @@ -702,7 +681,6 @@ class RuntimeContext: "_server_args", "_config_bags", "_overrides_log", - "_publish_role", "flags", "resources", "forward", @@ -713,7 +691,6 @@ class RuntimeContext: self._server_args: ServerArgs | None = None self._config_bags: dict | None = None self._overrides_log: list = [] - self._publish_role: str | None = None self.flags = Flags() self.resources = Resources() self.forward = ForwardFlags() @@ -838,11 +815,8 @@ class RuntimeContext: self._overrides_log.append((source, dict(fields))) def overrides_log(self) -> list: - """Provenance of post-publish ``override`` calls: ``[(source, {field: value})]``. - - Returns deep-ish copies (source, dict(fields)) so callers inspecting the - log cannot mutate the recorded provenance in place.""" - return [(source, dict(fields)) for source, fields in self._overrides_log] + """Provenance of post-publish ``override`` calls: ``[(source, {field: value})]``.""" + return list(self._overrides_log) def resolved_server_args_dict(self, base: dict | None = None) -> dict: """Serialize the *resolved* config: the pristine ``server_args`` fields @@ -884,35 +858,6 @@ class RuntimeContext: """ return _ServerArgsOverride(self, fields) - @contextmanager - def preserve_config(self): - """Snapshot the full config lifecycle and reinstate it verbatim on exit. - - Used when a nested construction step must leave the process-wide config - exactly as it found it — notably ``build_draft_tp_worker``, which builds - a draft worker off a private ``ServerArgs`` copy and must not disturb the - target's published config. Unlike ``set_server_args`` (which re-projects - the bags from a pristine record and so *discards* every post-publish - override made during target loading, e.g. ``kv_cache_dtype`` or - ``disable_shared_experts_fusion``), this restores the resolved bags - as-is, so namespace readers keep the target's resolved values afterward. - """ - prev_server_args = self._server_args - prev_bags = self._config_bags - prev_overrides_log = self._overrides_log - prev_publish_role = self._publish_role - prev_parallel_config = self.parallel._config - prev_capture = self.flags.capture.enable_torch_compile - try: - yield - finally: - self._server_args = prev_server_args - self._config_bags = prev_bags - self._overrides_log = prev_overrides_log - self._publish_role = prev_publish_role - self.parallel._config = prev_parallel_config - self.flags.capture.enable_torch_compile = prev_capture - class _ServerArgsOverride: """Scoped config override (see ``RuntimeContext.override_server_args``). @@ -924,21 +869,13 @@ class _ServerArgsOverride: nondeterministic point. """ - __slots__ = ( - "_context", - "_fields", - "_prev_server_args", - "_prev_bags", - "_prev_overrides_log", - "_prev_publish_role", - "_prev_parallel_config", - "_prev_capture", - "_installed", - ) + __slots__ = ("_context", "_fields", "_previous", "_previous_capture", "_installed") def __init__(self, context: RuntimeContext, fields: dict): self._context = context self._fields = fields + self._previous: ServerArgs | None = None + self._previous_capture = False self._installed = False def install(self) -> ServerArgs: @@ -948,18 +885,8 @@ class _ServerArgsOverride: from sglang.srt.server_args import ServerArgs assert not self._installed, "override_server_args already installed" - # Snapshot the ENTIRE pre-install lifecycle state so restore() reinstates - # it verbatim: reseeding only ``_server_args`` would leave the projected - # bags / parallel leaves / provenance from this override live after the - # scope (violating fail-closed and leaking config into later tests), and - # would also drop any outer override that was active before this one. - ctx = self._context - self._prev_server_args = ctx._server_args - self._prev_bags = ctx._config_bags - self._prev_overrides_log = ctx._overrides_log - self._prev_publish_role = ctx._publish_role - self._prev_parallel_config = ctx.parallel._config - self._prev_capture = ctx.flags.capture.enable_torch_compile + self._previous = self._context._server_args + self._previous_capture = self._context.flags.capture.enable_torch_compile server_args = ServerArgs(model_path="dummy") if self._fields: server_args.override(source="test-override", **self._fields) @@ -968,26 +895,24 @@ class _ServerArgsOverride: # materialized so bare post-publish writes raise like they do on a # fully resolved config. object.__setattr__(server_args, "_declarations_materialized", True) - ctx.set_server_args(server_args) + self._context.set_server_args(server_args) self._installed = True return server_args def restore(self) -> None: - """Reinstate the exact pre-install lifecycle state (or the empty slot).""" + """Reinstate the previously published config (or the empty slot).""" if not self._installed: return self._installed = False - ctx = self._context - ctx._server_args = self._prev_server_args - ctx._config_bags = self._prev_bags - ctx._overrides_log = self._prev_overrides_log - ctx._publish_role = self._prev_publish_role - ctx.parallel._config = self._prev_parallel_config - ctx.flags.capture.enable_torch_compile = self._prev_capture - self._prev_server_args = None - self._prev_bags = None - self._prev_overrides_log = None - self._prev_parallel_config = None + previous, self._previous = self._previous, None + if previous is None: + self._context._server_args = None + else: + self._context.set_server_args(previous) + # set_server_args reseeds the capture tier from the published object + # (and the empty-slot path does not touch it at all); the snapshot + # puts back the exact pre-install runtime state either way. + self._context.flags.capture.enable_torch_compile = self._previous_capture def __enter__(self) -> ServerArgs: return self.install() @@ -1073,26 +998,6 @@ def get_observability() -> _ConfigBag: return _CONTEXT.config_bag("observability") -def publish(server_args, *, role: str, hf_config: Any = None) -> RuntimeContext: - """Install process-wide config for this OS process. - - Records the process ``role`` (``tokenizer`` / ``scheduler`` / ``encoder`` / - ``expert_backup`` / ``launcher`` / ``test``) and projects the config bags. - One call per process; draft workers skip publish (they must not clobber the - target). ``role`` is provenance today — per-role namespace projection and - fail-closed enforcement is a later unit. ``hf_config`` is accepted for - forward-compat and currently unused. - """ - _CONTEXT._publish_role = role - _CONTEXT.set_server_args(server_args) - return _CONTEXT - - -def publish_role() -> str | None: - """The role recorded by the last ``publish`` (None for a legacy set).""" - return _CONTEXT._publish_role - - def get_stream(name: str) -> Any: return _CONTEXT.get_stream(name) @@ -1126,7 +1031,6 @@ def reset_context() -> None: _CONTEXT._server_args = None _CONTEXT._config_bags = None _CONTEXT._overrides_log = [] - _CONTEXT._publish_role = None _CONTEXT.parallel._config = None _CONTEXT.flags = Flags() _CONTEXT.resources = Resources() diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 14947fbf3..c0ca0679f 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -8605,19 +8605,14 @@ class ServerArgs: # (decrease-only) by test/registered/unit/test_legacy_global_ratchet.py. # Imports are in-function so the two modules stay cycle-free at import time. def set_global_server_args_for_scheduler(server_args: ServerArgs): - """Legacy publish shim (role=scheduler) — prefer - ``runtime_context.publish(server_args, role=...)`` in new code.""" - from sglang.srt.runtime_context import publish + """Legacy publish shim — prefer ``get_context().set_server_args()`` from + ``sglang.srt.runtime_context`` in new code.""" + from sglang.srt.runtime_context import get_context - publish(server_args, role="scheduler") + get_context().set_server_args(server_args) -def set_global_server_args_for_tokenizer(server_args: ServerArgs): - """Legacy publish shim (role=tokenizer). Not aliased to the scheduler shim: - the process role differs.""" - from sglang.srt.runtime_context import publish - - publish(server_args, role="tokenizer") +set_global_server_args_for_tokenizer = set_global_server_args_for_scheduler def get_global_server_args() -> ServerArgs: diff --git a/python/sglang/srt/speculative/dflash_info_v2.py b/python/sglang/srt/speculative/dflash_info_v2.py index 4646cb7cb..8893a6541 100644 --- a/python/sglang/srt/speculative/dflash_info_v2.py +++ b/python/sglang/srt/speculative/dflash_info_v2.py @@ -9,7 +9,7 @@ import torch from sglang.srt.environ import envs from sglang.srt.managers.schedule_batch import ScheduleBatch from sglang.srt.mem_cache.allocation import alloc_for_spec_decode -from sglang.srt.runtime_context import get_spec +from sglang.srt.runtime_context import get_server_args from sglang.srt.speculative.spec_info import SpecInput, SpecInputType from sglang.srt.utils.common import is_pin_memory_available @@ -134,7 +134,7 @@ class DFlashDraftInputV2(SpecInput): cur_kv_lens_cpu_t = self._prepare_cur_kv_lens_cpu_buf[:bs] # For DFLASH, each decode step needs a fixed-size verify block. - block_size = int(get_spec().speculative_num_draft_tokens) + block_size = int(get_server_args().speculative_num_draft_tokens) if block_size <= 0: raise ValueError( f"DFLASH invalid speculative_num_draft_tokens={block_size}." diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index a5fe3e1fd..a1095828f 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -27,7 +27,6 @@ from sglang.srt.model_executor.forward_batch_info import ( ForwardMode, compute_position, ) -from sglang.srt.runtime_context import get_exec from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.base_spec_worker import BaseSpecWorker from sglang.srt.speculative.dflash_info import DFlashVerifyInput @@ -330,7 +329,7 @@ class DFlashWorkerV2(BaseSpecWorker): def init_cuda_graphs(self): capture_decode_cuda_graph = ( - get_exec().graph.cuda_graph_config.decode.backend != Backend.DISABLED + self.server_args.cuda_graph_config.decode.backend != Backend.DISABLED ) if is_cuda() and capture_decode_cuda_graph: available_mem = get_available_gpu_memory(self.device, self.gpu_id) @@ -392,7 +391,7 @@ class DFlashWorkerV2(BaseSpecWorker): block_size=self.block_size, num_org=num_org, org_vocab_start=org_vocab_start, - max_bs=max(get_exec().graph.cuda_graph_config.decode.bs), + max_bs=max(self.server_args.cuda_graph_config.decode.bs), tp_group=tp_group if tp_group.world_size > 1 else None, ) @@ -1257,7 +1256,7 @@ class DFlashWorkerV2(BaseSpecWorker): mamba_steps_to_track = None if batch.mamba_track_indices is not None: - mamba_track_interval = get_exec().mamba.mamba_track_interval + mamba_track_interval = self.server_args.mamba_track_interval to_track_mask = ( seq_lens_pre_verify // mamba_track_interval != batch.seq_lens // mamba_track_interval diff --git a/python/sglang/srt/speculative/draft_utils.py b/python/sglang/srt/speculative/draft_utils.py index 3f7840bb3..bb7106eda 100644 --- a/python/sglang/srt/speculative/draft_utils.py +++ b/python/sglang/srt/speculative/draft_utils.py @@ -1,4 +1,3 @@ -from sglang.srt.runtime_context import get_exec, get_spec from sglang.srt.server_args import ServerArgs from sglang.srt.utils.common import ( cpu_has_amx_support, @@ -35,7 +34,7 @@ class DraftBackendFactory: else getattr(self.server_args, backend_name) ) if backend_type is None: - backend_type = get_exec().kernel.attention_backend + backend_type = self.server_args.attention_backend if backend_type not in backend_map: raise ValueError(error_template.format(backend_type=backend_type)) @@ -94,7 +93,7 @@ class DraftBackendFactory: } backend_name = ( "decode_attention_backend" - if get_spec().speculative_attention_mode == "decode" + if self.server_args.speculative_attention_mode == "decode" else "prefill_attention_backend" ) return self._create_backend( diff --git a/python/sglang/srt/speculative/draft_worker_common.py b/python/sglang/srt/speculative/draft_worker_common.py index 5b4aaa0f8..901aa88cf 100644 --- a/python/sglang/srt/speculative/draft_worker_common.py +++ b/python/sglang/srt/speculative/draft_worker_common.py @@ -10,7 +10,7 @@ import torch from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.managers.tp_worker import TpModelWorker from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode -from sglang.srt.runtime_context import get_context +from sglang.srt.runtime_context import get_context, get_server_args from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.dflash_info import DFlashVerifyInput from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2 @@ -90,17 +90,8 @@ def build_draft_tp_worker( context_length=target_model_config.context_len, ) - # Publish the draft copy for the duration of the build so the draft's model - # layers resolve config (e.g. kv_cache_dtype) from the draft's own bags, not - # the target's -- an independently configured draft can resolve a different - # KV-cache dtype than the target, and reading the target-global bag would - # make draft attention record the wrong dtype. ``preserve_config`` snapshots - # the target's resolved config on entry and reinstates it verbatim on exit - # (post-publish overrides intact), so the target is undisturbed afterwards -- - # unlike a plain ``set_server_args(saved)`` restore, which re-projects the - # bags from the pristine record and drops those overrides. - with get_context().preserve_config(): - get_context().set_server_args(draft_server_args) + saved_server_args = get_server_args() + try: draft_worker = TpModelWorker( server_args=draft_server_args, gpu_id=gpu_id, @@ -108,6 +99,8 @@ def build_draft_tp_worker( nccl_port=nccl_port, is_draft_worker=True, ) + finally: + get_context().set_server_args(saved_server_args) draft_model_runner = draft_worker.model_runner draft_worker.draft_runner = draft_model_runner diff --git a/python/sglang/srt/speculative/dspark_components/dspark_planner.py b/python/sglang/srt/speculative/dspark_components/dspark_planner.py index 54f7df8a0..e76206d5c 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_planner.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_planner.py @@ -16,7 +16,7 @@ from sglang.srt.managers.overlap_utils import ( ResolvedConfidence, ) from sglang.srt.managers.schedule_batch import ScheduleBatch -from sglang.srt.runtime_context import get_disagg, get_parallel, get_schedule, get_spec +from sglang.srt.runtime_context import get_parallel from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2 from sglang.srt.speculative.dflash_utils import apply_dflash_verify_logits_adjustments @@ -147,7 +147,7 @@ class DSparkVerifyPlanner: ) relay_lag_steps = ( 0 - if get_schedule().disable_overlap_schedule + if self.server_args.disable_overlap_schedule else CONFIDENCE_RELAY_RING_LAG ) self._budget_planner = HostConfidenceBudgetPlanner( @@ -163,15 +163,16 @@ class DSparkVerifyPlanner: and get_parallel().attn_tp_size == 1 and get_parallel().attn_cp_size == 1 and require_mlp_tp_gather(self.server_args) - and not get_schedule().disable_overlap_schedule - and not get_spec().speculative_skip_dp_mlp_sync - and get_disagg().disaggregation_mode == "null" + and not self.server_args.disable_overlap_schedule + and not self.server_args.speculative_skip_dp_mlp_sync + and self.server_args.disaggregation_mode == "null" and self.server_args.pp_size == 1 and not envs.SGLANG_SCHEDULER_SKIP_ALL_GATHER.get() ) if tp_rank == 0: sps_table_source = ( - get_spec().speculative_dspark_sps_table_path or "uninitialized" + self.server_args.speculative_dspark_sps_table_path + or "uninitialized" ) logger.info( "DSpark ragged-verify scheduler enabled (mode=%s, lag=%d, " @@ -370,7 +371,7 @@ class DSparkVerifyPlanner: the draft input by prepare_verify_budget; otherwise compute it now.""" if not self.schedules_verify_budget or confidence is None: return None - if not get_schedule().disable_overlap_schedule: + if not self.server_args.disable_overlap_schedule: return draft_input.verify_token_budget return self.compute_budget_sync( confidence=confidence, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index a8b29eb73..dfbff5730 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -14,7 +14,7 @@ from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, compute_position, ) -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.base_spec_worker import BaseSpecWorker from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2 @@ -294,7 +294,7 @@ class DSparkWorkerV2(BaseSpecWorker): self._draft_worker.init_attention_backends() def init_cuda_graphs(self): - capture_decode_cuda_graph = not get_exec().graph.disable_cuda_graph + capture_decode_cuda_graph = not self.server_args.disable_cuda_graph if is_cuda() and capture_decode_cuda_graph: available_mem = get_available_gpu_memory(self.device, self.gpu_id) if available_mem < 1.0: @@ -320,7 +320,7 @@ class DSparkWorkerV2(BaseSpecWorker): return maybe_build_draft_sampler( draft_model=self.draft_model, gamma=self.gamma, - max_bs=max(get_exec().graph.cuda_graph_config.decode.bs), + max_bs=max(self.server_args.cuda_graph_config.decode.bs), device=self.device, tp_rank=self.ps.tp_rank, confidence_fn=( @@ -375,7 +375,7 @@ class DSparkWorkerV2(BaseSpecWorker): self, batch: ScheduleBatch, on_publish ) -> GenerationBatchResult: if batch.forward_mode.is_idle(): - if get_parallel().enable_dp_attention: + if self.server_args.enable_dp_attention: self.target_worker.forward_batch_generation( batch, capture_hidden_mode=CaptureHiddenMode.FULL ) @@ -443,7 +443,7 @@ class DSparkWorkerV2(BaseSpecWorker): def _dp_verify_tier_num_tokens(self, batch: ScheduleBatch) -> Optional[int]: if not ( self._draft_is_moe - and get_parallel().enable_dp_attention + and self.server_args.enable_dp_attention and batch.global_num_tokens is not None and self._verify_planner.is_compact_mode ): @@ -487,7 +487,7 @@ class DSparkWorkerV2(BaseSpecWorker): if batch.forward_mode.is_idle(): self._observers.note_idle_decode_step() - if get_parallel().enable_dp_attention: + if self.server_args.enable_dp_attention: if self._draft_is_moe: self._proposer.run_idle_participation(batch) self._verify_executor.run_idle_participation( @@ -549,7 +549,7 @@ class DSparkWorkerV2(BaseSpecWorker): global_num_reqs = ( max(batch.global_num_tokens) if self._draft_is_moe - and get_parallel().enable_dp_attention + and self.server_args.enable_dp_attention and batch.global_num_tokens is not None else None ) diff --git a/python/sglang/srt/speculative/eagle_info.py b/python/sglang/srt/speculative/eagle_info.py index a997e76f5..60c702d19 100644 --- a/python/sglang/srt/speculative/eagle_info.py +++ b/python/sglang/srt/speculative/eagle_info.py @@ -8,7 +8,7 @@ from sglang.kernels.ops.attention.utils import create_flashinfer_kv_indices_trit from sglang.srt.constrained.base_grammar_backend import BaseGrammarObject from sglang.srt.environ import envs from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode -from sglang.srt.runtime_context import get_spec +from sglang.srt.runtime_context import get_server_args from sglang.srt.speculative.spec_info import SpecInput, SpecInputType logger = logging.getLogger(__name__) @@ -202,7 +202,7 @@ class EagleDraftInput(SpecInput): topk_index=torch.empty((0, topk), device=device, dtype=torch.int64), draft_probs=( torch.empty((0, vocab_size), device=device, dtype=torch.float32) - if get_spec().speculative_use_rejection_sampling + if get_server_args().speculative_use_rejection_sampling else None ), capture_hidden_mode=capture_hidden_mode, diff --git a/python/sglang/srt/speculative/eagle_utils.py b/python/sglang/srt/speculative/eagle_utils.py index 8f4c78442..c4725ebcc 100644 --- a/python/sglang/srt/speculative/eagle_utils.py +++ b/python/sglang/srt/speculative/eagle_utils.py @@ -17,7 +17,14 @@ from sglang.srt.hardware_backend.npu.dsv4.dsv4_common_hooks import ( from sglang.srt.mem_cache.allocation import alloc_for_spec_decode from sglang.srt.mem_cache.allocation_sizing import get_alloc_reserve_per_decode from sglang.srt.runtime_context import get_parallel, get_spec -from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_musa, is_npu, is_xpu +from sglang.srt.utils import ( + is_cpu, + is_cuda, + is_hip, + is_musa, + is_npu, + is_xpu, +) from sglang.srt.utils.async_probe import maybe_detect_oob if TYPE_CHECKING: @@ -481,7 +488,9 @@ def eagle_prepare_for_verify( batch: ScheduleBatch, target_worker: TpModelWorker, ): - from sglang.kernels.ops.speculative.cache_locs import assign_extend_cache_locs_func + from sglang.kernels.ops.speculative.cache_locs import ( + assign_extend_cache_locs_func, + ) from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, ForwardBatch, @@ -570,7 +579,10 @@ def eagle_sample( import torch.nn.functional as F from sglang.srt.distributed import get_tp_group - from sglang.srt.layers.dp_attention import is_dp_attention_enabled + from sglang.srt.layers.dp_attention import ( + is_dp_attention_enabled, + ) + from sglang.srt.runtime_context import get_server_args from sglang.srt.sampling.penaltylib.repetition_penalty import ( apply_scaling_penalties, ) @@ -660,7 +672,7 @@ def eagle_sample( chain_speculative_sampling_triton, ) - use_rejection_sampling = get_spec().speculative_use_rejection_sampling + use_rejection_sampling = get_server_args().speculative_use_rejection_sampling # Apply temperature and get target probs expanded_temperature = torch.repeat_interleave( @@ -830,8 +842,9 @@ def eagle_prepare_for_decode(batch: ScheduleBatch): # (get_alloc_reserve_per_decode) outgrows the req_to_token row: the write below # would OOB and free would leak KV. The row is widened to hold it in _init_pools # (PR #26972); fail here with a clear error, not on a later cryptic CUDA assert. + from sglang.srt.runtime_context import get_server_args - if page_size > 1 and (get_spec().speculative_eagle_topk or 1) > 1: + if page_size > 1 and (get_server_args().speculative_eagle_topk or 1) > 1: max_alloc_len = int(nxt_kv_lens_cpu.max()) row_width = batch.req_to_token_pool.req_to_token.shape[1] assert max_alloc_len <= row_width, ( diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index c178734ec..3c743f7fb 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -21,7 +21,9 @@ from sglang.srt.layers.attention.flashinfer_backend import FlashInferAttnBackend from sglang.srt.layers.attention.tokenspeed_mla_backend import TokenspeedMLABackend from sglang.srt.layers.attention.triton_backend import TritonAttnBackend from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend -from sglang.srt.layers.attention.trtllm_mla_backend import TRTLLMMLABackend +from sglang.srt.layers.attention.trtllm_mla_backend import ( + TRTLLMMLABackend, +) from sglang.srt.layers.moe.utils import ( speculative_moe_a2a_backend_context, speculative_moe_backend_context, @@ -41,13 +43,7 @@ from sglang.srt.model_executor.runner import ( DecodeCudaGraphRunner, get_batch_sizes_to_capture, ) -from sglang.srt.runtime_context import ( - get_context, - get_exec, - get_model, - get_parallel, - get_spec, -) +from sglang.srt.runtime_context import get_parallel from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.adaptive_runtime_state import ( AdaptiveController, @@ -140,7 +136,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): # Args for easy access self.device = server_args.device self.topk = server_args.speculative_eagle_topk - if get_spec().speculative_use_rejection_sampling: + if self.server_args.speculative_use_rejection_sampling: assert self.topk == 1, "Chain speculative sampling supports only topk=1" self.speculative_num_steps = server_args.speculative_num_steps self.speculative_num_draft_tokens = server_args.speculative_num_draft_tokens @@ -199,7 +195,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): self.init_token_map() self.init_lm_head() - if get_spec().speculative_use_rejection_sampling: + if self.server_args.speculative_use_rejection_sampling: target_vocab_size = self.target_worker.model_config.vocab_size draft_vocab_size = ( self.hot_token_id.shape[0] @@ -292,13 +288,13 @@ class EagleDraftWorker(EagleDraftWorkerBase): def init_token_map(self): # Load hot token ids if self.speculative_algorithm.is_eagle3(): - if get_spec().speculative_token_map is not None: + if self.server_args.speculative_token_map is not None: logger.warning( "Speculative token map specified, but EAGLE3 models already have this. Ignoring the specified token map." ) self.hot_token_id = None - elif get_spec().speculative_token_map is not None: - self.hot_token_id = load_token_map(get_spec().speculative_token_map) + elif self.server_args.speculative_token_map is not None: + self.hot_token_id = load_token_map(self.server_args.speculative_token_map) self.server_args.override( "eagle_worker.hot_token_map", json_model_override_args=( @@ -383,7 +379,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): if _is_cpu or check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED): return - if get_model().model_impl == "mindspore": + if self.server_args.model_impl == "mindspore": return Device2DraftCudaGraphRunner = { @@ -393,7 +389,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): "musa": EAGLEDraftCudaGraphRunner, } # Capture draft - decode_backend = get_exec().graph.cuda_graph_config.decode.backend + decode_backend = self.server_args.cuda_graph_config.decode.backend capture_bs, _ = get_batch_sizes_to_capture(self.draft_runner) if self.speculative_num_steps > 1: tic = time.perf_counter() @@ -590,7 +586,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): score_list: List[torch.Tensor] = [] token_list: List[torch.Tensor] = [] parents_list: List[torch.Tensor] = [] - if get_spec().speculative_use_rejection_sampling: + if self.server_args.speculative_use_rejection_sampling: draft_probs_list: List[torch.Tensor] = [spec_info.draft_probs] topk1_chain_fits = ( @@ -605,7 +601,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): topk1_chain_fits and _is_cuda and self.hot_token_id is None - and not get_spec().speculative_use_rejection_sampling + and not self.server_args.speculative_use_rejection_sampling ): draft_tokens_topk1 = torch.empty( (topk_index.shape[0], self.speculative_num_steps), @@ -670,7 +666,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): logits_output = self.draft_runner.forward(forward_batch).logits_output maybe_detect_nan(logits_output.next_token_logits, f"draft_forward step {i}") maybe_detect_inf(logits_output.next_token_logits, f"draft_forward step {i}") - if get_spec().speculative_use_rejection_sampling: + if self.server_args.speculative_use_rejection_sampling: probs, topk_p, topk_index = sample_draft_proposal( logits_output.next_token_logits, forward_batch.sampling_info.temperatures, @@ -696,7 +692,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): probs = renorm_draft_probs( logits_output.next_token_logits, forward_batch.sampling_info, - get_spec().speculative_use_rejection_sampling, + self.server_args.speculative_use_rejection_sampling, ) topk_p, topk_index = fast_topk(probs, self.topk, dim=-1) forward_batch.positions.add_(1) @@ -716,7 +712,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): draft_probs = ( torch.stack(draft_probs_list, dim=1) - if get_spec().speculative_use_rejection_sampling + if self.server_args.speculative_use_rejection_sampling else None ) @@ -836,7 +832,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): prefill_dsa_topk = self.dsa_extend_topk_buf[:bs].clone() # Assemble the next-iter draft spec_info from the extend output. - use_rejection_sampling = get_spec().speculative_use_rejection_sampling + use_rejection_sampling = self.server_args.speculative_use_rejection_sampling probs = renorm_draft_probs( logits_output.next_token_logits, batch.sampling_info, @@ -982,7 +978,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): ] # The draft-extend graph only anchors full logits; selected-row topk is # owned by the worker for both graph and eager paths. - if get_spec().speculative_use_rejection_sampling: + if self.server_args.speculative_use_rejection_sampling: ret_draft_probs, ret_topk_p, ret_topk_index = sample_draft_proposal( draft_logits_output.next_token_logits, batch.sampling_info.temperatures, @@ -999,7 +995,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): probs = renorm_draft_probs( draft_logits_output.next_token_logits, batch.sampling_info, - get_spec().speculative_use_rejection_sampling, + self.server_args.speculative_use_rejection_sampling, ) ret_topk_p, ret_topk_index = fast_topk(probs, self.topk, dim=-1) ret_draft_probs = None @@ -1016,7 +1012,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): ret_topk_index, ret_hidden_states, ) - if get_spec().speculative_use_rejection_sampling: + if self.server_args.speculative_use_rejection_sampling: next_draft_input.draft_probs = ret_draft_probs if self.seed_dsa_topk_from_draft_extend: next_draft_input.dsa_topk_indices = dsa_seed_topk_indices @@ -1119,7 +1115,7 @@ class EAGLEWorkerV2(BaseSpecWorker): cuda_graph_bs=( None if check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED) - else get_exec().graph.cuda_graph_bs_decode + else self.server_args.cuda_graph_bs_decode ), ) @@ -1422,30 +1418,13 @@ class EAGLEWorkerV2(BaseSpecWorker): state.target_graph_runner ) - # Sync the step/draft-token counts on both config stores. - self._apply_adaptive_config( + # Sync server_args + self.server_args.override( "adaptive_spec.restore", speculative_num_steps=state.speculative_num_steps, speculative_num_draft_tokens=state.speculative_num_draft_tokens, ) - def _apply_adaptive_config(self, source: str, **fields) -> None: - """Rebind adaptive-spec config on both stores that back these fields. - - Adaptive speculative decoding temporarily changes the step / draft-token - counts (and the decode graph-capture knobs) while it prebuilds per-step - runtime states. Those fields are read from two places: the resolved - config bag (runtime readers such as ``tp_worker`` / ``tokenizer_manager`` - via ``get_spec()``) *and* ``server_args`` — attention backends and graph - runners read the count as ``model_runner.server_args.`` and are - not migrated to the bag. Write both so an in-flight CUDA-graph capture - and later reads agree; updating only the bag leaves a rebuilt attention - backend sizing its metadata from the stale ``server_args`` count while - the graph is captured for the overridden count, which corrupts the - capture (illegal memory access).""" - get_context().override(source, **fields) - self.server_args.override(source, **fields) - @contextlib.contextmanager def _override_worker_state( self, @@ -1477,7 +1456,7 @@ class EAGLEWorkerV2(BaseSpecWorker): self.speculative_num_draft_tokens = speculative_num_draft_tokens dw.speculative_num_steps = speculative_num_steps dw.speculative_num_draft_tokens = speculative_num_draft_tokens - self._apply_adaptive_config( + sa.override( "adaptive_spec.capture_override", speculative_num_steps=speculative_num_steps, speculative_num_draft_tokens=speculative_num_draft_tokens, @@ -1487,7 +1466,7 @@ class EAGLEWorkerV2(BaseSpecWorker): # for steps that no BS range uses (e.g. step=1). Disable graph # capture for those steps; restore in finally so subsequent steps # are not affected. - self._apply_adaptive_config( + sa.override( "adaptive_spec.capture_override", cuda_graph_bs_decode=cuda_graph_bs, **({"disable_cuda_graph": True} if not cuda_graph_bs else {}), @@ -1509,7 +1488,7 @@ class EAGLEWorkerV2(BaseSpecWorker): dw.cuda_graph_runner, dw.cuda_graph_runner_for_draft_extend, ) = backup[:10] - self._apply_adaptive_config( + sa.override( "adaptive_spec.capture_restore", speculative_num_steps=backup[10], speculative_num_draft_tokens=backup[11], diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py b/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py index 0dd5f5483..98c210b4e 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_worker_v2.py @@ -42,7 +42,6 @@ from sglang.srt.model_executor.forward_batch_info import ( ) from sglang.srt.model_executor.forward_context import ForwardContext, forward_context from sglang.srt.model_executor.pool_configurator import MemoryPoolConfig -from sglang.srt.runtime_context import get_exec, get_spec from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.base_spec_worker import EagleDraftWorkerBase from sglang.srt.speculative.eagle_utils import ( @@ -224,9 +223,9 @@ class FrozenKVMTPDraftWorker(EagleDraftWorkerBase, TpModelWorker): def _resolve_draft_backend_type(self) -> str: return ( - get_spec().speculative_draft_attention_backend - or get_exec().kernel.decode_attention_backend - or get_exec().kernel.attention_backend + self.server_args.speculative_draft_attention_backend + or self.server_args.decode_attention_backend + or self.server_args.attention_backend ) def _init_draft_attn_backend(self): diff --git a/python/sglang/srt/speculative/spec_utils.py b/python/sglang/srt/speculative/spec_utils.py index 29419d12a..4671818d7 100644 --- a/python/sglang/srt/speculative/spec_utils.py +++ b/python/sglang/srt/speculative/spec_utils.py @@ -44,7 +44,7 @@ from sglang.srt.mem_cache.allocation import ( from sglang.srt.mem_cache.allocation import ( assign_req_to_token_pool_func as assign_req_to_token_pool_func, ) -from sglang.srt.runtime_context import get_exec, get_server_args +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import ( is_cpu, is_cuda, @@ -801,7 +801,7 @@ def commit_mamba_states_after_verify( # we need to update the mamba state for the request at the crossing point. seq_lens_pre_verify = batch.seq_lens seq_lens_post_verify = batch.seq_lens + accept_lens - mamba_track_interval = get_exec().mamba.mamba_track_interval + mamba_track_interval = get_server_args().mamba_track_interval to_track_mask = ( seq_lens_pre_verify // mamba_track_interval != seq_lens_post_verify // mamba_track_interval diff --git a/python/sglang/srt/state_capturer/indexer_topk.py b/python/sglang/srt/state_capturer/indexer_topk.py index 2dd33b3f4..afa652cec 100644 --- a/python/sglang/srt/state_capturer/indexer_topk.py +++ b/python/sglang/srt/state_capturer/indexer_topk.py @@ -6,7 +6,7 @@ import pybase64 import torch from sglang.srt.configs.model_config import ModelConfig, get_num_indexer_layers -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import get_parallel from sglang.srt.state_capturer.base import BaseTopkCapturer logger = logging.getLogger(__name__) @@ -89,8 +89,9 @@ def create_indexer_capturer( max_running_requests: int, device: str, ) -> Optional[IndexerTopkCapturer]: + from sglang.srt.runtime_context import get_server_args - enable = get_exec().features.enable_return_indexer_topk + enable = get_server_args().enable_return_indexer_topk # Producer wiring is CUDA-only (Indexer.forward_cuda + MLA skip_topk # path); other backends would create a capturer but never feed it. if enable and device != "cuda": diff --git a/python/sglang/srt/utils/profile_utils.py b/python/sglang/srt/utils/profile_utils.py index 4e3e24266..693a7418d 100644 --- a/python/sglang/srt/utils/profile_utils.py +++ b/python/sglang/srt/utils/profile_utils.py @@ -13,7 +13,7 @@ from sglang.srt.environ import envs from sglang.srt.managers.io_struct import ProfileReqOutput from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.platforms import current_platform -from sglang.srt.runtime_context import get_device +from sglang.srt.runtime_context import get_server_args from sglang.srt.utils import is_npu from sglang.srt.utils.torch_npu_patch_utils import apply_torch_npu_patches @@ -62,7 +62,7 @@ class ProfileManager: ) self.ps = ps self.cpu_group = cpu_group - self.first_rank_in_node = ps.gpu_id == get_device().base_gpu_id + self.first_rank_in_node = ps.gpu_id == get_server_args().base_gpu_id self.profiler_kwargs = None self.profiler = None diff --git a/test/registered/unit/test_legacy_global_ratchet.py b/test/registered/unit/test_legacy_global_ratchet.py index a0ecc59d2..a2c40b888 100644 --- a/test/registered/unit/test_legacy_global_ratchet.py +++ b/test/registered/unit/test_legacy_global_ratchet.py @@ -31,7 +31,7 @@ _RATCHETS = [ ( "set_global_server_args_for_*", r"\bset_global_server_args_for_(?:scheduler|tokenizer)\s*\(", - 4, + 5, ), ] diff --git a/test/registered/unit/test_runtime_context.py b/test/registered/unit/test_runtime_context.py index e95d0516b..35ea11e3f 100644 --- a/test/registered/unit/test_runtime_context.py +++ b/test/registered/unit/test_runtime_context.py @@ -10,7 +10,7 @@ import unittest from unittest.mock import patch import sglang.srt.server_args as server_args_module -from sglang.srt.arg_groups.arg_utils import NS, A, Arg +from sglang.srt.arg_groups.arg_utils import A, Arg from sglang.srt.runtime_context import ( Flags, ParallelContext, @@ -18,7 +18,6 @@ from sglang.srt.runtime_context import ( _FlagGroupBase, get_context, get_flags, - get_memory, get_parallel, get_server_args, reset_context, @@ -216,10 +215,8 @@ class TestServerArgsOwnership(_IsolatedServerArgs): self.assertIs(get_server_args(), sentinel) self.assertIs(get_context().server_args, sentinel) - def test_tokenizer_and_scheduler_setters_are_distinct_role_shims(self): - # The per-role publish shims are no longer aliases: each records its own - # process role via publish(role=...). - self.assertIsNot( + def test_tokenizer_alias_is_same_function(self): + self.assertIs( server_args_module.set_global_server_args_for_tokenizer, server_args_module.set_global_server_args_for_scheduler, ) @@ -376,10 +373,8 @@ class TestFlagsTier(_IsolatedServerArgs): class _FakeResolvedArgs: """Publishable fixture with a resolvable whitelist (real flat leaves).""" - page_size: A[int | None, Arg(help="p", resolvable=True), NS("memory")] = None - sampling_backend: A[ - str | None, Arg(help="s", resolvable=True), NS("exec.kernel") - ] = None + page_size: A[int | None, Arg(help="p", resolvable=True)] = None + sampling_backend: A[str | None, Arg(help="s", resolvable=True)] = None _resolved_overrides: list = dataclasses.field(default_factory=list) @@ -921,15 +916,12 @@ class TestPublishLifecycle(_IsolatedServerArgs): get_context().set_server_args(object()) self.assertFalse(get_flags().capture.enable_torch_compile) - def test_declare_load_time_override_writes_the_bag(self): + def test_declare_load_time_override_writes_through(self): from sglang.srt.arg_groups.overrides import declare_load_time_override args = self._publish(page_size=1) declare_load_time_override("model.load_time", {"page_size": 64}) - # The declaration lands on the config bag; the pristine startup record - # (server_args) is untouched. - self.assertEqual(get_memory().page_size, 64) - self.assertEqual(args.page_size, 1) + self.assertEqual(args.page_size, 64) def test_declare_load_time_override_validates_whitelist(self): from sglang.srt.arg_groups.overrides import declare_load_time_override @@ -941,14 +933,16 @@ class TestPublishLifecycle(_IsolatedServerArgs): def test_declare_load_time_override_records_provenance(self): from sglang.srt.arg_groups.overrides import declare_load_time_override + from sglang.srt.server_args import ServerArgs - self._publish(page_size=1) + class _Args(_FakeResolvedArgs): + override = ServerArgs.override + + args = _Args(page_size=1) + get_context().set_server_args(args) declare_load_time_override("model.load_time", {"page_size": 64}) - self.assertEqual(get_memory().page_size, 64) - self.assertIn( - ("model.load_time", {"page_size": 64}), - get_context().overrides_log(), - ) + self.assertEqual(args.page_size, 64) + self.assertIn(("model.load_time", {"page_size": 64}), args._resolved_overrides) if __name__ == "__main__": diff --git a/test/registered/unit/test_runtime_context_override.py b/test/registered/unit/test_runtime_context_override.py index 12db59aa5..2b9b5ba7f 100644 --- a/test/registered/unit/test_runtime_context_override.py +++ b/test/registered/unit/test_runtime_context_override.py @@ -109,27 +109,6 @@ class TestContextOverride(CustomTestCase): with self.assertRaises(AttributeError): sa.page_size = 999 - def test_publish_records_role(self): - rc.publish(ServerArgs(model_path="dummy"), role="scheduler") - self.assertEqual(rc.publish_role(), "scheduler") - - def test_legacy_shims_record_roles(self): - # Unit 2a: the legacy setters publish with their process role. - from sglang.srt.server_args import ( - set_global_server_args_for_scheduler, - set_global_server_args_for_tokenizer, - ) - - set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) - self.assertEqual(rc.publish_role(), "scheduler") - set_global_server_args_for_tokenizer(ServerArgs(model_path="dummy")) - self.assertEqual(rc.publish_role(), "tokenizer") - - def test_reset_clears_role(self): - rc.publish(ServerArgs(model_path="dummy"), role="test") - rc.reset_context() - self.assertIsNone(rc.publish_role()) - if __name__ == "__main__": unittest.main()