From a0a22952717a023d7c11210d963d24bb258a9275 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Thu, 27 Aug 2026 12:16:16 -0700 Subject: [PATCH] Refactor server_args constants and layout (#36676) --- python/sglang/srt/constants.py | 7 ++ .../logprob_result_processor.py | 2 +- .../managers/tokenizer_manager_score_mixin.py | 2 +- python/sglang/srt/server_args.py | 100 ++++++++++++------ .../unit/managers/test_embed_overrides.py | 2 +- 5 files changed, 80 insertions(+), 33 deletions(-) diff --git a/python/sglang/srt/constants.py b/python/sglang/srt/constants.py index 2d47c6337..b3e74dd79 100644 --- a/python/sglang/srt/constants.py +++ b/python/sglang/srt/constants.py @@ -11,4 +11,11 @@ GPU_MEMORY_ALL_TYPES = [ HEALTH_CHECK_RID_PREFIX = "HEALTH_CHECK" +# Placeholder token inserted between items in Multi-Item Scoring sequences: +# queryitem1item2... Positions are pre-computed from item +# lengths (multi_item_delimiter_indices); the token only exists for FlashInfer +# attention mask compat and logprob column indexing. Will be removed once the +# attention backend supports position-only MIS. +MIS_DELIMITER_TOKEN_ID = 9999 + GIB_BYTES = 1073741824 # 1024**3 diff --git a/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py b/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py index 8409c4a21..c4eb98ab1 100644 --- a/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/logprob_result_processor.py @@ -10,11 +10,11 @@ from typing import ( import torch from sglang.srt.configs.model_config import ModelConfig +from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID from sglang.srt.layers.logits_processor import LogitsProcessorOutput from sglang.srt.managers.io_struct import build_flat_input_top_logprobs_arrays from sglang.srt.managers.schedule_batch import Req from sglang.srt.runtime_context import get_exec -from sglang.srt.server_args import MIS_DELIMITER_TOKEN_ID logger = logging.getLogger(__name__) diff --git a/python/sglang/srt/managers/tokenizer_manager_score_mixin.py b/python/sglang/srt/managers/tokenizer_manager_score_mixin.py index d7a5b6e6a..60936376b 100644 --- a/python/sglang/srt/managers/tokenizer_manager_score_mixin.py +++ b/python/sglang/srt/managers/tokenizer_manager_score_mixin.py @@ -6,9 +6,9 @@ from typing import Any, Dict, List, Optional, Tuple, Union import torch from sglang.srt.configs.model_config import is_cross_encoding_pooler_model +from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID from sglang.srt.managers.embed_types import PositionalEmbeds from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput -from sglang.srt.server_args import MIS_DELIMITER_TOKEN_ID logger = logging.getLogger(__name__) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 71937631a..66909b9e4 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -11,7 +11,25 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""The arguments of the server.""" +"""Server argument declarations, resolution, and CLI registration. + +Keep this file in the following top-level order: + +1. Imports and the module logger. +2. Public extension-point choice lists, with each legacy ``add_*`` alias + immediately below the choice list it extends. +3. Shared (non-extensible) choice lists, scalar defaults, and deprecated + aliases. A choice list used by only one field belongs inline in that field. +4. ``ServerArgs``: fields first, then resolution/validation helpers, then CLI + registration and small query helpers. New resolution steps are appended at + the end of ``_run_resolution_pipeline``, immediately before resolution is + marked complete, unless an earlier dependency is documented explicitly. +5. Module-level ``ServerArgs`` construction/runtime shims. +6. Networking constants and ``PortArgs``. + +Model- or vendor-specific utilities belong in ``sglang.srt.arg_groups`` (or +their owning subsystem), not before ``ServerArgs`` in this module. +""" from __future__ import annotations @@ -110,10 +128,14 @@ from sglang.utils import is_in_ci logger = logging.getLogger(__name__) -# Define constants -DEFAULT_UVICORN_ACCESS_LOG_EXCLUDE_PREFIXES = () +# -------------------------------------------------------------------------- +# Extension points: out-of-tree platforms and plugins extend these lists +# before ServerArgs is constructed. Each list owns its adder on the line +# below it. A list with no adder is not an extension point -- inline it into +# the field's Arg(choices=...) instead of hoisting it here. +# -------------------------------------------------------------------------- -SAMPLING_BACKEND_CHOICES = {"flashinfer", "pytorch", "ascend"} +# --- Model loading and quantization --- LOAD_FORMAT_CHOICES = [ "auto", @@ -187,6 +209,8 @@ QUANTIZATION_CHOICES = [ ] add_quantization_method_choices = QUANTIZATION_CHOICES.extend +# --- Attention backends --- + ATTENTION_BACKEND_CHOICES = [ # Common "triton", @@ -267,6 +291,8 @@ add_radix_supported_deterministic_attention_backend_choices = ( RADIX_SUPPORTED_DETERMINISTIC_ATTENTION_BACKEND.extend ) +# --- Transport --- + DISAGG_TRANSFER_BACKEND_CHOICES = [ "mooncake", "nixl", @@ -277,15 +303,14 @@ DISAGG_TRANSFER_BACKEND_CHOICES = [ ] add_disagg_transfer_backend_choices = DISAGG_TRANSFER_BACKEND_CHOICES.extend +# --- Sampling and grammar --- + GRAMMAR_BACKEND_CHOICES = ["xgrammar", "outlines", "llguidance", "none"] add_grammar_backend_choices = GRAMMAR_BACKEND_CHOICES.extend -# Placeholder token inserted between items in Multi-Item Scoring sequences: -# queryitem1item2... Positions are pre-computed from item -# lengths (multi_item_delimiter_indices); the token only exists for FlashInfer -# attention mask compat and logprob column indexing. Will be removed once the -# attention backend supports position-only MIS. -MIS_DELIMITER_TOKEN_ID = 9999 +SAMPLING_BACKEND_CHOICES = {"flashinfer", "pytorch", "ascend"} + +# --- MoE and GEMM runners --- MOE_RUNNER_BACKEND_CHOICES = [ "auto", @@ -309,15 +334,6 @@ MOE_RUNNER_BACKEND_CHOICES = [ ] add_moe_runner_backend_choices = MOE_RUNNER_BACKEND_CHOICES.extend -# These architectures take the A2A MoE path and skip post-expert all-reduce. -_DEEPEP_V2_VALIDATED_ARCHITECTURES = frozenset( - { - "DeepseekV3ForCausalLM", - "DeepseekV4ForCausalLM", - "Qwen3MoeForCausalLM", - } -) - MXFP8_MOE_RUNNER_BACKEND_CHOICES = [ "cutlass", "deep_gemm", @@ -349,17 +365,17 @@ FP4_GEMM_RUNNER_BACKEND_CHOICES = [ ] add_fp4_gemm_runner_backend_choices = FP4_GEMM_RUNNER_BACKEND_CHOICES.extend +# --- Cache and scheduling policy --- + RADIX_EVICTION_POLICY_CHOICES = ["lru", "lfu", "slru", "priority"] add_radix_eviction_policy_choices = RADIX_EVICTION_POLICY_CHOICES.extend +# --- Reinforcement learning --- + RL_ON_POLICY_TARGET_CHOICES = ["fsdp"] add_rl_on_policy_target_choices = RL_ON_POLICY_TARGET_CHOICES.extend -# Speculative algorithms whose verify forward presents a uniform per-request -# token width, which is what the LoRA segment layout assumes. -_LORA_SPEC_ALGORITHMS = ("EAGLE", "EAGLE3", "DFLASH", "DSPARK") - -DEFAULT_LORA_EVICTION_POLICY = "lru" +# --- Linear attention --- LINEAR_ATTN_KERNEL_BACKEND_CHOICES = [ "triton", @@ -373,6 +389,12 @@ LINEAR_ATTN_KERNEL_BACKEND_CHOICES = [ ] add_linear_attn_kernel_backend_choices = LINEAR_ATTN_KERNEL_BACKEND_CHOICES.extend +# -------------------------------------------------------------------------- +# Add new extension points at the end of the matching group above. A new +# choice list is inlined into its field by default; hoisting one here makes +# it public API for out-of-tree code and is a deliberate decision. +# -------------------------------------------------------------------------- + @dataclasses.dataclass class ServerArgs: @@ -1491,9 +1513,7 @@ class ServerArgs: nargs="*", ), NS("observability"), - ] = dataclasses.field( - default_factory=lambda: list(DEFAULT_UVICORN_ACCESS_LOG_EXCLUDE_PREFIXES) - ) + ] = dataclasses.field(default_factory=list) crash_dump_folder: A[ Optional[str], "Folder path to dump requests from the last 5 min before a crash (if any). If not specified, crash dumping is disabled.", @@ -7434,10 +7454,17 @@ class ServerArgs: ) architecture = architectures[0] if architectures else None - if architecture not in _DEEPEP_V2_VALIDATED_ARCHITECTURES: + # These architectures take the A2A MoE path and skip post-expert + # all-reduce. + validated_architectures = ( + "DeepseekV3ForCausalLM", + "DeepseekV4ForCausalLM", + "Qwen3MoeForCausalLM", + ) + if architecture not in validated_architectures: raise ValueError( f"DeepEP v2 MoE is not validated for {architecture!r}; supported " - f"architectures are {sorted(_DEEPEP_V2_VALIDATED_ARCHITECTURES)}. " + f"architectures are {sorted(validated_architectures)}. " "Other model workflows may require an all-reduce after A2A " "combine. Use --moe-a2a-backend deepep." ) @@ -10459,7 +10486,10 @@ class ServerArgs: if cfg.speculative_algorithm in ["NGRAM", None]: return - if cfg.speculative_algorithm not in _LORA_SPEC_ALGORITHMS: + # These algorithms present a uniform per-request token width during + # verify, which is what the LoRA segment layout assumes. + lora_spec_algorithms = ("EAGLE", "EAGLE3", "DFLASH", "DSPARK") + if cfg.speculative_algorithm not in lora_spec_algorithms: promoted = ( " (NEXTN/EAGLE with a Gemma4 assistant draft is automatically " "promoted to FROZEN_KV_MTP, which does not support LoRA)" @@ -10785,6 +10815,11 @@ class ServerArgs: return cfg.expert_balancedness_report_mode in ("prometheus", "both") +# -------------------------------------------------------------------------- +# Module-level ServerArgs helpers and runtime shims. +# -------------------------------------------------------------------------- + + def resolve_encoder_transfer_backend( backend: str, model_arch: str, tp_size: int ) -> str: @@ -10925,6 +10960,11 @@ def prepare_server_args(argv: List[str]) -> ServerArgs: return ServerArgs.from_cli_args(raw_args) +# -------------------------------------------------------------------------- +# Networking constants and PortArgs. +# -------------------------------------------------------------------------- + + ZMQ_TCP_PORT_DELTA = 233 DP_ATTENTION_HANDSHAKE_PORT_DELTA = 13 diff --git a/test/registered/unit/managers/test_embed_overrides.py b/test/registered/unit/managers/test_embed_overrides.py index 07f8f6066..015c1d59d 100644 --- a/test/registered/unit/managers/test_embed_overrides.py +++ b/test/registered/unit/managers/test_embed_overrides.py @@ -13,6 +13,7 @@ from unittest.mock import AsyncMock, MagicMock import torch +from sglang.srt.constants import MIS_DELIMITER_TOKEN_ID from sglang.srt.entrypoints.openai.utils import convert_embeds_to_tensors from sglang.srt.managers.embed_types import PositionalEmbeds from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput @@ -20,7 +21,6 @@ from sglang.srt.managers.tokenizer_manager import TokenizerManager from sglang.srt.managers.tokenizer_manager_score_mixin import ( TokenizerManagerScoreMixin, ) -from sglang.srt.server_args import MIS_DELIMITER_TOKEN_ID from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase