diff --git a/python/sglang/srt/arg_groups/arg_utils.py b/python/sglang/srt/arg_groups/arg_utils.py index 8cda59fb4..a9b091e5f 100644 --- a/python/sglang/srt/arg_groups/arg_utils.py +++ b/python/sglang/srt/arg_groups/arg_utils.py @@ -82,6 +82,43 @@ class Arg: resolvable: bool = False +@dataclasses.dataclass(frozen=True) +class NS: + """Namespace-path marker for a ServerArgs field, attached alongside the + field's metadata in ``Annotated``: + + field: A[int, "help", NS("parallel")] = 1 + field: A[str, Arg(help="…"), NS("exec.moe")] = "auto" + + Kept separate from ``Arg`` (CLI metadata) so the ~400 existing bare-string / + multiline field annotations gain a namespace by *appending* one element, + without rewriting each ``Arg(...)`` call. ``namespace_of`` reads it to build + the RuntimeContext config-bag tree.""" + + path: str + + +@functools.lru_cache(maxsize=None) +def namespace_of(cls) -> dict: + """``{field_name: dotted namespace path}`` from the ``NS`` marker in each + field's ``Annotated`` metadata. + + Fields without an ``NS`` marker are absent from the map (the coverage lint + flags them). Non-dataclass types yield an empty map.""" + if not dataclasses.is_dataclass(cls): + return {} + hints = get_type_hints(cls, include_extras=True) + out = {} + for field in dataclasses.fields(cls): + tp = hints.get(field.name, field.type) + if get_origin(tp) is Annotated: + for a in get_args(tp)[1:]: + if isinstance(a, NS): + out[field.name] = a.path + break + return out + + @functools.lru_cache(maxsize=None) def resolvable_fields(cls) -> frozenset: """Names of ``cls`` dataclass fields whose ``Arg`` metadata declares diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 9c8801bae..6ceaea6c2 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -33,7 +33,7 @@ from typing import Any, Callable, Dict, List, Literal, Optional, Union from sglang.jit_kernel.kv_canary.consts import RealKvHashMode from sglang.kernels.ops.attention.fla.chunk_delta_h import CHUNK_SIZE as FLA_CHUNK_SIZE -from sglang.srt.arg_groups.arg_utils import A, Arg, add_cli_args_from_dataclass +from sglang.srt.arg_groups.arg_utils import NS, A, Arg, add_cli_args_from_dataclass from sglang.srt.arg_groups.argparse_actions import ( DeprecatedAction, DeprecatedAliasStoreAction, @@ -459,8 +459,9 @@ class ServerArgs: help="The path of the model weights. This can be a local folder or a Hugging Face repo ID.", aliases=["--model"], ), + NS("model"), ] - tokenizer_path: A[Optional[str], "The path of the tokenizer."] = None + tokenizer_path: A[Optional[str], "The path of the tokenizer.", NS("serving")] = None tokenizer_mode: A[ str, Arg( @@ -468,6 +469,7 @@ class ServerArgs: "and 'slow' will always use the slow tokenizer.", choices=["auto", "slow"], ), + NS("serving"), ] = "auto" tokenizer_backend: A[ str, @@ -477,11 +479,18 @@ class ServerArgs: "for faster tokenization. Requires the fastokens package to be installed.", choices=["huggingface", "fastokens"], ), + NS("serving"), ] = "huggingface" - tokenizer_worker_num: A[int, "The worker num of the tokenizer manager."] = 1 - detokenizer_worker_num: A[int, "The worker num of the detokenizer manager."] = 1 + tokenizer_worker_num: A[ + int, "The worker num of the tokenizer manager.", NS("serving") + ] = 1 + detokenizer_worker_num: A[ + int, "The worker num of the detokenizer manager.", NS("serving") + ] = 1 skip_tokenizer_init: A[ - bool, "If set, skip init tokenizer and pass input_ids in generate request." + bool, + "If set, skip init tokenizer and pass input_ids in generate request.", + NS("serving"), ] = False load_format: A[ str, @@ -504,14 +513,17 @@ class ServerArgs: "smaller.", choices=LOAD_FORMAT_CHOICES, ), + NS("model"), ] = "auto" model_loader_extra_config: A[ str, "Extra config for model loader. This will be passed to the model loader corresponding to the chosen load_format.", + NS("model"), ] = "{}" trust_remote_code: A[ bool, "Whether or not to allow for custom models defined on the Hub in their own modeling files.", + NS("model"), ] = False context_length: A[ Optional[int], @@ -520,15 +532,20 @@ class ServerArgs: f"\n\n{human_readable_int.__doc__}", type_parser=human_readable_int, ), + NS("model"), ] = None - is_embedding: A[bool, "Whether to use a CausalLM as an embedding model."] = False + is_embedding: A[ + bool, "Whether to use a CausalLM as an embedding model.", NS("model") + ] = False enable_multimodal: A[ Optional[bool], "Enable the multimodal functionality for the served model. If the model being served is not multimodal, nothing will happen", + NS("mm"), ] = None revision: A[ Optional[str], "The specific model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.", + NS("model"), ] = None model_impl: A[ str, @@ -544,6 +561,7 @@ class ServerArgs: "implementation.\n" ) ), + NS("model"), ] = "auto" model_config_parser: A[ str, @@ -555,10 +573,12 @@ class ServerArgs: "parsers via @register_model_config_parser." ) ), + NS("model"), ] = "auto" json_model_override_args: A[ str, "A dictionary in JSON string format used to override default model configurations.", + NS("model"), ] = "{}" # ------------------------------------------------------------------------- @@ -580,6 +600,7 @@ class ServerArgs: choices=["auto", "half", "float16", "bfloat16", "float", "float32"], resolvable=True, ), + NS("model"), ] = "auto" quantization: A[ Optional[str], @@ -588,6 +609,7 @@ class ServerArgs: choices=QUANTIZATION_CHOICES, resolvable=True, ), + NS("model"), ] = None quantization_param_path: A[ Optional[str], @@ -600,6 +622,7 @@ class ServerArgs: ), type_parser=nullable_str, ), + NS("model"), ] = None kv_cache_dtype: A[ str, @@ -626,9 +649,10 @@ class ServerArgs: ], resolvable=True, ), + NS("model"), ] = "auto" enable_fp32_lm_head: A[ - bool, "If set, the LM head outputs (logits) are in FP32." + bool, "If set, the LM head outputs (logits) are in FP32.", NS("exec.features") ] = False modelopt_quant: A[ Optional[Union[str, Dict]], @@ -638,6 +662,7 @@ class ServerArgs: "NVIDIA Model Optimizer library to be installed: pip install " "nvidia-modelopt" ), + NS("model"), ] = None modelopt_checkpoint_restore_path: A[ Optional[str], @@ -646,6 +671,7 @@ class ServerArgs: "If provided, the quantization process will be skipped and the model " "will be loaded from this checkpoint." ), + NS("model"), ] = None modelopt_checkpoint_save_path: A[ Optional[str], @@ -653,6 +679,7 @@ class ServerArgs: "Path to save the ModelOpt quantized checkpoint after quantization. " "This allows reusing the quantized model in future runs." ), + NS("model"), ] = None modelopt_export_path: A[ Optional[str], @@ -662,6 +689,7 @@ class ServerArgs: "with SGLang for inference. If not provided, the model will not be " "exported." ), + NS("model"), ] = None quantize_and_serve: A[ bool, @@ -671,10 +699,12 @@ class ServerArgs: "production, it's recommended to use separate quantization and " "deployment steps." ), + NS("model"), ] = False rl_quant_profile: A[ Optional[str], "Path to the FlashRL quantization profile. Required when using --load-format flash_rl.", + NS("model"), ] = None # For flash_rl load format enable_tf32_matmul: A[ bool, @@ -682,6 +712,7 @@ class ServerArgs: help="Enable float32 matmuls to use TensorFloat32 precision for better performance (via torch.set_float32_matmul_precision). CUDA only.", resolvable=True, ), + NS("exec.features"), ] = False # ------------------------------------------------------------------------- @@ -690,13 +721,15 @@ class ServerArgs: mem_fraction_static: A[ Optional[float], "The fraction of the memory used for static allocation (model weights and KV cache memory pool). Use a smaller value if you see out-of-memory errors.", + NS("schedule"), ] = None max_running_requests: A[ - Optional[int], "The maximum number of running requests." + Optional[int], "The maximum number of running requests.", NS("schedule") ] = None max_queued_requests: A[ Optional[int], "The maximum number of queued requests. This option is ignored when using disaggregation-mode.", + NS("schedule"), ] = None max_total_tokens: A[ Optional[int], @@ -710,14 +743,17 @@ class ServerArgs: ), type_parser=human_readable_int, ), + NS("schedule"), ] = None chunked_prefill_size: A[ Optional[int], "The maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked prefill.", + NS("schedule"), ] = None enable_dynamic_chunking: A[ bool, "Enable dynamic chunk size adjustment for pipeline parallelism. When enabled, chunk sizes are dynamically calculated based on fitted function to maintain consistent execution time across chunks.", + NS("schedule"), ] = False max_prefill_tokens: A[ int, @@ -729,10 +765,12 @@ class ServerArgs: ), type_parser=human_readable_int, ), + NS("schedule"), ] = 16384 prefill_max_requests: A[ Optional[int], "The maximum number of requests in a prefill batch. If not specified, there is no limit.", + NS("schedule"), ] = None schedule_policy: A[ str, @@ -748,28 +786,35 @@ class ServerArgs: "routing-key", ], ), + NS("schedule"), ] = "fcfs" enable_priority_scheduling: A[ bool, "Enable priority scheduling. Requests with higher priority integer values will be scheduled first by default.", + NS("schedule"), + ] = False + disable_priority_preemption: A[ + bool, "Disable priority scheduling preemption.", NS("schedule") ] = False - disable_priority_preemption: A[bool, "Disable priority scheduling preemption."] = ( - False - ) default_priority_value: A[ - Optional[int], "Default priority for requests without explicit priority." + Optional[int], + "Default priority for requests without explicit priority.", + NS("schedule"), ] = None abort_on_priority_when_disabled: A[ bool, "If set, abort requests that specify a priority when priority scheduling is disabled.", + NS("schedule"), ] = False schedule_low_priority_values_first: A[ bool, "If specified with --enable-priority-scheduling, the scheduler will schedule requests with lower priority integer values first.", + NS("schedule"), ] = False priority_scheduling_preemption_threshold: A[ int, "Minimum difference in priorities for an incoming request to have to preempt running request(s).", + NS("schedule"), ] = 10 retraction_policy: A[ str, @@ -783,14 +828,17 @@ class ServerArgs: ), choices=RETRACTION_POLICY_CHOICES, ), + NS("schedule"), ] = "length" schedule_conservativeness: A[ float, "How conservative the schedule policy is. A larger value means more conservative scheduling. Use a larger value if you see requests being retracted frequently.", + NS("schedule"), ] = 1.0 page_size: A[ Optional[int], Arg(help="The number of tokens in a page.", resolvable=True), + NS("schedule"), ] = None swa_full_tokens_ratio: A[ float, @@ -803,10 +851,12 @@ class ServerArgs: ), resolvable=True, ), + NS("schedule"), ] = 0.8 disable_hybrid_swa_memory: A[ bool, Arg(help="Disable the hybrid SWA memory pool.", resolvable=True), + NS("schedule"), ] = False radix_eviction_policy: A[ str, @@ -819,12 +869,16 @@ class ServerArgs: ), choices=RADIX_EVICTION_POLICY_CHOICES, ), + NS("memory"), ] = "lru" prefill_only_disable_kv_cache: A[ bool, "Skip the physical KV cache allocation for embedding-mode prefill-only workloads. Currently only valid with --is-embedding, --chunked-prefill-size=-1, --disable-radix-cache, an FA prefill backend, and non-FP4 KV cache so the fa_skip_kv_cache path is active (no layer reads or writes the cache). Other prefill-only workloads such as scoring/MIS may benefit from this later once their attention paths stop using paged KV. Scheduler admission accounting is unchanged; per-layer K/V tensors are sized to (page_size, head_num, head_dim) placeholders so GPU memory is not wasted.", + NS("schedule"), + ] = False + disable_radix_cache: A[ + bool, "Disable RadixAttention for prefix caching.", NS("memory") ] = False - disable_radix_cache: A[bool, "Disable RadixAttention for prefix caching."] = False enable_page_major_kv_layout: A[ bool, "Enable the page-major KV layout: lay out the Mamba state and full/SWA " @@ -832,6 +886,7 @@ class ServerArgs: "layer-major within a page) instead of the default per-layer " "(layer-major) layout. Requires the Triton attention / linear-attn / " "Mamba backends.", + NS("memory"), ] = False enable_unified_memory: A[ bool, @@ -840,10 +895,12 @@ class ServerArgs: "sub-pools. Requires the Triton attention / linear-attn / Mamba " "backends; not yet compatible with PD disaggregation or speculative " "decoding.", + NS("memory"), ] = False disable_chunked_prefix_cache: A[ bool, "Disable chunked prefix cache feature for deepseek, which should save overhead for short sequences.", + NS("schedule"), ] = False disable_overlap_schedule: A[ bool, @@ -851,18 +908,22 @@ class ServerArgs: help="Disable the overlap scheduler, which overlaps the CPU scheduler with GPU model worker.", resolvable=True, ), + NS("schedule"), ] = False num_continuous_decode_steps: A[ int, "Run multiple continuous decoding steps to reduce scheduling overhead. This can potentially increase throughput but may also increase time-to-first-token latency. The default value is 1, meaning only run one decoding step at a time.", + NS("schedule"), ] = 1 scheduler_recv_interval: A[ int, "The interval to poll requests in scheduler. Can be set to >1 to reduce the overhead of this.", + NS("schedule"), ] = 1 enable_mixed_chunk: A[ bool, "Enabling mixing prefill and decode in a batch when using chunked prefill.", + NS("schedule"), ] = False # ------------------------------------------------------------------------- @@ -871,10 +932,12 @@ class ServerArgs: nccl_port: A[ Optional[int], "The port for NCCL distributed environment setup. Defaults to a random port.", + NS("parallel"), ] = None dist_timeout: A[ Optional[int], "Set timeout for torch.distributed initialization.", + NS("parallel"), ] = None dist_init_addr: A[ Optional[str], @@ -882,15 +945,17 @@ class ServerArgs: help="The host address for initializing distributed backend (e.g., `192.168.0.2:25000`).", aliases=["--nccl-init-addr"], ), + NS("parallel"), ] = None - nnodes: A[int, "The number of nodes."] = 1 - node_rank: A[int, "The node rank."] = 0 + nnodes: A[int, "The number of nodes.", NS("parallel")] = 1 + node_rank: A[int, "The node rank.", NS("parallel")] = 0 tp_size: A[ int, Arg( help="The tensor parallelism size.", aliases=["--tensor-parallel-size"], ), + NS("parallel"), ] = 1 dcp_size: A[ int, @@ -898,6 +963,7 @@ class ServerArgs: help="The decode context parallelism size.", aliases=["--decode-context-parallel-size"], ), + NS("parallel"), ] = 1 pp_size: A[ int, @@ -905,18 +971,23 @@ class ServerArgs: help="The pipeline parallelism size.", aliases=["--pipeline-parallel-size"], ), + NS("parallel"), ] = 1 pp_max_micro_batch_size: A[ Optional[int], "The maximum micro batch size in pipeline parallelism.", + NS("parallel"), ] = None - pp_async_batch_depth: A[int, "The async batch depth of pipeline parallelism."] = 0 + pp_async_batch_depth: A[ + int, "The async batch depth of pipeline parallelism.", NS("parallel") + ] = 0 dp_size: A[ int, Arg( help="The data parallelism size.", aliases=["--data-parallel-size"], ), + NS("parallel"), ] = 1 load_balance_method: A[ str, @@ -930,6 +1001,7 @@ class ServerArgs: "total_tokens", ], ), + NS("parallel"), ] = "auto" attn_cp_size: A[ int, @@ -938,6 +1010,7 @@ class ServerArgs: aliases=["--attention-context-parallel-size"], resolvable=True, ), + NS("parallel"), ] = 1 moe_dp_size: A[ int, @@ -945,6 +1018,7 @@ class ServerArgs: help="The moe data parallelism size.", aliases=["--moe-data-parallel-size"], ), + NS("parallel"), ] = 1 dcp_size: A[ int, @@ -952,6 +1026,7 @@ class ServerArgs: help="The decode context parallelism size.", aliases=["--decode-context-parallel-size"], ), + NS("parallel"), ] = 1 dwdp_size: A[ int, @@ -960,10 +1035,12 @@ class ServerArgs: "When > 1, MoE prefill uses weight prefetch instead of token all-to-all. " "Must equal tp_size. Only supported with --disaggregation-mode null or prefill.", ), + NS("parallel"), ] = 1 enable_prefill_cp: A[ bool, "Enable context parallelism for the prefill phase. Select the layout with --cp-strategy.", + NS("parallel"), ] = False cp_strategy: A[ Optional[str], @@ -971,16 +1048,20 @@ class ServerArgs: help="Sharding strategy for prefill CP. 'zigzag' is the former in-seq-split mode; 'interleave' is the former round-robin-split mode.", choices=("zigzag", "interleave"), ), + NS("parallel"), ] = None # Split DSA GPU KV/indexer cache layers across CP ranks. enable_dsa_cache_layer_split: A[ bool, "Split DSA (DeepSeek Sparse Attention) GPU KV/indexer cache layers across context-parallel ranks to reduce per-rank KV memory. Currently only supported with the mooncake transfer backend (mooncake / mooncake_tcp); mori/nixl support will be added later by the community.", + NS("parallel"), ] = False - enable_dsa_prefill_context_parallel: A[bool, Arg(no_cli=True)] = False - dsa_prefill_cp_mode: A[str, Arg(no_cli=True)] = "round-robin-split" - enable_prefill_context_parallel: A[bool, Arg(no_cli=True)] = False - prefill_cp_mode: A[str, Arg(no_cli=True)] = "in-seq-split" + enable_dsa_prefill_context_parallel: A[bool, Arg(no_cli=True), NS("parallel")] = ( + False + ) + dsa_prefill_cp_mode: A[str, Arg(no_cli=True), NS("parallel")] = "round-robin-split" + enable_prefill_context_parallel: A[bool, Arg(no_cli=True), NS("parallel")] = False + prefill_cp_mode: A[str, Arg(no_cli=True), NS("parallel")] = "in-seq-split" # DP attention enable_dp_attention: A[ bool, @@ -988,10 +1069,12 @@ class ServerArgs: help="Enabling data parallelism for attention and tensor parallelism for FFN. The dp size should be equal to the tp size. Currently DeepSeek-V2 and Qwen 2/3 MoE models are supported.", resolvable=True, ), + NS("parallel"), ] = False enable_dp_attention_local_control_broadcast: A[ bool, "With DP-attention, send control messages to every DP group leader and broadcast within attn_tp_group instead of the full tp_group. Eliminates a costly all-ranks gloo sync on every scheduler iteration.", + NS("parallel"), ] = False enable_dp_lm_head: A[ bool, @@ -999,10 +1082,12 @@ class ServerArgs: help="Enable vocabulary parallel across the attention TP group to avoid all-gather across DP groups, optimizing performance under DP attention.", resolvable=True, ), + NS("parallel"), ] = False enable_attn_tp_input_scattered: A[ bool, "Allow input of attention to be scattered when only using tensor parallelism, to reduce the computational load of operations such as qkv latent.", + NS("parallel"), ] = False disable_attn_tp_gather: A[ bool, @@ -1014,10 +1099,12 @@ class ServerArgs: "Without this, the cuda graph runner pads num_tokens to attn_tp_size, " "which can cause kernel autotuners to select wrong-sized variants " "at small batches.", + NS("parallel"), ] = False enable_p2p_check: A[ bool, "Enable P2P check for GPU access, otherwise the p2p access is allowed by default.", + NS("parallel"), ] = False # ------------------------------------------------------------------------- @@ -1026,50 +1113,65 @@ class ServerArgs: device: A[ Optional[str], "The device to use ('cuda', 'xpu', 'hpu', 'npu', 'cpu', 'musa'). Defaults to auto-detection if not specified.", + NS("device"), ] = None base_gpu_id: A[ int, "The base GPU ID to start allocating GPUs from. Useful when running multiple instances on the same machine.", + NS("device"), ] = 0 gpu_id_step: A[ int, "The delta between consecutive GPU IDs that are used. For example, setting it to 2 will use GPU 0,2,4,...", + NS("device"), ] = 1 - random_seed: A[Optional[int], "The random seed."] = None + random_seed: A[Optional[int], "The random seed.", NS("device")] = None watchdog_timeout: A[ float, "Set watchdog timeout in seconds. If a forward batch takes longer than this, the server will crash to prevent hanging.", + NS("device"), ] = 300 soft_watchdog_timeout: A[ Optional[float], "Set soft watchdog timeout in seconds. If a forward batch takes longer than this, the server will dump information for debugging.", + NS("device"), ] = None - sleep_on_idle: A[bool, "Reduce CPU usage when sglang is idle."] = False - use_ray: A[bool, "Use Ray actors for scheduler process management."] = False - custom_sigquit_handler: Optional[Callable] = None + sleep_on_idle: A[bool, "Reduce CPU usage when sglang is idle.", NS("device")] = ( + False + ) + use_ray: A[ + bool, "Use Ray actors for scheduler process management.", NS("device") + ] = False + custom_sigquit_handler: A[Optional[Callable], NS("device")] = None numa_node: A[ Optional[List[int]], "Sets the numa node for the subprocesses. i-th element corresponds to i-th subprocess. If unset, will be automatically detected on NUMA systems.", + NS("device"), ] = None gc_threshold: A[ Optional[List[int]], "Set the garbage collection thresholds (the collection frequency). Accepts 1 to 3 integers.", + NS("device"), ] = None # ------------------------------------------------------------------------- # HTTP server # ------------------------------------------------------------------------- - host: A[str, "The host of the HTTP server."] = "127.0.0.1" - port: A[int, "The port of the HTTP server."] = 30000 - fastapi_root_path: A[str, "App is behind a path based routing proxy."] = "" + host: A[str, "The host of the HTTP server.", NS("serving")] = "127.0.0.1" + port: A[int, "The port of the HTTP server.", NS("serving")] = 30000 + fastapi_root_path: A[ + str, "App is behind a path based routing proxy.", NS("serving") + ] = "" smg_grpc_mode: A[ bool, "Use the legacy SMG gRPC server (smg-grpc-servicer) instead of the HTTP " "server. Replaces the deprecated --grpc-mode.", + NS("serving"), ] = False grpc_mode: A[ bool, "(Deprecated, use --smg-grpc-mode) Legacy SMG gRPC server selector.", + NS("serving"), ] = False grpc_port: A[ Optional[int], @@ -1077,29 +1179,37 @@ class ServerArgs: "(or SGLANG_GRPC_PORT) enables the native gRPC server; it is off by " "default. In legacy --smg-grpc-mode this is the SMG server port and " "defaults to --port + 10000.", + NS("serving"), ] = None - skip_server_warmup: A[bool, "If set, skip warmup."] = False + skip_server_warmup: A[bool, "If set, skip warmup.", NS("serving")] = False warmups: A[ Optional[str], "Specify custom warmup functions (csv) to run before server starts eg. --warmups=warmup_name1,warmup_name2 will run the functions `warmup_name1` and `warmup_name2` specified in warmup.py before the server starts listening for requests", + NS("serving"), ] = None enable_http2: A[ bool, "Use Granian instead of Uvicorn as the ASGI server, enabling HTTP/1.1 and HTTP/2 auto-negotiation. Clients may use h2c (cleartext HTTP/2) or plain HTTP/1.1. Requires 'pip install sglang[http2]'.", + NS("serving"), ] = False # ------------------------------------------------------------------------- # SSL/TLS # ------------------------------------------------------------------------- - ssl_keyfile: A[Optional[str], "The file path to the SSL key file."] = None - ssl_certfile: A[Optional[str], "The file path to the SSL certificate file."] = None - ssl_ca_certs: A[Optional[str], "The CA certificates file."] = None + ssl_keyfile: A[ + Optional[str], "The file path to the SSL key file.", NS("serving") + ] = None + ssl_certfile: A[ + Optional[str], "The file path to the SSL certificate file.", NS("serving") + ] = None + ssl_ca_certs: A[Optional[str], "The CA certificates file.", NS("serving")] = None ssl_keyfile_password: A[ - Optional[str], "The password to decrypt the SSL keyfile." + Optional[str], "The password to decrypt the SSL keyfile.", NS("serving") ] = None enable_ssl_refresh: A[ bool, "Enable automatic SSL certificate hot-reloading when cert/key files change on disk. Requires --ssl-certfile and --ssl-keyfile.", + NS("serving"), ] = False # ------------------------------------------------------------------------- @@ -1108,39 +1218,47 @@ class ServerArgs: api_key: A[ Optional[str], "Set API key of the server. It is also used in the OpenAI API compatible server.", + NS("serving"), ] = None admin_api_key: A[ Optional[str], "Set admin API key for sensitive management endpoints (e.g. /clear_hicache_storage_backend). When set, admin endpoints require this key and do NOT accept --api-key.", + NS("serving"), ] = None served_model_name: A[ Optional[str], "Override the model name returned by the v1/models endpoint in OpenAI API server.", + NS("serving"), ] = None weight_version: A[ str, "Version identifier for the model weights. Defaults to 'default' if not specified.", + NS("serving"), ] = "default" chat_template: A[ Optional[str], "The buliltin chat template name or the path of the chat template file. This is only used for OpenAI-compatible API server.", + NS("serving"), ] = None hf_chat_template_name: A[ Optional[str], "When the HuggingFace tokenizer has multiple chat templates (e.g., 'default', 'tool_use', 'rag'), specify which named template to use. If not set, the first available template is used.", + NS("serving"), ] = None completion_template: A[ Optional[str], "The buliltin completion template name or the path of the completion template file. This is only used for OpenAI-compatible API server. only for code completion currently.", + NS("serving"), ] = None - file_storage_path: A[str, "The path of the file storage in backend."] = ( - "sglang_storage" - ) + file_storage_path: A[ + str, "The path of the file storage in backend.", NS("serving") + ] = "sglang_storage" enable_cache_report: A[ bool, "Return number of cached tokens in usage.prompt_tokens_details for each openai request.", + NS("serving"), ] = False - reasoning_parser: Optional[str] = None + reasoning_parser: A[Optional[str], NS("serving")] = None default_chat_template_kwargs: A[ Optional[Dict[str, Any]], Arg( @@ -1150,19 +1268,23 @@ class ServerArgs: "chat_template_kwargs takes precedence.", type_parser=json.loads, ), + NS("serving"), ] = None strip_thinking_cache: A[ bool, "Skip caching reasoning-model output (thinking + answer) in the radix tree on finish; keep only the prompt prefix. Opt-in: changes cache contents.", + NS("serving"), ] = False enable_strict_thinking: A[ bool, "Enable strict token filtering during the thinking phase. Blocks model-specific excluded tokens (e.g., tool call markers) during reasoning. Requires a grammar backend that supports token filtering.", + NS("serving"), ] = False - tool_call_parser: Optional[str] = None + tool_call_parser: A[Optional[str], NS("serving")] = None tool_server: A[ Optional[str], "Either 'demo' or a comma-separated list of tool server urls to use for the model. If not specified, no tool server will be used.", + NS("serving"), ] = None sampling_defaults: A[ str, @@ -1170,14 +1292,17 @@ class ServerArgs: help="Where to get default sampling parameters. 'openai' uses SGLang/OpenAI defaults (temperature=1.0, top_p=1.0, etc.). 'model' uses the model's generation_config.json to get the recommended sampling parameters if available. Default is 'model'.", choices=["openai", "model"], ), + NS("serving"), ] = "model" asr_max_buffer_seconds: A[ int, "Maximum seconds of PCM audio the streaming ASR WebSocket handler will accumulate before closing the session with a buffer_overflow error. Guards against OOM when a client streams audio faster than inference can consume it. Default 60s.", + NS("serving"), ] = 60 asr_max_concurrent_sessions: A[ int, "Maximum number of concurrent realtime ASR WebSocket sessions served by /v1/realtime. New connections beyond this cap are accepted, sent an error{code:too_many_sessions} frame, and closed. Default 32.", + NS("serving"), ] = 32 preferred_sampling_params: A[ Optional[str], @@ -1185,10 +1310,12 @@ class ServerArgs: help="json-formatted sampling settings that will be returned in /get_model_info", type_parser=json.loads, ), + NS("serving"), ] = None allow_auto_truncate: A[ bool, "Allow automatically truncating requests that exceed the maximum input length instead of returning an error.", + NS("serving"), ] = False # ------------------------------------------------------------------------- @@ -1197,39 +1324,45 @@ class ServerArgs: stream_interval: A[ int, "The interval (or buffer size) for streaming in terms of the token length. A smaller value makes streaming smoother, while a larger value makes the throughput higher", + NS("serving"), ] = 1 batch_notify_size: A[ int, "Number of streaming notifications to batch before yielding to the event loop. Reduces asyncio wakeup overhead under high concurrency.", + NS("serving"), ] = 16 stream_response_default_include_usage: A[ bool, "Include usage in every streaming response (even when stream_options is not specified).", + NS("serving"), ] = False incremental_streaming_output: A[ - bool, - "Whether to output as a sequence of disjoint segments.", + bool, "Whether to output as a sequence of disjoint segments.", NS("serving") ] = False enable_streaming_session: A[ bool, "Enable streaming session mode and StreamingSession wrapper.", + NS("serving"), ] = False enable_session_radix_cache: A[ bool, "Hold per-session KV as ordinary evictable radix entries, tagged by session id and bulk-evicted on close. Requires --radix-eviction-policy priority.", + NS("memory"), ] = False # ------------------------------------------------------------------------- # Logging, metrics, and tracing # ------------------------------------------------------------------------- - log_level: A[str, "The logging level of all loggers."] = "info" + log_level: A[str, "The logging level of all loggers.", NS("observability")] = "info" log_level_http: A[ Optional[str], "The logging level of HTTP server. If not set, reuse --log-level by default.", + NS("observability"), ] = None log_requests: A[ bool, "Log metadata, inputs, outputs of all requests. The verbosity is decided by --log-requests-level", + NS("observability"), ] = False log_requests_level: A[ int, @@ -1237,6 +1370,7 @@ class ServerArgs: help="0: Log metadata (no sampling parameters). 1: Log metadata and sampling parameters. 2: Log metadata, sampling parameters and partial input/output. 3: Log every input/output.", choices=[0, 1, 2, 3], ), + NS("observability"), ] = 2 log_requests_format: A[ str, @@ -1244,10 +1378,12 @@ class ServerArgs: help="Format for request logging: 'text' (human-readable) or 'json' (structured)", choices=["text", "json"], ), + NS("observability"), ] = "text" log_requests_target: A[ Optional[List[str]], "Target(s) for request logging: 'stdout' and/or directory path(s) for file output. Can specify multiple targets, e.g., '--log-requests-target stdout /my/path'. ", + NS("observability"), ] = None uvicorn_access_log_exclude_prefixes: A[ List[str], @@ -1255,40 +1391,51 @@ class ServerArgs: help="Exclude uvicorn access logs whose request path starts with any of these prefixes. Defaults to empty (disabled). Example: --uvicorn-access-log-exclude-prefixes /metrics /health", nargs="*", ), + NS("observability"), ] = dataclasses.field( default_factory=lambda: list(DEFAULT_UVICORN_ACCESS_LOG_EXCLUDE_PREFIXES) ) crash_dump_folder: A[ Optional[str], "Folder path to dump requests from the last 5 min before a crash (if any). If not specified, crash dumping is disabled.", + NS("observability"), ] = None - show_time_cost: A[bool, "Show time cost of custom marks."] = False - enable_metrics: A[bool, "Enable log prometheus metrics."] = False + show_time_cost: A[bool, "Show time cost of custom marks.", NS("observability")] = ( + False + ) + enable_metrics: A[bool, "Enable log prometheus metrics.", NS("observability")] = ( + False + ) smg_http_sidecar_port: A[ Optional[int], Arg( help="Port for the HTTP sidecar server in legacy SMG gRPC mode (--smg-grpc-mode). Serves Prometheus metrics and profiling endpoints. Defaults to --port + 1. Not used in HTTP mode.", aliases=["--grpc-http-sidecar-port"], ), + NS("observability"), ] = None - enable_mfu_metrics: A[bool, "Enable estimated MFU-related prometheus metrics."] = ( - False - ) + enable_mfu_metrics: A[ + bool, "Enable estimated MFU-related prometheus metrics.", NS("observability") + ] = False enable_metrics_for_all_schedulers: A[ bool, "Enable --enable-metrics-for-all-schedulers when you want schedulers on all TP ranks (not just TP 0) to record request metrics separately. This is especially useful when dp_attention is enabled, as otherwise all metrics appear to come from TP 0.", + NS("observability"), ] = False load_snapshot_publish_interval: A[ int, "Publish load snapshot to shared memory every N decode iterations. Prefill and idle always publish immediately.", + NS("observability"), ] = 15 tokenizer_metrics_custom_labels_header: A[ str, "Specify the HTTP header for passing custom labels for tokenizer metrics.", + NS("observability"), ] = "x-custom-labels" tokenizer_metrics_allowed_custom_labels: A[ Optional[List[str]], "The custom labels allowed for tokenizer metrics. The labels are specified via a dict in '--tokenizer-metrics-custom-labels-header' field in HTTP requests, e.g., {'label1': 'value1', 'label2': 'value2'} is allowed if '--tokenizer-metrics-allowed-custom-labels label1 label2' is set.", + NS("observability"), ] = None extra_metric_labels: A[ Optional[Dict[str, str]], @@ -1296,18 +1443,22 @@ class ServerArgs: help='The custom labels for metrics. e.g. \'{"label1": "value1", "label2": "value2"}\'', type_parser=json.loads, ), + NS("observability"), ] = None bucket_time_to_first_token: A[ Optional[List[float]], "The buckets of time to first token, specified as a list of floats.", + NS("observability"), ] = None bucket_inter_token_latency: A[ Optional[List[float]], "The buckets of inter-token latency, specified as a list of floats.", + NS("observability"), ] = None bucket_e2e_request_latency: A[ Optional[List[float]], "The buckets of end-to-end request latency, specified as a list of floats.", + NS("observability"), ] = None prompt_tokens_buckets: A[ Optional[List[str]], @@ -1316,6 +1467,7 @@ class ServerArgs: "generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets " "[984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'custom " " ...' uses custom bucket values (e.g., 'custom 10 50 100 500').", + NS("observability"), ] = None generation_tokens_buckets: A[ Optional[List[str]], @@ -1324,45 +1476,58 @@ class ServerArgs: "generates two sides exponential distributed buckets (e.g., 'tse 1000 2 8' generates buckets " "[984.0, 992.0, 996.0, 998.0, 1000.0, 1002.0, 1004.0, 1008.0, 1016.0]).); 'custom " " ...' uses custom bucket values (e.g., 'custom 10 50 100 500').", + NS("observability"), ] = None gc_warning_threshold_secs: A[ float, "The threshold for long GC warning. If a GC takes longer than this, a warning will be logged. Set to 0 to disable.", + NS("observability"), ] = 0.0 decode_log_interval: A[ int, "The log and metrics reporting interval (in decode iterations) for decode batches.", + NS("observability"), ] = 40 enable_request_time_stats_logging: A[ - bool, "Enable per request time stats logging" + bool, "Enable per request time stats logging", NS("observability") ] = False kv_events_config: A[ Optional[str], "Config in json format for NVIDIA dynamo KV event publishing. Publishing will be enabled if this flag is used.", + NS("observability"), ] = None enable_forward_pass_metrics: A[ bool, "Enable per-iteration forward pass metrics via ZMQ IPC. External consumers (e.g. Dynamo planner) subscribe to the IPC endpoint exposed in server_args.forward_pass_metrics_ipc_name.", + NS("observability"), ] = False - forward_pass_metrics_worker_id: A[str, Arg(help=argparse.SUPPRESS)] = "" - forward_pass_metrics_ipc_name: A[Optional[str], Arg(help=argparse.SUPPRESS)] = None - enable_trace: A[bool, "Enable opentelemetry trace"] = False + forward_pass_metrics_worker_id: A[ + str, Arg(help=argparse.SUPPRESS), NS("observability") + ] = "" + forward_pass_metrics_ipc_name: A[ + Optional[str], Arg(help=argparse.SUPPRESS), NS("observability") + ] = None + enable_trace: A[bool, "Enable opentelemetry trace", NS("observability")] = False trace_modules: A[ str, "Select the components to trace. Available options are 'request' and 'mooncake'. Format: ,,...", + NS("observability"), ] = "request" otlp_traces_endpoint: A[ str, "Config opentelemetry collector endpoint if --enable-trace is set. format: :", + NS("observability"), ] = "localhost:4317" # RequestMetricsExporter configuration export_metrics_to_file: A[ bool, "Export performance metrics for each request to local file (e.g. for forwarding to external systems).", + NS("observability"), ] = False export_metrics_to_file_dir: A[ Optional[str], "Directory path for writing performance metrics files (required when --export-metrics-to-file is enabled).", + NS("observability"), ] = None # Class-level DI for the five *MetricsCollector classes. Maps collector role # (one of: "scheduler", "tokenizer", "storage", "radix_cache", "expert_dispatch") @@ -1370,7 +1535,7 @@ class ServerArgs: # read from this map and fall back to the base class. Class-object only (no # CLI surface) since this exists for embedded use cases that pass a Python # class directly. Default None preserves existing behavior. - stat_loggers: Optional[Dict[str, type]] = None + stat_loggers: A[Optional[Dict[str, type]], NS("observability")] = None # ------------------------------------------------------------------------- # Constrained decoding @@ -1378,10 +1543,12 @@ class ServerArgs: constrained_json_whitespace_pattern: A[ Optional[str], "(outlines and llguidance backends only) Regex pattern for syntactic whitespaces allowed in JSON constrained output. For example, to allow the model generate consecutive whitespaces, set the pattern to [\n\t ]*", + NS("serving"), ] = None constrained_json_disable_any_whitespace: A[ bool, "(xgrammar and llguidance backends only) Enforce compact representation in JSON constrained output.", + NS("serving"), ] = False # ------------------------------------------------------------------------- @@ -1394,6 +1561,7 @@ class ServerArgs: choices=ATTENTION_BACKEND_CHOICES, resolvable=True, ), + NS("exec.kernel"), ] = None decode_attention_backend: A[ Optional[str], @@ -1402,6 +1570,7 @@ class ServerArgs: choices=ATTENTION_BACKEND_CHOICES, resolvable=True, ), + NS("exec.kernel"), ] = None prefill_attention_backend: A[ Optional[str], @@ -1410,6 +1579,7 @@ class ServerArgs: choices=ATTENTION_BACKEND_CHOICES, resolvable=True, ), + NS("exec.kernel"), ] = None sampling_backend: A[ Optional[str], @@ -1418,6 +1588,7 @@ class ServerArgs: choices=SAMPLING_BACKEND_CHOICES, resolvable=True, ), + NS("exec.kernel"), ] = None grammar_backend: A[ Optional[str], @@ -1425,10 +1596,12 @@ class ServerArgs: help="Choose the backend for grammar-guided decoding.", choices=GRAMMAR_BACKEND_CHOICES, ), + NS("exec.kernel"), ] = None radix_cache_backend: A[ Optional[str], "Name of a radix-cache backend previously registered via register_radix_cache_backend. Omit this flag to use the built-in default cache selection chain.", + NS("memory"), ] = None mm_attention_backend: A[ Optional[str], @@ -1446,6 +1619,7 @@ class ServerArgs: "xpu_attn", ], ), + NS("mm"), ] = None fp8_gemm_runner_backend: A[ str, @@ -1455,6 +1629,7 @@ class ServerArgs: choices=FP8_GEMM_RUNNER_BACKEND_CHOICES, resolvable=True, ), + NS("exec.kernel"), ] = "auto" fp4_gemm_runner_backend: A[ str, @@ -1463,6 +1638,7 @@ class ServerArgs: cli_name="--fp4-gemm-backend", choices=FP4_GEMM_RUNNER_BACKEND_CHOICES, ), + NS("exec.kernel"), ] = "auto" bf16_gemm_backend: A[ str, @@ -1471,6 +1647,7 @@ class ServerArgs: cli_name="--bf16-gemm-backend", choices=BF16_GEMM_BACKEND_CHOICES, ), + NS("exec.kernel"), ] = "auto" dsa_prefill_backend: A[ Optional[str], @@ -1479,6 +1656,7 @@ class ServerArgs: choices=DSA_CHOICES, resolvable=True, ), + NS("exec.kernel"), ] = None dsa_decode_backend: A[ Optional[str], @@ -1487,6 +1665,7 @@ class ServerArgs: choices=DSA_CHOICES, resolvable=True, ), + NS("exec.kernel"), ] = None dsa_paged_mqa_logits_backend: A[ str, @@ -1494,6 +1673,7 @@ class ServerArgs: help="DSA indexer paged MQA logits kernel backend. Options: 'auto' (default; DeepGEMM on CUDA, aiter on ROCm), 'deepgemm', 'cutedsl' (CuTe DSL kernel, SM 100 (Blackwell) only; wins at low batch size and long context), 'aiter' (ROCm only).", choices=DSA_PAGED_MQA_LOGITS_BACKEND_CHOICES, ), + NS("exec.kernel"), ] = "auto" dsa_topk_backend: A[ str, @@ -1501,14 +1681,18 @@ class ServerArgs: help="DSA indexer top-k backend. Options: 'sgl-kernel', 'torch', 'flashinfer'. The 'torch' backend currently requires SGLANG_DSA_FUSE_TOPK=false.", choices=DSA_TOPK_BACKEND_CHOICES, ), + NS("exec.kernel"), ] = "sgl-kernel" - disable_flashinfer_autotune: A[bool, "Disable FlashInfer autotuning."] = False + disable_flashinfer_autotune: A[ + bool, "Disable FlashInfer autotuning.", NS("exec.kernel") + ] = False mamba_backend: A[ str, Arg( help="Choose the kernel backend for Mamba SSM operations. Default is 'triton'. Options: 'triton' (default), 'flashinfer' (requires FlashInfer with Mamba support).", choices=MAMBA_BACKEND_CHOICES, ), + NS("exec.mamba"), ] = "triton" # ------------------------------------------------------------------------- @@ -1520,6 +1704,7 @@ class ServerArgs: help='Per-phase CUDA graph settings as JSON, e.g. \'{"decode":{"backend":"full","max_bs":256},"prefill":{"backend":"tc_piecewise","tc_compiler":"eager"}}\'. Allowed backends per phase: full, breakable, tc_piecewise, disabled (full is decode-only). JSON wins over the per-phase --cuda-graph-* convenience flags and over legacy flags.', type_parser=parse_cuda_graph_config_arg, ), + NS("exec.graph"), ] = None cuda_graph_backend_decode: A[ Optional[Literal["full", "breakable", "tc_piecewise", "disabled"]], @@ -1527,6 +1712,7 @@ class ServerArgs: help="Backend for the decode phase. Folds into cuda_graph_config[decode].backend.", choices=Backend.ALL, ), + NS("exec.graph"), ] = None cuda_graph_backend_prefill: A[ Optional[Literal["full", "breakable", "tc_piecewise", "disabled"]], @@ -1534,50 +1720,61 @@ class ServerArgs: help="Backend for the prefill phase. Folds into cuda_graph_config[prefill].backend.", choices=Backend.ALL, ), + NS("exec.graph"), ] = None cuda_graph_max_bs_decode: A[ Optional[int], "Maximum batch size captured for the decode cuda graph.", + NS("exec.graph"), ] = None cuda_graph_max_bs_prefill: A[ Optional[int], "Maximum batch size captured for the prefill cuda graph.", + NS("exec.graph"), ] = None cuda_graph_bs_decode: A[ Optional[List[int]], "Explicit list of batch sizes to capture for the decode cuda graph.", + NS("exec.graph"), ] = None cuda_graph_bs_prefill: A[ Optional[List[int]], "Explicit list of batch sizes to capture for the prefill cuda graph.", + NS("exec.graph"), ] = None cuda_graph_tc_compiler: A[ Optional[Literal["eager", "inductor"]], "Compiler used by the tc_piecewise backend (currently only the prefill phase consumes it).", + NS("exec.graph"), ] = None disable_prefill_cuda_graph: A[ bool, "Disable the prefill-phase CUDA graph. Convenience for --cuda-graph-backend-prefill=disabled.", + NS("exec.graph"), ] = False disable_decode_cuda_graph: A[ bool, "Disable the decode-phase CUDA graph. Convenience for --cuda-graph-backend-decode=disabled.", + NS("exec.graph"), ] = False - disable_cuda_graph: A[bool, Arg(no_cli=True)] = False + disable_cuda_graph: A[bool, Arg(no_cli=True), NS("exec.graph")] = False disable_cuda_graph_padding: A[ bool, "Disable cuda graph when padding is needed. Still uses cuda graph when padding is not needed.", + NS("exec.graph"), + ] = False + enable_profile_cuda_graph: A[ + bool, "Enable profiling of cuda graph capture.", NS("exec.graph") ] = False - enable_profile_cuda_graph: A[bool, "Enable profiling of cuda graph capture."] = ( - False - ) enable_cudagraph_gc: A[ bool, "Enable garbage collection during CUDA graph capture. If disabled (default), GC is frozen during capture to speed up the process.", + NS("exec.graph"), ] = False debug_cuda_graph: A[ bool, "Enable debug/eager mode for CUDA graph using breakable CUDA graph. When enabled, graph breaks are inserted so every operation runs eagerly while still going through the CUDA graph capture / replay path. Useful for debugging CUDA graph capture / replay issues.", + NS("exec.graph"), ] = False # ------------------------------------------------------------------------- @@ -1586,47 +1783,54 @@ class ServerArgs: enable_layerwise_nvtx_marker: A[ bool, "Enable layerwise NVTX profiling annotations for the model.", + NS("exec.comm"), ] = False enable_nccl_nvls: A[ bool, "Enable NCCL NVLS for prefill heavy requests when available.", + NS("exec.comm"), ] = False enable_symm_mem: A[ - bool, - "Enable NCCL symmetric memory for fast collectives.", + bool, "Enable NCCL symmetric memory for fast collectives.", NS("exec.comm") ] = False triton_attention_reduce_in_fp32: A[ bool, "Cast the intermediate attention results to fp32 to avoid possible crashes related to fp16." "This only affects Triton attention kernels.", + NS("exec.kernel"), ] = False triton_attention_num_kv_splits: A[ int, "The number of KV splits in flash decoding Triton kernel. Larger value is better in longer context scenarios. The default value is 8.", + NS("exec.kernel"), ] = 8 triton_attention_split_tile_size: A[ Optional[int], "The size of split KV tile in flash decoding Triton kernel. Used for deterministic inference.", + NS("exec.kernel"), ] = None flashinfer_mla_disable_ragged: A[ bool, "Not using ragged prefill wrapper when running flashinfer mla", + NS("exec.kernel"), ] = False enable_fused_qk_norm_rope: A[ bool, "Enable fused qk normalization and rope rotary embedding.", + NS("exec.kernel"), ] = False enable_precise_embedding_interpolation: A[ bool, "Enable corner alignment for resize of embeddings grid to ensure more accurate(but slower) evaluation of interpolated embedding values.", + NS("exec.kernel"), ] = False enable_fused_moe_sum_all_reduce: A[ - bool, - "Enable fused moe triton and sum all reduce.", + bool, "Enable fused moe triton and sum all reduce.", NS("exec.moe") ] = False enable_deepseek_v4_fp4_indexer: A[ bool, "Enable the experimental FP4 C4 indexer path for DeepSeek V4. Default keeps the existing indexer implementation.", + NS("exec.kernel"), ] = False disable_custom_all_reduce: A[ bool, @@ -1634,31 +1838,38 @@ class ServerArgs: help="Disable the custom all-reduce kernel and fall back to NCCL.", resolvable=True, ), + NS("exec.comm"), ] = False enable_mscclpp: A[ bool, "Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL.", + NS("exec.comm"), ] = False enable_torch_symm_mem: A[ bool, "Enable using torch symm mem for all-reduce kernel and fall back to NCCL. Only supports CUDA device SM90 and above. SM90 supports world size 4, 6, 8. SM100 supports world size 6, 8.", + NS("exec.comm"), ] = False enable_scattered_sconv: A[ bool, "Inkling: replace the attention/MLP output all-reduce with a hidden-dimension reduce-scatter, run the channelwise output short convolution on the [T, H/P] shard, then all-gather before the residual add. This shards the convolution cache across tensor-parallel ranks without changing communication volume.", + NS("exec.comm"), ] = False pre_warm_nccl: A[ bool, "Pre-warm NCCL/RCCL communicators during startup to reduce P99 TTFT cold-start latency. Default: enabled for AMD/HIP (RCCL), disabled for NVIDIA/CUDA (NCCL).", + NS("exec.comm"), ] = False enable_quant_communications: A[ Optional[bool], "Enable INT8 quantization of TP communications (limited support).", + NS("exec.comm"), ] = False - enable_flashinfer_allreduce_fusion: A[bool, Arg(no_cli=True)] = False + enable_flashinfer_allreduce_fusion: A[bool, Arg(no_cli=True), NS("exec.comm")] = ( + False + ) enforce_disable_flashinfer_allreduce_fusion: A[ - bool, - "Enforce disable FlashInfer allreduce fusion.", + bool, "Enforce disable FlashInfer allreduce fusion.", NS("exec.comm") ] = False flashinfer_allreduce_fusion_backend: A[ Optional[Literal["auto", "trtllm", "mnnvl"]], @@ -1676,10 +1887,12 @@ class ServerArgs: ), resolvable=True, ), + NS("exec.comm"), ] = None enable_aiter_allreduce_fusion: A[ bool, Arg(help="Enable Aiter AllReduce Fusion.", resolvable=True), + NS("exec.comm"), ] = False # ------------------------------------------------------------------------- @@ -1688,17 +1901,18 @@ class ServerArgs: enable_torch_compile: A[ bool, "Optimize the model with torch.compile. Experimental feature.", + NS("exec.graph"), + ] = False + enable_torch_compile_debug_mode: A[ + bool, "Enable debug mode for torch compile", NS("exec.graph") ] = False - enable_torch_compile_debug_mode: A[bool, "Enable debug mode for torch compile"] = ( - False - ) torch_compile_max_bs: A[ - int, - "Set the maximum batch size when using torch compile.", + int, "Set the maximum batch size when using torch compile.", NS("exec.graph") ] = 32 torchao_config: A[ str, "Optimize the model with torchao. Experimental feature. Current choices are: int8dq, int8wo, int4wo-, fp8wo, fp8dq-per_tensor, fp8dq-per_row", + NS("exec.graph"), ] = "" # ------------------------------------------------------------------------- @@ -1707,6 +1921,7 @@ class ServerArgs: speculative_algorithm: A[ Optional[str], "Speculative algorithm. Builtins: EAGLE, EAGLE3, NEXTN, STANDALONE, NGRAM, DFLASH, DSPARK. Or any name registered via `SpeculativeAlgorithm.register`.", + NS("spec"), ] = None speculative_draft_model_path: A[ Optional[str], @@ -1714,10 +1929,12 @@ class ServerArgs: help="The path of the draft model weights. This can be a local folder or a Hugging Face repo ID.", aliases=["--speculative-draft-model"], ), + NS("spec"), ] = None speculative_draft_model_revision: A[ Optional[str], "The specific draft model version to use. It can be a branch name, a tag name, or a commit id. If unspecified, will use the default version.", + NS("spec"), ] = None speculative_draft_load_format: A[ Optional[str], @@ -1725,26 +1942,32 @@ class ServerArgs: help="The format of the draft model weights to load. If not specified, will use the same format as --load-format. Use 'dummy' to initialize draft model weights with random values for profiling.", choices=LOAD_FORMAT_CHOICES, ), + NS("spec"), ] = None speculative_num_steps: A[ Optional[int], "The number of steps sampled from draft model in Speculative Decoding.", + NS("spec"), ] = None speculative_eagle_topk: A[ Optional[int], "The number of tokens sampled from the draft model in eagle2 each step.", + NS("spec"), ] = None speculative_num_draft_tokens: A[ Optional[int], "The number of tokens sampled from the draft model in Speculative Decoding.", + NS("spec"), ] = None speculative_dflash_block_size: A[ Optional[int], "DFLASH only. Block size (verify window length). Alias of --speculative-num-draft-tokens for DFLASH.", + NS("spec"), ] = None speculative_dspark_block_size: A[ Optional[int], "DSPARK only. Draft block size gamma (number of proposed draft tokens). The verify window is gamma + 1, so this sets --speculative-num-draft-tokens = gamma + 1. Omit to auto-infer gamma from the draft checkpoint block_size.", + NS("spec"), ] = None speculative_dspark_sps_table_path: A[ Optional[str], @@ -1753,6 +1976,7 @@ class ServerArgs: "scheduler (cap-accept / compact). Omit for an uninitialized flat " "constant-SPS table: the budget degenerates to verify-all (zero throughput " "gain by itself).", + NS("spec"), ] = None speculative_dspark_confidence_sts_path: A[ Optional[str], @@ -1761,6 +1985,7 @@ class ServerArgs: "Calibrates the confidence-head survival probabilities the ragged-verify " "scheduler consumes. Omit to use identity (no calibration); losslessness is " "unaffected either way.", + NS("spec"), ] = None speculative_dspark_align_verify_tokens_to_graph_tier: A[ bool, @@ -1772,22 +1997,25 @@ class ServerArgs: "-- both the cuda-graph bucket round-up and the dp cross-rank max -- turning " "it into extra real verification at the same step time. Off by default; when " "off the schedule is byte-for-byte unchanged.", + NS("spec"), ] = False speculative_accept_threshold_single: A[ float, "Accept a draft token if its probability in the target model is greater than this threshold.", + NS("spec"), ] = 1.0 speculative_accept_threshold_acc: A[ float, "The accept probability of a draft token is raised from its target probability p to min(1, p / threshold_acc).", + NS("spec"), ] = 1.0 speculative_use_rejection_sampling: A[ bool, "Use rejection sampling for speculative decoding (requires topk=1).", + NS("spec"), ] = False speculative_token_map: A[ - Optional[str], - "The path of the draft model's small vocab table.", + Optional[str], "The path of the draft model's small vocab table.", NS("spec") ] = None speculative_attention_mode: A[ str, @@ -1795,14 +2023,17 @@ class ServerArgs: help="Attention backend for speculative decoding operations (both target verify and draft extend). Can be one of 'prefill' (default) or 'decode'.", choices=["prefill", "decode"], ), + NS("spec"), ] = "prefill" speculative_draft_attention_backend: A[ Optional[str], "Attention backend for speculative decoding drafting.", + NS("spec"), ] = None speculative_draft_window_size: A[ Optional[int], "Sliding window size for the draft model. Honored by Llama EAGLE-3 (`LlamaForCausalLMEagle3`) and DFLASH only; other EAGLE-3 backends (e.g. MLA-based drafters) silently ignore it. For Llama EAGLE-3, the drafter only attends to the most recent N keys (verifier hidden states + its own outputs); the verifier is unaffected. For DFLASH, the draft worker keeps a recent target-token window in its local KV cache (paged backends may retain up to one extra page on the left for alignment). Default is full attention/context.", + NS("spec"), ] = None speculative_moe_runner_backend: A[ Optional[str], @@ -1811,6 +2042,7 @@ class ServerArgs: choices=MOE_RUNNER_BACKEND_CHOICES, resolvable=True, ), + NS("spec"), ] = None speculative_moe_a2a_backend: A[ Optional[str], @@ -1819,6 +2051,7 @@ class ServerArgs: choices=MOE_A2A_BACKEND_CHOICES, resolvable=True, ), + NS("spec"), ] = None speculative_draft_model_quantization: A[ Optional[str], @@ -1826,10 +2059,12 @@ class ServerArgs: help="The quantization method for speculative model.", choices=SPECULATIVE_DRAFT_MODEL_QUANTIZATION_CHOICES, ), + NS("spec"), ] = None speculative_skip_dp_mlp_sync: A[ bool, "Skip the extra MLP sync that the scheduler performs before merging a new batch when speculative decoding + DP attention are both enabled.", + NS("spec"), ] = False enable_multi_layer_eagle: A[ bool, @@ -1837,14 +2072,17 @@ class ServerArgs: help="Enable multi-layer Eagle speculative decoding.", resolvable=True, ), + NS("spec"), ] = False speculative_adaptive: A[ bool, "Enable adaptive speculative decoding that dynamically adjusts num_steps based on acceptance rate.", + NS("spec"), ] = False speculative_adaptive_config: A[ Optional[str], "Path to a JSON config file for adaptive speculative decoding tuning knobs.", + NS("spec"), ] = None # Decoupled speculative decoding: draft and verify run as @@ -1853,6 +2091,7 @@ class ServerArgs: Optional[str], "ZMQ endpoint this engine binds for its inbound channel in decoupled " "speculative decoding (verifier: result PULL; drafter: control PULL).", + NS("disagg"), ] = None decoupled_spec_connect_endpoints: A[ Optional[List[str]], @@ -1861,20 +2100,24 @@ class ServerArgs: "rank, for decoupled speculative decoding.", type_parser=json_list_type, ), + NS("disagg"), ] = None decoupled_spec_rank: A[ Optional[int], "This engine's rank within its own role space (verifier-rank or " "drafter-rank) for decoupled speculative decoding.", + NS("disagg"), ] = None decoupled_spec_role: A[ Literal["null", "verifier", "drafter"], "Role in decoupled speculative decoding: 'null' disables it, 'verifier' " "runs the target/verify half, 'drafter' runs the draft half.", + NS("disagg"), ] = "null" spec_trace_dir: A[ Optional[str], "Directory to write decoupled speculative decoding trace files.", + NS("spec"), ] = None # Speculative decoding (ngram) @@ -1882,36 +2125,36 @@ class ServerArgs: speculative_ngram_min_bfs_breadth: A[ int, "The minimum breadth for BFS (Breadth-First Search) in ngram speculative decoding.", + NS("spec"), ] = 1 speculative_ngram_max_bfs_breadth: A[ int, "The maximum breadth for BFS (Breadth-First Search) in ngram speculative decoding.", + NS("spec"), ] = 10 speculative_ngram_match_type: A[ - Literal["BFS", "PROB"], - "The match type for cache tree.", + Literal["BFS", "PROB"], "The match type for cache tree.", NS("spec") ] = "BFS" speculative_ngram_max_trie_depth: A[ - int, - "The max trie depth for ngram speculative decoding.", + int, "The max trie depth for ngram speculative decoding.", NS("spec") ] = 18 speculative_ngram_capacity: A[ - int, - "The cache capacity for ngram speculative decoding.", - ] = ( - 10 * 1000 * 1000 - ) + int, "The cache capacity for ngram speculative decoding.", NS("spec") + ] = (10 * 1000 * 1000) speculative_ngram_external_corpus_path: A[ Optional[str], "Path to an external JSONL corpus to pre-load into SAM at startup. Additional corpora can be added at runtime via POST /add_external_corpus.", + NS("spec"), ] = None speculative_ngram_external_sam_budget: A[ int, "Number of draft nodes reserved for the external SAM subtree in ngram speculative decoding.", + NS("spec"), ] = 0 speculative_ngram_external_corpus_max_tokens: A[ int, "Fail startup if the tokenized external ngram corpus exceeds this many tokens. Tune this based on your CPU memory budget.", + NS("spec"), ] = 10000000 # ------------------------------------------------------------------------- @@ -1924,6 +2167,7 @@ class ServerArgs: aliases=["--expert-parallel-size", "--ep"], resolvable=True, ), + NS("parallel"), ] = 1 moe_a2a_backend: A[ Literal[ @@ -1942,6 +2186,7 @@ class ServerArgs: choices=MOE_A2A_BACKEND_CHOICES, resolvable=True, ), + NS("exec.moe"), ] = "none" moe_runner_backend: A[ str, @@ -1950,61 +2195,73 @@ class ServerArgs: choices=MOE_RUNNER_BACKEND_CHOICES, resolvable=True, ), + NS("exec.moe"), ] = "auto" flashinfer_mxfp4_moe_precision: A[ Literal["default", "bf16"], "Choose the computation precision of flashinfer mxfp4 moe", + NS("exec.moe"), ] = "default" deepep_mode: A[ Literal["auto", "normal", "low_latency"], "Select the mode when enable DeepEP or MoriEP MoE, could be `normal`, `low_latency` or `auto`. Default is `auto`, which means `low_latency` for decode batch and `normal` for prefill batch.", + NS("exec.moe"), ] = "auto" fuseep_mode: A[ Literal[1, 2], "Select the mode when enable Ascend FuseEP MoE, 1 -> dispatch_gmm_combine_decode is executed；2 -> dispatch_ffn_combine is executed (support hybrid deployment when 2).", + NS("exec.moe"), ] = 2 deepep_dispatcher_output_dtype: A[ Literal["auto", "bf16", "fp8", "int8", "nvfp4"], "Select DeepEP dispatcher output dtype", + NS("exec.moe"), ] = "auto" ep_num_redundant_experts: A[ int, "Allocate this number of redundant experts in expert parallel.", + NS("exec.moe"), ] = 0 ep_dispatch_algorithm: A[ Optional[Literal["static", "dynamic", "fake", "lp"]], "The algorithm to choose ranks for redundant experts in expert parallel.", + NS("exec.moe"), ] = None - init_expert_location: A[str, "Initial location of EP experts."] = "trivial" - enable_eplb: A[bool, "Enable EPLB algorithm"] = False - eplb_algorithm: A[str, "Chosen EPLB algorithm"] = "auto" + init_expert_location: A[str, "Initial location of EP experts.", NS("exec.moe")] = ( + "trivial" + ) + enable_eplb: A[bool, "Enable EPLB algorithm", NS("exec.moe")] = False + eplb_algorithm: A[str, "Chosen EPLB algorithm", NS("exec.moe")] = "auto" eplb_rebalance_num_iterations: A[ int, "Number of iterations to automatically trigger a EPLB re-balance.", + NS("exec.moe"), ] = 1000 eplb_rebalance_layers_per_chunk: A[ - Optional[int], - "Number of layers to rebalance per forward pass.", + Optional[int], "Number of layers to rebalance per forward pass.", NS("exec.moe") ] = None eplb_min_rebalancing_utilization_threshold: A[ float, "Minimum threshold for GPU average utilization to trigger EPLB rebalancing. Must be in the range [0.0, 1.0].", + NS("exec.moe"), ] = 1.0 expert_distribution_recorder_mode: A[ Optional[Literal["stat", "stat_approx", "per_pass", "per_token"]], "Mode of expert distribution recorder.", + NS("exec.moe"), ] = None expert_distribution_recorder_buffer_size: A[ Optional[int], "Circular buffer size of expert distribution recorder. Set to -1 to denote infinite buffer.", + NS("exec.moe"), ] = None enable_expert_distribution_metrics: A[ - bool, - "Enable logging metrics for expert balancedness", + bool, "Enable logging metrics for expert balancedness", NS("exec.moe") ] = False deepep_config: A[ Optional[str], "Tuned DeepEP config suitable for your own cluster. It can be either a string with JSON content or a file path.", + NS("exec.moe"), ] = None moe_dense_tp_size: A[ Optional[int], @@ -2012,6 +2269,7 @@ class ServerArgs: help="TP size for MoE dense MLP layers. This flag is useful when, with large TP size, there are errors caused by weights in MLP layers having dimension smaller than the min dimension GEMM supports.", resolvable=True, ), + NS("parallel"), ] = None elastic_ep_backend: A[ Literal[None, "mooncake", "nixl"], @@ -2019,17 +2277,20 @@ class ServerArgs: help="Specify the collective communication backend for elastic EP. Supports 'mooncake' and 'nixl'.", choices=["none", "mooncake", "nixl"], ), + NS("exec.moe"), ] = None - enable_elastic_expert_backup: A[bool, "Enable elastic expert backup feature."] = ( - False - ) + enable_elastic_expert_backup: A[ + bool, "Enable elastic expert backup feature.", NS("exec.moe") + ] = False mooncake_ib_device: A[ Optional[str], "The InfiniBand devices for Mooncake Backend transfer, accepts multiple comma-separated devices (e.g., --mooncake-ib-device mlx5_0,mlx5_1). Default is None, which triggers automatic device detection when Mooncake Backend is enabled.", + NS("exec.moe"), ] = None enable_waterfill: A[ bool, "Enable Waterfill: dispatch the fused shared expert as an extra routed expert slot to the least-loaded EP rank. Supports DeepEP and MegaMOE MoE A2A backends, implicitly enables shared-expert fusion, and supports --deepep-mode auto, normal, or low_latency when used with DeepEP. Use auto or low_latency for production DeepEP decode so CUDA graph remains enabled. Supported on DeepSeek-V3/R1 with EP >= 2.", + NS("exec.moe"), ] = False ep_join_mode: A[ Optional[Literal["scale", "recover"]], @@ -2038,6 +2299,7 @@ class ServerArgs: cli_name="--elastic-ep-join-mode", choices=["scale", "recover"], ), + NS("exec.moe"), ] = None ep_join_rank_offset: A[ int, @@ -2048,27 +2310,31 @@ class ServerArgs: ), cli_name="--elastic-ep-join-rank-offset", ), + NS("parallel"), ] = 0 elastic_ep_initial_size: A[ Optional[int], "EP size used to define the immutable per-rank expert storage layout. " "Scale joiners must use the primary deployment's launch-time EP size.", + NS("parallel"), ] = None max_ep_size: A[ Optional[int], "Maximum EP size the server can scale to at runtime. Pre-allocates active-rank state and backend buffers to this size. Defaults to the launch-time world size.", + NS("parallel"), ] = None elastic_ep_scale_timeout: A[ float, "Timeout in seconds for a pending elastic EP scale operation.", + NS("exec.moe"), ] = 600 elastic_ep_rejoin: A[ - bool, - "[Deprecated] Alias for --elastic-ep-join-mode recover.", + bool, "[Deprecated] Alias for --elastic-ep-join-mode recover.", NS("exec.moe") ] = False disable_flashinfer_cutlass_moe_fp4_allgather: A[ bool, "Disables quantize before all-gather for flashinfer cutlass moe.", + NS("exec.moe"), ] = False disable_shared_experts_fusion: A[ bool, @@ -2076,32 +2342,37 @@ class ServerArgs: help="Disable the built-in shared experts fusion optimization for DeepSeek V3/R1. Note: Waterfill (--enable-waterfill) routes the shared expert as an extra MoE slot, so the shared expert is not separated from the MoE path when Waterfill is enabled.", resolvable=True, ), + NS("exec.moe"), ] = False enforce_shared_experts_fusion: A[ bool, "Enforce shared experts fusion even when it would normally be disabled (e.g. under DeepEP). Mutually exclusive with --disable-shared-experts-fusion.", + NS("exec.moe"), ] = False # ------------------------------------------------------------------------- # Mamba cache and linear attn # ------------------------------------------------------------------------- - max_mamba_cache_size: A[Optional[int], "The maximum size of the mamba cache."] = ( - None - ) + max_mamba_cache_size: A[ + Optional[int], "The maximum size of the mamba cache.", NS("schedule") + ] = None mamba_ssm_dtype: A[ Optional[str], Arg( help="The data type of the SSM states in mamba cache. If not set, will be read from model config (mamba_ssm_dtype).", choices=["float32", "bfloat16", "float16"], ), + NS("exec.mamba"), ] = None enable_mamba_cache_stochastic_rounding: A[ bool, "Enable stochastic rounding when writing FP16 Mamba SSM cache states. Requires --mamba-ssm-dtype float16 and CUDA. With --mamba-backend triton, requires SM100.", + NS("exec.mamba"), ] = False mamba_cache_philox_rounds: A[ int, "Number of Philox rounds to use for stochastic rounding of FP16 Mamba SSM cache writes. Triton uses the Triton default when set to 0; FlashInfer uses 10 rounds when set to 0.", + NS("exec.mamba"), ] = 0 mamba_full_memory_ratio: A[ float, @@ -2109,6 +2380,7 @@ class ServerArgs: help="The ratio of mamba state memory to full kv cache memory.", resolvable=True, ), + NS("schedule"), ] = 0.9 mamba_radix_cache_strategy: A[ str, @@ -2117,6 +2389,7 @@ class ServerArgs: choices=MAMBA_RADIX_CACHE_STRATEGY_CHOICES, resolvable=True, ), + NS("exec.mamba"), ] = "auto" uses_mamba_radix_cache: A[ bool, @@ -2127,18 +2400,20 @@ class ServerArgs: no_cli=True, resolvable=True, ), + NS("exec.mamba"), ] = False mamba_track_interval: A[ - int, - "The interval to track the mamba state during decode.", + int, "The interval to track the mamba state during decode.", NS("exec.mamba") ] = 256 enable_int8_mamba_checkpoint: A[ bool, "Store radix-cached linear-attn (mamba) states in int8 (separate checkpoint pool) for ~2x cached-prefix capacity at fixed memory.", + NS("exec.mamba"), ] = False int8_mamba_ckpt_size: A[ Optional[int], "Number of int8 mamba checkpoint slots (default: 2x the active mamba pool size).", + NS("exec.mamba"), ] = None linear_attn_backend: A[ str, @@ -2146,6 +2421,7 @@ class ServerArgs: help="The default kernel backend for linear attention (GDN/KDA). Can be overridden per-mode by --linear-attn-decode-backend and --linear-attn-prefill-backend.", choices=LINEAR_ATTN_KERNEL_BACKEND_CHOICES, ), + NS("exec.mamba"), ] = "triton" linear_attn_decode_backend: A[ Optional[str], @@ -2153,6 +2429,7 @@ class ServerArgs: help="Override the kernel backend for linear attention decode. If not set, uses --linear-attn-backend.", choices=LINEAR_ATTN_KERNEL_BACKEND_CHOICES, ), + NS("exec.mamba"), ] = None linear_attn_prefill_backend: A[ Optional[str], @@ -2160,6 +2437,7 @@ class ServerArgs: help="Override the kernel backend for linear attention prefill/extend. If not set, uses --linear-attn-backend; compatible SM100 GDN models may automatically select FlashInfer.", choices=LINEAR_ATTN_KERNEL_BACKEND_CHOICES, ), + NS("exec.mamba"), ] = None # ReplaySSM buffered output-only linear-attn decode (GDN + KDA): per-slot # ring + periodic flush to cut per-step HBM state traffic. @@ -2173,10 +2451,12 @@ class ServerArgs: "refolds the per-K decay every step), so it is not recommended for KDA " "models. Requires the Triton linear-attn decode backend and " "--mamba-radix-cache-strategy no_buffer (the default).", + NS("exec.mamba"), ] = False linear_replayssm_cache_len: A[ int, "Ring-buffer length L for ReplaySSM linear-attn decode. The full recurrent state is flushed to HBM every L decode steps.", + NS("exec.mamba"), ] = 16 # ReplaySSM spec-verify (Part B of RFC #28511): GDN linear-chain target-verify # via a per-slot circular (d, k, g) ring + periodic flush instead of per-draft @@ -2187,19 +2467,24 @@ class ServerArgs: enable_gdn_replayssm_spec: A[ bool, "Enable the ReplaySSM GDN spec-verify kernel (Part B of RFC #28511): a per-slot circular (d, k, g) ring + periodic flush replacing the recurrent verify's per-draft full-state snapshots. GDN only, linear-chain (--speculative-eagle-topk in {None, 1}) only. Reuses --linear-replayssm-cache-len for the ring length.", + NS("exec.mamba"), ] = False # ------------------------------------------------------------------------- # Hierarchical cache # ------------------------------------------------------------------------- - enable_hierarchical_cache: A[bool, "Enable hierarchical cache"] = False + enable_hierarchical_cache: A[bool, "Enable hierarchical cache", NS("memory")] = ( + False + ) hicache_ratio: A[ float, "The ratio of the size of host KV cache memory pool to the size of device pool.", + NS("memory"), ] = 2.0 hicache_size: A[ int, "The size of host KV cache memory pool in gigabytes, which will override the hicache_ratio if set.", + NS("memory"), ] = 0 hicache_write_policy: A[ str, @@ -2207,6 +2492,7 @@ class ServerArgs: help="The write policy of hierarchical cache.", choices=["write_back", "write_through", "write_through_selective"], ), + NS("memory"), ] = "write_through" hicache_io_backend: A[ str, @@ -2214,6 +2500,7 @@ class ServerArgs: help="The IO backend for KV cache transfer between CPU and GPU", choices=["direct", "kernel", "kernel_ascend"], ), + NS("memory"), ] = "kernel" hicache_mem_layout: A[ str, @@ -2227,6 +2514,7 @@ class ServerArgs: "page_head", ], ), + NS("memory"), ] = "page_first" hicache_storage_backend: A[ Optional[str], @@ -2244,6 +2532,7 @@ class ServerArgs: "mori", ], ), + NS("memory"), ] = None hicache_storage_prefetch_policy: A[ str, @@ -2251,38 +2540,44 @@ class ServerArgs: help="Control when prefetching from the storage backend should stop.", choices=["best_effort", "wait_complete", "timeout"], ), + NS("memory"), ] = "timeout" hicache_storage_backend_extra_config: A[ Optional[str], "A dictionary in JSON string format, or a string starting with a leading '@' and a config file in JSON/YAML/TOML format, containing extra configuration for the storage backend.", + NS("memory"), ] = None # ------------------------------------------------------------------------- # Hierarchical sparse attention # ------------------------------------------------------------------------- - enable_hisparse: A[bool, "Enable hierarchical sparse attention"] = False + enable_hisparse: A[bool, "Enable hierarchical sparse attention", NS("memory")] = ( + False + ) hisparse_config: A[ Optional[str], Arg( help='A dictionary in JSON string format for hierarchical sparse attention configuration. Example: \'{"top_k": 2048, "device_buffer_size": 4096, "host_to_device_ratio": 2}\'', aliases=["--hierarchical-sparse-attention-extra-config"], ), + NS("memory"), ] = None # ------------------------------------------------------------------------- # Multi-modal optimization configs # ------------------------------------------------------------------------- enable_broadcast_mm_inputs_process: A[ - bool, - "Enable broadcast mm-inputs process in scheduler.", + bool, "Enable broadcast mm-inputs process in scheduler.", NS("mm") ] = False enable_prefix_mm_cache: A[ bool, "Enable prefix multimodal cache. Currently only supports mm-only.", + NS("mm"), ] = False mm_enable_dp_encoder: A[ bool, "Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.", + NS("mm"), ] = False mm_process_config: A[ Optional[Dict[str, Any]], @@ -2290,18 +2585,21 @@ class ServerArgs: help="Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`", type_parser=json.loads, ), + NS("mm"), ] = None mm_processor_worker_num: A[ int, "Number of threads for multimodal processor calls. 0 selects the " "model-specific default. Only processors with isolated-worker support " "can use more than one thread.", + NS("mm"), ] = 0 mm_io_worker_num: A[ int, "Number of threads for multimodal data loading and decoding. 0 selects " "the model-specific default. SGLANG_IO_WORKERS remains supported as an " "environment override when this argument is 0.", + NS("mm"), ] = 0 limit_mm_data_per_request: A[ Optional[Union[str, Dict[str, int]]], @@ -2309,24 +2607,27 @@ class ServerArgs: help='Limit the number of multimodal inputs per request. e.g. \'{"image": 1, "video": 1, "audio": 1}\'', type_parser=json.loads, ), + NS("mm"), ] = None enable_mm_global_cache: A[ bool, "Enable global multimodal embedding cache to skip redundant ViT inference.", + NS("mm"), ] = False disable_fast_image_processor: A[ - bool, - "Adopt base image processor instead of fast image processor.", + bool, "Adopt base image processor instead of fast image processor.", NS("mm") ] = False mm_feature_transport: A[ Optional[Literal["cpu", "cuda_ipc"]], "Transport multimodal features through CPU memory or a bounded CUDA IPC pool. " "The default is CPU transport; CUDA IPC reserves GPU memory on the base GPU.", + NS("mm"), ] = None keep_mm_feature_on_device: A[ bool, "Deprecated. Use --mm-feature-transport=cuda_ipc for bounded GPU-resident " "multimodal feature transport.", + NS("mm"), ] = False # ------------------------------------------------------------------------- @@ -2335,14 +2636,17 @@ class ServerArgs: enable_lora: A[ Optional[bool], "Enable LoRA support for the model. This argument is automatically set to True if `--lora-paths` is provided for backward compatibility.", + NS("lora"), ] = None enable_lora_overlap_loading: A[ Optional[bool], "Enable asynchronous LoRA weight loading in order to overlap H2D transfers with GPU compute. This should be enabled if you find that your LoRA workloads are bottlenecked by adapter weight loading, for example when frequently loading large LoRA adapters.", + NS("lora"), ] = None max_lora_rank: A[ Optional[int], "The maximum rank of LoRA adapters. If not specified, it will be automatically inferred from the adapters provided in --lora-paths.", + NS("lora"), ] = None lora_target_modules: A[ Optional[Union[set[str], List[str]]], @@ -2351,6 +2655,7 @@ class ServerArgs: nargs="*", choices=SUPPORTED_LORA_TARGET_MODULES + [LORA_TARGET_ALL_MODULES], ), + NS("lora"), ] = None lora_paths: A[ Optional[Union[dict[str, str], List[dict[str, str]], List[str], List[LoRARef]]], @@ -2359,14 +2664,17 @@ class ServerArgs: action=LoRAPathAction, action_kwargs={"type": str, "nargs": "*"}, ), + NS("lora"), ] = None max_loaded_loras: A[ Optional[int], "If specified, it limits the maximum number of LoRA adapters loaded in CPU memory at a time. The value must be greater than or equal to `--max-loras-per-batch`.", + NS("lora"), ] = None max_loras_per_batch: A[ int, "Maximum number of adapters for a running batch, include base-only request.", + NS("lora"), ] = 8 lora_eviction_policy: A[ str, @@ -2374,6 +2682,7 @@ class ServerArgs: help="LoRA adapter eviction policy when memory pool is full. 'lru': Least Recently Used (default, better cache efficiency). 'fifo': First-In-First-Out.", choices=["lru", "fifo"], ), + NS("lora"), ] = "lru" lora_backend: A[ str, @@ -2381,6 +2690,7 @@ class ServerArgs: help="Choose the kernel backend for multi-LoRA serving.", choices=LORA_BACKEND_CHOICES, ), + NS("lora"), ] = "csgmv" max_lora_chunk_size: A[ Optional[int], @@ -2388,6 +2698,7 @@ class ServerArgs: help="Maximum chunk size for the ChunkedSGMV LoRA backend. Only used when --lora-backend is 'csgmv'. Choosing a larger value might improve performance.", choices=[16, 32, 64, 128], ), + NS("lora"), ] = 16 experts_shared_outer_loras: A[ Optional[bool], @@ -2395,10 +2706,12 @@ class ServerArgs: help="Force shared outer LoRA mode for MoE models. When set, w1/w3 lora_A and w2 lora_B are shared across experts (expert_dim=1). Use --no-experts-shared-outer-loras to force disable. By default this is auto-detected from adapter weights.", action=argparse.BooleanOptionalAction, ), + NS("lora"), ] = None lora_use_virtual_experts: A[ bool, "Enable virtual expert computation for MoE models. When set, the model will use virtual expert computation.", + NS("lora"), ] = False lora_strict_loading: A[ bool, @@ -2406,36 +2719,47 @@ class ServerArgs: help="Enable strict loading for LoRA adapters. When set, mismatched or missing keys in the adapter weights will raise an error.", action=argparse.BooleanOptionalAction, ), + NS("lora"), ] = False lora_drain_wait_threshold: A[ float, "When any LoRA adapter request waits longer than this threshold (in seconds), the scheduler will selectively drain one running adapter to make room. This mitigates extreme tail latency under high or skewed workloads by preventing a small set of adapters from monopolizing batch slots. Set to 0 to disable draining (default).", + NS("lora"), ] = 0.0 # ------------------------------------------------------------------------- # Two batch overlap # ------------------------------------------------------------------------- - enable_two_batch_overlap: A[bool, "Enabling two micro batches to overlap."] = False + enable_two_batch_overlap: A[ + bool, "Enabling two micro batches to overlap.", NS("exec.overlap") + ] = False enable_single_batch_overlap: A[ bool, "Let computation and communication overlap within one micro batch.", + NS("exec.overlap"), ] = False tbo_token_distribution_threshold: A[ float, "The threshold of token distribution between two batches in micro-batch-overlap, determines whether to two-batch-overlap or two-chunk-overlap. Set to 0 denote disable two-chunk-overlap.", + NS("exec.overlap"), ] = 0.48 # ------------------------------------------------------------------------- # Offloading # ------------------------------------------------------------------------- - cpu_offload_gb: A[int, "How many GBs of RAM to reserve for CPU offloading."] = 0 - offload_group_size: A[int, "Number of layers per group in offloading."] = -1 + cpu_offload_gb: A[ + int, "How many GBs of RAM to reserve for CPU offloading.", NS("exec.offload") + ] = 0 + offload_group_size: A[ + int, "Number of layers per group in offloading.", NS("exec.offload") + ] = -1 offload_num_in_group: A[ - int, - "Number of layers to be offloaded within a group.", + int, "Number of layers to be offloaded within a group.", NS("exec.offload") ] = 1 - offload_prefetch_step: A[int, "Steps to prefetch in offloading."] = 1 - offload_mode: A[str, "Mode of offloading."] = "cpu" + offload_prefetch_step: A[ + int, "Steps to prefetch in offloading.", NS("exec.offload") + ] = 1 + offload_mode: A[str, "Mode of offloading.", NS("exec.offload")] = "cpu" # ------------------------------------------------------------------------- # LMCache @@ -2443,10 +2767,10 @@ class ServerArgs: enable_lmcache: A[ bool, "Using LMCache as an alternative hierarchical cache solution", + NS("memory"), ] = False lmcache_config_file: A[ - Optional[str], - "Path to the LMCache YAML configuration file", + Optional[str], "Path to the LMCache YAML configuration file", NS("memory") ] = None # ------------------------------------------------------------------------- @@ -2460,6 +2784,7 @@ class ServerArgs: "to --radix-cache-backend=flexkv but also participates in the " "auto-selection chain alongside --enable-lmcache." ), + NS("memory"), ] = False flexkv_config_file: A[ Optional[str], @@ -2468,6 +2793,7 @@ class ServerArgs: "Equivalent to setting the FLEXKV_CONFIG_PATH environment " "variable." ), + NS("memory"), ] = None # ------------------------------------------------------------------------- @@ -2476,26 +2802,32 @@ class ServerArgs: kt_weight_path: A[ Optional[str], "[ktransformers parameter] The path of the quantized expert weights for amx kernel. A local folder.", + NS("exec.moe"), ] = None kt_method: A[ str, "[ktransformers parameter] Quantization formats for CPU execution.", + NS("exec.moe"), ] = "AMXINT4" kt_cpuinfer: A[ Optional[int], "[ktransformers parameter] The number of CPUInfer threads.", + NS("exec.moe"), ] = None kt_threadpool_count: A[ int, "[ktransformers parameter] One-to-one with the number of NUMA nodes (one thread pool per NUMA).", + NS("exec.moe"), ] = 2 kt_num_gpu_experts: A[ Optional[int], "[ktransformers parameter] The number of GPU experts.", + NS("exec.moe"), ] = None kt_max_deferred_experts_per_token: A[ Optional[int], "[ktransformers parameter] Maximum number of experts deferred to CPU per token. All MoE layers except the final one use this value; the final layer always uses 0.", + NS("exec.moe"), ] = None # ------------------------------------------------------------------------- @@ -2504,10 +2836,12 @@ class ServerArgs: dllm_algorithm: A[ Optional[str], "The diffusion LLM algorithm, such as LowConfidence.", + NS("exec.dllm"), ] = None dllm_algorithm_config: A[ Optional[str], "The diffusion LLM algorithm configurations. Must be a YAML file.", + NS("exec.dllm"), ] = None dllm_fdfo: A[ bool, @@ -2515,6 +2849,7 @@ class ServerArgs: help="Enable First-Done-First-Out (FDFO) scheduling for diffusion LLM inference. Enabled by default; use --no-dllm-fdfo to fall back to synchronous block scheduling.", action=argparse.BooleanOptionalAction, ), + NS("exec.dllm"), ] = True # ------------------------------------------------------------------------- @@ -2523,6 +2858,7 @@ class ServerArgs: disaggregation_mode: A[ Literal["null", "prefill", "decode"], 'Only used for PD disaggregation. "prefill" for prefill-only server, and "decode" for decode-only server. If not specified, it is not PD disaggregated', + NS("disagg"), ] = "null" disaggregation_transfer_backend: A[ str, @@ -2530,50 +2866,57 @@ class ServerArgs: help="The backend for disaggregation transfer. Default is mooncake.", choices=DISAGG_TRANSFER_BACKEND_CHOICES, ), + NS("disagg"), ] = "mooncake" disaggregation_bootstrap_port: A[ int, "Bootstrap server port on the prefill server. Default is 8998.", + NS("disagg"), ] = 8998 disaggregation_ib_device: A[ Optional[str], 'The InfiniBand devices for disaggregation transfer. Supports a single device (e.g., --disaggregation-ib-device mlx5_0), a shared comma-separated list (e.g., --disaggregation-ib-device mlx5_0,mlx5_1), a per-GPU JSON mapping (e.g., --disaggregation-ib-device \'{"0": "mlx5_0,mlx5_1", "1": "mlx5_2"}\'), or a path to a JSON file containing that mapping. Default is None, which triggers automatic device detection when mooncake backend is enabled.', + NS("disagg"), ] = None disaggregation_decode_enable_radix_cache: A[ bool, "Enable radix cache on decode server (PD mode). Caches KV prefixes to avoid redundant transfers. Incompatible with --enable-hisparse, speculative decoding, and --disaggregation-transfer-backend fake.", + NS("disagg"), ] = False disaggregation_decode_enable_offload_kvcache: A[ bool, "Enable async KV cache offloading on decode server (PD mode).", + NS("disagg"), ] = False num_reserved_decode_tokens: A[ int, "Number of decode tokens that will have memory reserved when adding new request to the running batch.", + NS("disagg"), ] = 512 disaggregation_decode_extra_slots: A[ Optional[int], "Number of extra decode req_to_token slots pre-allocated for in-transfer requests (PD mode). If unset, defaults to 0 (or 2x the per-worker running batch for small batches).", + NS("disagg"), ] = None disaggregation_decode_polling_interval: A[ int, "The interval to poll requests in decode server. Can be set to >1 to reduce the overhead of this.", + NS("disagg"), ] = 1 optimistic_prefill_attempts: A[ int, "Number of optimistic prefill forward passes that skip the bootstrap wait.", + NS("disagg"), ] = 0 # ------------------------------------------------------------------------- # Encode prefill disaggregation # ------------------------------------------------------------------------- encoder_only: A[ - bool, - "For MLLM with an encoder, launch an encoder-only server", + bool, "For MLLM with an encoder, launch an encoder-only server", NS("disagg") ] = False language_only: A[ - bool, - "For VLM, load weights for the language model only.", + bool, "For VLM, load weights for the language model only.", NS("disagg") ] = False encoder_transfer_backend: A[ str, @@ -2581,36 +2924,37 @@ class ServerArgs: help="The backend for encoder disaggregation transfer. Default is zmq_to_scheduler.", choices=ENCODER_TRANSFER_BACKEND_CHOICES, ), + NS("disagg"), ] = ENCODER_TRANSFER_BACKEND_CHOICES[0] - encoder_urls: A[ - List[str], - "List of encoder server urls.", - ] = dataclasses.field(default_factory=list) + encoder_urls: A[List[str], "List of encoder server urls.", NS("disagg")] = ( + dataclasses.field(default_factory=list) + ) encoder_bootstrap_port: A[ int, "Port for the EncoderBootstrapServer that runs in the language-only tokenizer manager process. Encoders register here, and language-only receivers fetch the current URL list from here.", + NS("disagg"), ] = 8997 encoder_register_urls: A[ List[str], "One or more EncoderBootstrapServer URLs to register this encoder with on startup, for dynamic encoder discovery. Example: --encoder-register-urls http://prefill0:8997 http://prefill1:8997. Used with --encoder-only servers.", + NS("disagg"), ] = dataclasses.field(default_factory=list) enable_adaptive_dispatch_to_encoder: A[ bool, "When enabled, adaptively dispatch: multi-image requests go to encoder in language_only epd mode, single-image requests are processed locally.", + NS("disagg"), ] = False # ------------------------------------------------------------------------- # PD-Multiplexing # ------------------------------------------------------------------------- enable_pdmux: A[ - bool, - "Enable PD-Multiplexing, PD running on greenctx stream.", + bool, "Enable PD-Multiplexing, PD running on greenctx stream.", NS("disagg") ] = False pdmux_config_path: A[ - Optional[str], - "The path of the PD-Multiplexing config file.", + Optional[str], "The path of the PD-Multiplexing config file.", NS("disagg") ] = None - sm_group_num: A[int, "Number of sm partition groups."] = 8 + sm_group_num: A[int, "Number of sm partition groups.", NS("disagg")] = 8 # ------------------------------------------------------------------------- # Model weight update and weight loading @@ -2621,30 +2965,35 @@ class ServerArgs: help="The custom dataloader which used to update the model. Should be set with a valid import path, such as my_package.weight_load_func", nargs="*", ), + NS("model"), ] = None weight_loader_disable_mmap: A[ - bool, - "Disable mmap while loading weight using safetensors.", + bool, "Disable mmap while loading weight using safetensors.", NS("model") ] = False weight_loader_prefetch_checkpoints: A[ bool, "Prefetch checkpoint files into OS page cache before loading. Each rank prefetches a fraction of the shards, reducing total network I/O on shared filesystems (NFS/Lustre) from N*checkpoint to 1*checkpoint. Recommended for models on network storage. When enabled, multi-threaded safetensors loading is disabled by default to avoid I/O oversubscription with the prefetch threads; set enable_multithread_load=true in --model-loader-extra-config to keep multi-threaded loading (e.g. on local NVMe where prefetch is a no-op).", + NS("model"), ] = False weight_loader_prefetch_num_threads: A[ int, "Number of threads per rank for checkpoint prefetching (default: 4).", + NS("model"), ] = 4 weight_loader_drop_cache_after_load: A[ bool, "Call posix_fadvise(DONTNEED) on each safetensors shard after loading it.", + NS("model"), ] = False remote_instance_weight_loader_seed_instance_ip: A[ Optional[str], "The ip of the seed instance for loading weights from remote instance.", + NS("model"), ] = None remote_instance_weight_loader_seed_instance_service_port: A[ Optional[int], "The service port of the seed instance for loading weights from remote instance.", + NS("model"), ] = None remote_instance_weight_loader_send_weights_group_ports: A[ Optional[List[int]], @@ -2652,24 +3001,31 @@ class ServerArgs: help="The communication group ports for loading weights from remote instance.", type_parser=json_list_type, ), + NS("model"), ] = None remote_instance_weight_loader_backend: A[ Literal["transfer_engine", "nccl", "modelexpress"], "The backend for loading weights from remote instance. Can be 'transfer_engine', 'nccl', or 'modelexpress'. Default is 'nccl'.", + NS("model"), ] = "nccl" remote_instance_weight_loader_start_seed_via_transfer_engine: A[ bool, "Start seed server via transfer engine backend for remote instance weight loader.", + NS("model"), ] = False engine_info_bootstrap_port: A[ int, "Port for the engine info bootstrap server. Default is 6789. Must be set explicitly when running multiple instances on the same node.", + NS("model"), ] = 6789 modelexpress_config: A[ Optional[str], 'JSON config for ModelExpress P2P weight loading. Keys: "url" (optional gRPC host:port override), "transport" ("nixl" or "transfer_engine"). Example: \'{"url": "localhost:8001", "transport": "nixl"}\'', + NS("model"), + ] = None + download_dir: A[ + Optional[str], "Model download directory for huggingface.", NS("model") ] = None - download_dir: A[Optional[str], "Model download directory for huggingface."] = None model_checksum: A[ Optional[str], Arg( @@ -2677,44 +3033,49 @@ class ServerArgs: nargs="?", const="", ), + NS("model"), ] = None delete_ckpt_after_loading: A[ - bool, - "Delete the model checkpoint after loading the model.", + bool, "Delete the model checkpoint after loading the model.", NS("model") ] = False # Checkpoint decryption decrypted_config_file: A[ - Optional[str], - "The path of the decrypted config file.", + Optional[str], "The path of the decrypted config file.", NS("model") ] = None decrypted_draft_config_file: A[ - Optional[str], - "The path of the decrypted draft config file.", + Optional[str], "The path of the decrypted draft config file.", NS("model") ] = None checkpoint_engine_wait_weights_before_ready: A[ bool, "If set, the server will wait for initial weights to be loaded via checkpoint-engine or other update methods before serving inference requests.", + NS("model"), ] = False # ------------------------------------------------------------------------- # Prefill delayer # ------------------------------------------------------------------------- enable_prefill_delayer: A[ - bool, "Enable prefill delayer for DP attention to reduce idle time." + bool, + "Enable prefill delayer for DP attention to reduce idle time.", + NS("schedule"), ] = False prefill_delayer_max_delay_passes: A[ - int, "Maximum forward passes to delay prefill." + int, "Maximum forward passes to delay prefill.", NS("schedule") ] = 30 prefill_delayer_token_usage_low_watermark: A[ - Optional[float], "Token usage low watermark for prefill delayer." + Optional[float], + "Token usage low watermark for prefill delayer.", + NS("schedule"), ] = None prefill_delayer_forward_passes_buckets: A[ Optional[List[float]], "Custom buckets for prefill delayer forward passes histogram. 0 and max_delay_passes-1 will be auto-added.", + NS("schedule"), ] = None prefill_delayer_wait_seconds_buckets: A[ Optional[List[float]], "Custom buckets for prefill delayer wait seconds histogram. 0 will be auto-added.", + NS("schedule"), ] = None prefill_delayer_queue_min_ratio: A[ Optional[float], @@ -2725,6 +3086,7 @@ class ServerArgs: "into a larger prefill. Unset (default) keeps the original slot-only " "behavior. Typical: 0.1 ~ 0.5." ), + NS("schedule"), ] = None prefill_delayer_max_delay_ms: A[ Optional[float], @@ -2734,6 +3096,7 @@ class ServerArgs: "when --prefill-delayer-queue-min-ratio is set. Typical: 1000 ~ " "5000; defaults to 5000 if unset." ), + NS("schedule"), ] = None # ------------------------------------------------------------------------- @@ -2750,6 +3113,7 @@ class ServerArgs: "min(4, max(2, (max-run + 5) // 6))). DFlash workloads auto-enable " "this with the formula when unset; other workloads stay disabled." ), + NS("schedule"), ] = None # ------------------------------------------------------------------------- @@ -2758,6 +3122,7 @@ class ServerArgs: enable_deterministic_inference: A[ bool, "Enable deterministic inference mode with batch invariant ops.", + NS("exec.deterministic"), ] = False rl_on_policy_target: A[ Optional[str], @@ -2765,6 +3130,7 @@ class ServerArgs: help="The training system that SGLang needs to match for true on-policy.", choices=RL_ON_POLICY_TARGET_CHOICES, ), + NS("exec.deterministic"), ] = None # ------------------------------------------------------------------------- @@ -2776,11 +3142,11 @@ class ServerArgs: help="KV cache canary mode. 'none' disables the canary (default). 'log' prints them while the server keeps running (production-safe). 'raise' fails the server on the first detected mismatch (CI lane).", choices=["none", "log", "raise"], ), + NS("observability"), ] = "none" - kv_canary_real_data: str = "none" + kv_canary_real_data: A[str, NS("observability")] = "none" kv_canary_sweep_interval: A[ - int, - "Every N forward steps, run a full-pool sweep.", + int, "Every N forward steps, run a full-pool sweep.", NS("observability") ] = 0 # ------------------------------------------------------------------------- @@ -2789,22 +3155,27 @@ class ServerArgs: enable_dynamic_batch_tokenizer: A[ bool, "Enable async dynamic batch tokenizer for improved performance when multiple requests arrive concurrently.", + NS("serving"), ] = False dynamic_batch_tokenizer_batch_size: A[ int, "[Only used if --enable-dynamic-batch-tokenizer is set] Maximum batch size for dynamic batch tokenizer.", + NS("serving"), ] = 32 dynamic_batch_tokenizer_batch_timeout: A[ float, "[Only used if --enable-dynamic-batch-tokenizer is set] Timeout in seconds for batching tokenization requests.", + NS("serving"), ] = 0.002 enable_tokenizer_batch_encode: A[ bool, "Enable batch tokenization for improved performance when processing multiple text inputs. Do not use with image inputs, pre-tokenized input_ids, or input_embeds.", + NS("serving"), ] = False disable_tokenizer_batch_decode: A[ bool, "Disable batch decoding when decoding multiple completions.", + NS("serving"), ] = False # ------------------------------------------------------------------------- @@ -2813,16 +3184,17 @@ class ServerArgs: debug_tensor_dump_output_folder: A[ Optional[str], "The output folder for dumping tensors. In Eagle mode, tensor outputs from draft and target models are stored in separate subdirectories ('draft' and 'target').", + NS("observability"), ] = None # None means dump all layers. debug_tensor_dump_layers: A[ Optional[List[int]], "The layer ids to dump. Dump all layers if not specified.", + NS("observability"), ] = None # TODO(guoyuhong): clean the old dumper code. debug_tensor_dump_input_file: A[ - Optional[str], - "The input filename for dumping tensors", + Optional[str], "The input filename for dumping tensors", NS("observability") ] = None # ------------------------------------------------------------------------- @@ -2831,38 +3203,45 @@ class ServerArgs: enable_memory_saver: A[ bool, "Allow saving memory using release_memory_occupation and resume_memory_occupation", + NS("exec.features"), ] = False enable_weights_cpu_backup: A[ bool, "Save model weights (both main model and draft model, if any) to CPU memory during release_weights_occupation and resume_weights_occupation", + NS("exec.features"), ] = False enable_draft_weights_cpu_backup: A[ bool, "Save draft model weights to CPU memory during release_weights_occupation and resume_weights_occupation", + NS("exec.features"), ] = False enable_custom_logit_processor: A[ bool, "Enable users to pass custom logit processors to the server (disabled by default for security)", + NS("exec.features"), ] = False enable_return_hidden_states: A[ - bool, - "Enable returning hidden states with responses.", + bool, "Enable returning hidden states with responses.", NS("exec.features") ] = False enable_return_routed_experts: A[ bool, "Enable returning routed experts of each layer with responses.", + NS("exec.features"), ] = False enable_return_indexer_topk: A[ bool, "Enable returning indexer topk indices of layers with indexer with responses.", + NS("exec.features"), ] = False disable_outlines_disk_cache: A[ bool, "Disable disk cache of outlines to avoid possible crashes related to file system or high concurrency.", + NS("exec.features"), ] = False enable_mis: A[ bool, "Enable Multi-Item Scoring optimization. Combines query and multiple items into a single sequence for efficient batch processing. Requires --attention-backend flashinfer; auto-disables CUDA graph, radix cache, and chunked prefill.", + NS("exec.features"), ] = False # ------------------------------------------------------------------------- @@ -2874,10 +3253,12 @@ class ServerArgs: help="JSON-formatted forward hook specifications to attach to the model.", type_parser=json_list_type, ), + NS("observability"), ] = None msprobe_dump_config: A[ Optional[str], "The path of the JSON configuration file for msProbe. If specified, enables msProbe dump.", + NS("observability"), ] = None def __post_init__(self): diff --git a/test/registered/unit/test_server_args_namespaces.py b/test/registered/unit/test_server_args_namespaces.py new file mode 100644 index 000000000..e942b7b7d --- /dev/null +++ b/test/registered/unit/test_server_args_namespaces.py @@ -0,0 +1,71 @@ +"""Coverage lint for the ServerArgs -> RuntimeContext namespace split. + +Every ServerArgs field must carry an ``NS("")`` marker in its ``Annotated`` +metadata, and every path must be one of the known domains. This is the guardrail +that fails when an upstream PR adds a ServerArgs field without assigning it a +namespace (the property that retires the old hand-maintained mirror file). +""" + +import dataclasses +import unittest + +from sglang.srt.arg_groups.arg_utils import namespace_of +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +# Locked taxonomy (global_context/11-server-args-namespace-split.md). +VALID_NAMESPACES = { + "parallel", + "device", + "model", + "schedule", + "memory", + "spec", + "lora", + "mm", + "disagg", + "serving", + "observability", + "exec.kernel", + "exec.moe", + "exec.graph", + "exec.comm", + "exec.mamba", + "exec.overlap", + "exec.offload", + "exec.dllm", + "exec.deterministic", + "exec.features", +} + + +def _field_names(): + return {f.name for f in dataclasses.fields(ServerArgs)} + + +class TestServerArgsNamespaces(CustomTestCase): + def test_every_field_has_a_namespace(self): + nsmap = namespace_of(ServerArgs) + missing = sorted(_field_names() - set(nsmap)) + self.assertFalse( + missing, + "ServerArgs fields missing an NS(...) marker " + f"(assign a namespace in server_args.py): {missing}", + ) + + def test_all_namespaces_are_known(self): + nsmap = namespace_of(ServerArgs) + bad = {f: p for f, p in nsmap.items() if p not in VALID_NAMESPACES} + self.assertFalse(bad, f"unknown namespace paths (typo or new domain?): {bad}") + + def test_namespace_map_covers_all_fields(self): + nsmap = namespace_of(ServerArgs) + self.assertEqual(set(nsmap), _field_names()) + self.assertGreaterEqual(len(nsmap), 440) + + +if __name__ == "__main__": + unittest.main()