minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Clean up server args and engine startup processes (#15015)

Browse Source
This commit is contained in:
Lianmin Zheng
2025-12-12 18:46:07 -08:00
committed by GitHub
parent 313f59ad80
commit 267170bf1d
7 changed files with 105 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/server_args.py
+45 -37
View File
@@ -23,7 +23,7 @@ import logging
import os
import random
import tempfile
from typing import Any, Dict, List, Literal, Optional, Union
from typing import Any, Callable, Dict, List, Literal, Optional, Union
import orjson
@@ -252,7 +252,6 @@ class ServerArgs:
skip_tokenizer_init: bool = False
load_format: str = "auto"
model_loader_extra_config: str = "{}"
rl_quant_profile: Optional[str] = None # For flash_rl load format
trust_remote_code: bool = False
context_length: Optional[int] = None
is_embedding: bool = False
@@ -281,6 +280,7 @@ class ServerArgs:
modelopt_checkpoint_save_path: Optional[str] = None
modelopt_export_path: Optional[str] = None
quantize_and_serve: bool = False
rl_quant_profile: Optional[str] = None # For flash_rl load format
# Memory and scheduling
mem_fraction_static: Optional[float] = None
@@ -320,7 +320,7 @@ class ServerArgs:
base_gpu_id: int = 0
gpu_id_step: int = 1
sleep_on_idle: bool = False
mm_process_config: Optional[Dict[str, Any]] = None
custom_sigquit_handler: Optional[Callable] = None
# Logging
log_level: str = "info"
@@ -606,14 +606,13 @@ class ServerArgs:
mm_max_concurrent_calls: int = 32
mm_per_request_timeout: float = 10.0
enable_broadcast_mm_inputs_process: bool = False
mm_enable_dp_encoder: bool = False
mm_process_config: Optional[Dict[str, Any]] = None
# For checkpoint decryption
decrypted_config_file: Optional[str] = None
decrypted_draft_config_file: Optional[str] = None
# For encoder dp
mm_enable_dp_encoder: bool = False
# For forward hooks
forward_hooks: Optional[List[dict[str, Any]]] = None
@@ -643,9 +642,6 @@ class ServerArgs:
self._handle_cpu_backends()
self._handle_npu_backends()
# Handle compilation config
self._handle_compilation_cfg()
# Apply model-specific adjustments.
self._handle_model_specific_adjustments()
@@ -709,7 +705,7 @@ class ServerArgs:
self._handle_elastic_ep()
def _handle_deprecated_args(self):
# handle deprecated tool call parsers
# Handle deprecated tool call parsers
deprecated_tool_call_parsers = {"qwen25": "qwen", "glm45": "glm"}
if self.tool_call_parser in deprecated_tool_call_parsers:
logger.warning(
@@ -729,6 +725,16 @@ class ServerArgs:
if self.mm_process_config is None:
self.mm_process_config = {}
# Handle ModelScope model downloads
if get_bool_env_var("SGLANG_USE_MODELSCOPE"):
if not os.path.exists(self.model_path):
from modelscope import snapshot_download
self.model_path = snapshot_download(self.model_path)
self.tokenizer_path = snapshot_download(
self.tokenizer_path, ignore_patterns=["*.bin", "*.safetensors"]
)
def _handle_gpu_memory_settings(self, gpu_mem):
"""
Configure GPU memory-dependent settings including
@@ -908,7 +914,7 @@ class ServerArgs:
if self.disable_cuda_graph_padding:
capture_bs = list(range(1, self.cuda_graph_max_bs + 1))
elif self.speculative_algorithm is None:
# Normal case: [1, 2, 4, 8, 12] + list(range(16, 257, 8)) + list(range(272, 512, 16)) + list(range(512, cuda_graph_max_bs + 1))
# Normal case:
capture_bs = (
[1, 2, 4, 8, 12]
+ list(range(16, 257, 8))
@@ -916,7 +922,7 @@ class ServerArgs:
+ list(range(512, self.cuda_graph_max_bs + 1, 32))
)
else:
# Spec decoding case: list(range(1, 9, 1)) + list(range(10, 33, 2)) + list(range(40, 64, 4)) + list(range(72, 257, 8))
# Spec decoding case: less padding for smaller batch sizes
capture_bs = (
list(range(1, 9, 1))
+ list(range(10, 33, 2))
@@ -959,21 +965,19 @@ class ServerArgs:
self.attention_backend = "intel_amx"
self.sampling_backend = "pytorch"
def _handle_compilation_cfg(self):
# NPU platform
if is_npu() and self.piecewise_cuda_graph_compiler != "eager":
logger.warning(
"At this moment Ascend platform only support prefill graph compilation with "
"piecewise_cuda_graph_compiler='eager', change piecewise_cuda_graph_compiler to 'eager'."
)
self.piecewise_cuda_graph_compiler = "eager"
def _handle_npu_backends(self):
if self.device == "npu":
from sglang.srt.hardware_backend.npu.utils import set_default_server_args
set_default_server_args(self)
if self.piecewise_cuda_graph_compiler != "eager":
logger.warning(
"At this moment Ascend platform only support prefill graph compilation with "
"piecewise_cuda_graph_compiler='eager', change piecewise_cuda_graph_compiler to 'eager'."
)
self.piecewise_cuda_graph_compiler = "eager"
def _handle_model_specific_adjustments(self):
from sglang.srt.configs.model_config import is_deepseek_nsa
@@ -2283,12 +2287,6 @@ class ServerArgs:
"This will be passed to the model loader corresponding to the chosen load_format.",
default=ServerArgs.model_loader_extra_config,
)
parser.add_argument(
"--rl-quant-profile",
type=str,
default=ServerArgs.rl_quant_profile,
help="Path to the FlashRL quantization profile. Required when using --load-format flash_rl.",
)
parser.add_argument(
"--trust-remote-code",
action="store_true",
@@ -2464,6 +2462,12 @@ class ServerArgs:
"This is useful for development and prototyping. For production, it's recommended "
"to use separate quantization and deployment steps.",
)
parser.add_argument(
"--rl-quant-profile",
type=str,
default=ServerArgs.rl_quant_profile,
help="Path to the FlashRL quantization profile. Required when using --load-format flash_rl.",
)
# Memory and scheduling
parser.add_argument(
@@ -2687,10 +2691,8 @@ class ServerArgs:
help="Reduce CPU usage when sglang is idle.",
)
parser.add_argument(
"--mm-process-config",
type=json.loads,
default=ServerArgs.mm_process_config,
help="Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`",
"--custom-sigquit-handler",
help="Register a custom sigquit handler so you can do additional cleanup after the server is shutdown. This is only available for Engine, not for CLI.",
)
# Logging
@@ -4110,6 +4112,18 @@ class ServerArgs:
default=ServerArgs.enable_broadcast_mm_inputs_process,
help="Enable broadcast mm-inputs process in scheduler.",
)
parser.add_argument(
"--mm-process-config",
type=json.loads,
default=ServerArgs.mm_process_config,
help="Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`",
)
parser.add_argument(
"--mm-enable-dp-encoder",
action="store_true",
default=ServerArgs.mm_enable_dp_encoder,
help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
)
# For checkpoint decryption
parser.add_argument(
@@ -4124,12 +4138,6 @@ class ServerArgs:
default=ServerArgs.decrypted_draft_config_file,
help="The path of the decrypted draft config file.",
)
parser.add_argument(
"--mm-enable-dp-encoder",
action="store_true",
default=ServerArgs.mm_enable_dp_encoder,
help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
)
# For registering hooks
parser.add_argument(