Clean up server args and engine startup processes (#15015)

.github/workflows/pr-gate.yml

@@ -55,7 +55,7 @@ jobs:
         run: |
           labels='${{ steps.pr.outputs.labels }}'
           if [[ "${{ contains(fromJson(steps.pr.outputs.labels), 'run-ci') }}" == "false" ]]; then
-            echo "Missing required label 'run-ci'."
+            echo "Missing required label 'run-ci'. See https://docs.sglang.io/developer_guide/contribution_guide.html#how-to-trigger-ci-tests for more details."
             exit 1
           fi
 

python/pyproject.toml

@@ -98,9 +98,12 @@ diffusion = [
   "cache-dit==1.1.8"
 ]
 
-[tool.uv.extra-build-dependencies]
+tracing = [
-st-attn = ["torch", "setuptools"]
+  "opentelemetry-api",
-vsa = ["torch", "setuptools"]
+  "opentelemetry-exporter-otlp",
+  "opentelemetry-exporter-otlp-proto-grpc",
+  "opentelemetry-sdk",
+]
 
 test = [
   "accelerate",
@@ -109,18 +112,18 @@ test = [
   "jsonlines",
   "matplotlib",
   "pandas",
+  "parameterized",
   "peft",
   "pytest",
   "sentence_transformers",
   "tabulate",
 ]
+
 dev = ["sglang[test]"]
-tracing = [
+
-  "opentelemetry-api",
+[tool.uv.extra-build-dependencies]
-  "opentelemetry-exporter-otlp",
+st-attn = ["torch", "setuptools"]
-  "opentelemetry-exporter-otlp-proto-grpc",
+vsa = ["torch", "setuptools"]
-  "opentelemetry-sdk",
-]
 
 [project.urls]
 "Homepage" = "https://github.com/sgl-project/sglang"

python/sglang/srt/entrypoints/engine.py

@@ -76,7 +76,6 @@ from sglang.srt.utils import (
     launch_dummy_health_check_server,
     maybe_reindex_device_id,
     numa_utils,
-    prepare_model_and_tokenizer,
     set_prometheus_multiproc_dir,
     set_ulimit,
 )
@@ -105,11 +104,6 @@ def _launch_subprocesses(
         port_args = PortArgs.init_new(server_args)
         logger.info(f"{server_args=}")
 
-    # If using model from www.modelscope.cn, first download the model
-    server_args.model_path, server_args.tokenizer_path = prepare_model_and_tokenizer(
-        server_args.model_path, server_args.tokenizer_path
-    )
-
     # Launch scheduler processes
     scheduler_procs, scheduler_pipe_readers = _launch_scheduler_processes(
         server_args=server_args,
@@ -826,6 +820,7 @@ def _set_envs_and_config(server_args: ServerArgs):
     set_ulimit()
 
     # Check flashinfer version
+    if not get_bool_env_var("SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK"):
         if server_args.attention_backend == "flashinfer":
             assert_pkg_version(
                 "flashinfer_python",
@@ -834,14 +829,14 @@ def _set_envs_and_config(server_args: ServerArgs):
                 "reinstall the latest version by following the instructions "
                 "at https://docs.flashinfer.ai/installation.html.",
             )
-    if _is_cuda and not get_bool_env_var("SGLANG_SKIP_SGL_KERNEL_VERSION_CHECK"):
+        if _is_cuda:
             assert_pkg_version(
                 "sgl-kernel",
                 "0.3.19",
                 "Please reinstall the latest version with `pip install sgl-kernel --force-reinstall`",
             )
 
-    if True:  # Keep this check for internal code compatibility
+    if server_args.custom_sigquit_handler is None:
         # Register the signal handler.
         # The child processes will send SIGQUIT to this process when any error happens
         # This process then clean up the whole process tree
@@ -854,6 +849,12 @@ def _set_envs_and_config(server_args: ServerArgs):
             kill_process_tree(os.getpid())
 
         signal.signal(signal.SIGQUIT, launch_phase_sigquit_handler)
+    else:
+        # Allow users to register a custom SIGQUIT handler for things like crash dump
+        logger.error(
+            f"Using custom SIGQUIT handler: {server_args.custom_sigquit_handler}"
+        )
+        signal.signal(signal.SIGQUIT, server_args.custom_sigquit_handler)
 
     # Set mp start method
     mp.set_start_method("spawn", force=True)

python/sglang/srt/server_args.py

@@ -23,7 +23,7 @@ import logging
 import os
 import random
 import tempfile
-from typing import Any, Dict, List, Literal, Optional, Union
+from typing import Any, Callable, Dict, List, Literal, Optional, Union
 
 import orjson
 
@@ -252,7 +252,6 @@ class ServerArgs:
     skip_tokenizer_init: bool = False
     load_format: str = "auto"
     model_loader_extra_config: str = "{}"
-    rl_quant_profile: Optional[str] = None  # For flash_rl load format
     trust_remote_code: bool = False
     context_length: Optional[int] = None
     is_embedding: bool = False
@@ -281,6 +280,7 @@ class ServerArgs:
     modelopt_checkpoint_save_path: Optional[str] = None
     modelopt_export_path: Optional[str] = None
     quantize_and_serve: bool = False
+    rl_quant_profile: Optional[str] = None  # For flash_rl load format
 
     # Memory and scheduling
     mem_fraction_static: Optional[float] = None
@@ -320,7 +320,7 @@ class ServerArgs:
     base_gpu_id: int = 0
     gpu_id_step: int = 1
     sleep_on_idle: bool = False
-    mm_process_config: Optional[Dict[str, Any]] = None
+    custom_sigquit_handler: Optional[Callable] = None
 
     # Logging
     log_level: str = "info"
@@ -606,14 +606,13 @@ class ServerArgs:
     mm_max_concurrent_calls: int = 32
     mm_per_request_timeout: float = 10.0
     enable_broadcast_mm_inputs_process: bool = False
+    mm_enable_dp_encoder: bool = False
+    mm_process_config: Optional[Dict[str, Any]] = None
 
     # For checkpoint decryption
     decrypted_config_file: Optional[str] = None
     decrypted_draft_config_file: Optional[str] = None
 
-    # For encoder dp
-    mm_enable_dp_encoder: bool = False
-
     # For forward hooks
     forward_hooks: Optional[List[dict[str, Any]]] = None
 
@@ -643,9 +642,6 @@ class ServerArgs:
         self._handle_cpu_backends()
         self._handle_npu_backends()
 
-        # Handle compilation config
-        self._handle_compilation_cfg()
-
         # Apply model-specific adjustments.
         self._handle_model_specific_adjustments()
 
@@ -709,7 +705,7 @@ class ServerArgs:
         self._handle_elastic_ep()
 
     def _handle_deprecated_args(self):
-        # handle deprecated tool call parsers
+        # Handle deprecated tool call parsers
         deprecated_tool_call_parsers = {"qwen25": "qwen", "glm45": "glm"}
         if self.tool_call_parser in deprecated_tool_call_parsers:
             logger.warning(
@@ -729,6 +725,16 @@ class ServerArgs:
         if self.mm_process_config is None:
             self.mm_process_config = {}
 
+        # Handle ModelScope model downloads
+        if get_bool_env_var("SGLANG_USE_MODELSCOPE"):
+            if not os.path.exists(self.model_path):
+                from modelscope import snapshot_download
+
+                self.model_path = snapshot_download(self.model_path)
+                self.tokenizer_path = snapshot_download(
+                    self.tokenizer_path, ignore_patterns=["*.bin", "*.safetensors"]
+                )
+
     def _handle_gpu_memory_settings(self, gpu_mem):
         """
         Configure GPU memory-dependent settings including
@@ -908,7 +914,7 @@ class ServerArgs:
         if self.disable_cuda_graph_padding:
             capture_bs = list(range(1, self.cuda_graph_max_bs + 1))
         elif self.speculative_algorithm is None:
-            # Normal case: [1, 2, 4, 8, 12] + list(range(16, 257, 8)) + list(range(272, 512, 16)) + list(range(512, cuda_graph_max_bs + 1))
+            # Normal case:
             capture_bs = (
                 [1, 2, 4, 8, 12]
                 + list(range(16, 257, 8))
@@ -916,7 +922,7 @@ class ServerArgs:
                 + list(range(512, self.cuda_graph_max_bs + 1, 32))
             )
         else:
-            # Spec decoding case: list(range(1, 9, 1)) + list(range(10, 33, 2)) + list(range(40, 64, 4)) + list(range(72, 257, 8))
+            # Spec decoding case: less padding for smaller batch sizes
             capture_bs = (
                 list(range(1, 9, 1))
                 + list(range(10, 33, 2))
@@ -959,21 +965,19 @@ class ServerArgs:
                 self.attention_backend = "intel_amx"
             self.sampling_backend = "pytorch"
 
-    def _handle_compilation_cfg(self):
-        # NPU platform
-        if is_npu() and self.piecewise_cuda_graph_compiler != "eager":
-            logger.warning(
-                "At this moment Ascend platform only support prefill graph compilation with "
-                "piecewise_cuda_graph_compiler='eager', change piecewise_cuda_graph_compiler to 'eager'."
-            )
-            self.piecewise_cuda_graph_compiler = "eager"
-
     def _handle_npu_backends(self):
         if self.device == "npu":
             from sglang.srt.hardware_backend.npu.utils import set_default_server_args
 
             set_default_server_args(self)
 
+            if self.piecewise_cuda_graph_compiler != "eager":
+                logger.warning(
+                    "At this moment Ascend platform only support prefill graph compilation with "
+                    "piecewise_cuda_graph_compiler='eager', change piecewise_cuda_graph_compiler to 'eager'."
+                )
+                self.piecewise_cuda_graph_compiler = "eager"
+
     def _handle_model_specific_adjustments(self):
         from sglang.srt.configs.model_config import is_deepseek_nsa
 
@@ -2283,12 +2287,6 @@ class ServerArgs:
             "This will be passed to the model loader corresponding to the chosen load_format.",
             default=ServerArgs.model_loader_extra_config,
         )
-        parser.add_argument(
-            "--rl-quant-profile",
-            type=str,
-            default=ServerArgs.rl_quant_profile,
-            help="Path to the FlashRL quantization profile. Required when using --load-format flash_rl.",
-        )
         parser.add_argument(
             "--trust-remote-code",
             action="store_true",
@@ -2464,6 +2462,12 @@ class ServerArgs:
             "This is useful for development and prototyping. For production, it's recommended "
             "to use separate quantization and deployment steps.",
         )
+        parser.add_argument(
+            "--rl-quant-profile",
+            type=str,
+            default=ServerArgs.rl_quant_profile,
+            help="Path to the FlashRL quantization profile. Required when using --load-format flash_rl.",
+        )
 
         # Memory and scheduling
         parser.add_argument(
@@ -2687,10 +2691,8 @@ class ServerArgs:
             help="Reduce CPU usage when sglang is idle.",
         )
         parser.add_argument(
-            "--mm-process-config",
+            "--custom-sigquit-handler",
-            type=json.loads,
+            help="Register a custom sigquit handler so you can do additional cleanup after the server is shutdown. This is only available for Engine, not for CLI.",
-            default=ServerArgs.mm_process_config,
-            help="Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`",
         )
 
         # Logging
@@ -4110,6 +4112,18 @@ class ServerArgs:
             default=ServerArgs.enable_broadcast_mm_inputs_process,
             help="Enable broadcast mm-inputs process in scheduler.",
         )
+        parser.add_argument(
+            "--mm-process-config",
+            type=json.loads,
+            default=ServerArgs.mm_process_config,
+            help="Multimodal preprocessing config, a json config contains keys: `image`, `video`, `audio`",
+        )
+        parser.add_argument(
+            "--mm-enable-dp-encoder",
+            action="store_true",
+            default=ServerArgs.mm_enable_dp_encoder,
+            help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
+        )
 
         # For checkpoint decryption
         parser.add_argument(
@@ -4124,12 +4138,6 @@ class ServerArgs:
             default=ServerArgs.decrypted_draft_config_file,
             help="The path of the decrypted draft config file.",
         )
-        parser.add_argument(
-            "--mm-enable-dp-encoder",
-            action="store_true",
-            default=ServerArgs.mm_enable_dp_encoder,
-            help="Enabling data parallelism for mm encoder. The dp size will be set to the tp size automatically.",
-        )
 
         # For registering hooks
         parser.add_argument(

python/sglang/srt/utils/common.py

@@ -1145,18 +1145,6 @@ def add_api_key_middleware(app, api_key: str):
         return await call_next(request)
 
 
-def prepare_model_and_tokenizer(model_path: str, tokenizer_path: str):
-    if get_bool_env_var("SGLANG_USE_MODELSCOPE"):
-        if not os.path.exists(model_path):
-            from modelscope import snapshot_download
-
-            model_path = snapshot_download(model_path)
-            tokenizer_path = snapshot_download(
-                tokenizer_path, ignore_patterns=["*.bin", "*.safetensors"]
-            )
-    return model_path, tokenizer_path
-
-
 def configure_logger(server_args, prefix: str = ""):
     if SGLANG_LOGGING_CONFIG_PATH := os.getenv("SGLANG_LOGGING_CONFIG_PATH"):
         if not os.path.exists(SGLANG_LOGGING_CONFIG_PATH):

scripts/code_sync/guideline.md

@@ -25,3 +25,27 @@ It learns from [Copybara](https://github.com/google/copybara), a tool used at Go
     - For example, you can have a PR that changes both `python/sglang/srt` and `python/sglang/private/srt`. Once you merge the PR into the private repo, `python/sglang/srt` becomes desynced between the two repos. You need to run this action on your merge commit immediately to open a PR to send your diff to the OSS repo. Then, we need to merge the OSS PR as soon as possible. Once your OSS PR is merged, we can run action A again.
     - Action A copies files directly, but Action B applies diff. This is because OSS is the source of truth; action A can just copy files. Action B cannot copy, so it uses diff instead.
 - This action currently needs a manual trigger in order to prevent incidental code leaks. One can also consider making it automatic.
+
+## Examples
+- If you want to have some private server arguments, you can create a new file `python/sglang/private/server_args.py`. It defines a class that inherits the oss ServerArgs.
+    ```python
+    from sglang.srt.server_args import ServerArgs as ServerArgsOSS
+
+    @dataclasses.dataclass
+    class ServerArgs(ServerArgsOSS):
+        private_flag: str = "foo"
+
+        @staticmethod
+        def add_cli_args(parser: argparse.ArgumentParser):
+            # Get all public args
+            ServerArgsOSS.add_cli_args(parser)
+
+            # Add your private flags
+            parser.add_argument(
+                "--private-flag",
+                type=str,
+                default=ServerArgs.private_flag,
+            )
+    ```
+- Similarly, you can inherit `Engine` and override `launch_subprocesses_func`, `server_args_class`.
+- You can pass your own subprocesses launch functions to `launch_server.py::launch_server`

test/srt/quant/test_awq.py

@@ -71,7 +71,7 @@ class TestAWQMarlinBfloat16(CustomTestCase):
         )
 
         metrics = run_eval(args)
-        self.assertGreater(metrics["score"], 0.85)
+        self.assertGreater(metrics["score"], 0.83)
 
 
 class TestAWQMarlinFloat16(CustomTestCase):