[Utils] Support configure log level at runtime (#26583)

docs_new/docs/advanced_features/observability.mdx

@@ -18,6 +18,12 @@ By default, SGLang does not log any request contents. You can log them by using
 You can control the verbosity by using `--log-request-level`.
 See [Logging](./server_arguments#logging) for more details.
 
+You can change verbosity at runtime:
+
+```bash Command
+python3 -m sglang.srt.managers.configure_logging --url http://localhost:30000 --log-level=debug
+```
+
 ## Request Dump and Replay
 
 You can dump all requests and replay them later for benchmarking or other purposes.

python/sglang/srt/managers/configure_logging.py

@@ -27,6 +27,13 @@ import requests
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--url", type=str, default="http://localhost:30000")
+    parser.add_argument(
+        "--log-level",
+        type=str,
+        default=None,
+        choices=["debug", "info", "warning", "error", "critical"],
+        help="Set runtime log level",
+    )
     parser.add_argument("--log-requests", action="store_true")
     parser.add_argument("--log-requests-level", type=int, default=3)
     parser.add_argument(
@@ -50,6 +57,7 @@ if __name__ == "__main__":
         "log_requests_level": args.log_requests_level,  # Log full requests
         "dump_requests_folder": args.dump_requests_folder,
         "dump_requests_threshold": args.dump_requests_threshold,
+        "log_level": args.log_level,
     }
     if args.dump_requests_exclude_meta_keys is not None:
         payload["dump_requests_exclude_meta_keys"] = [

python/sglang/srt/managers/detokenizer_manager.py

@@ -32,6 +32,7 @@ from sglang.srt.managers.io_struct import (
     BatchEmbeddingOutput,
     BatchStrOutput,
     BatchTokenIDOutput,
+    ConfigureLoggingReq,
     FreezeGCReq,
 )
 from sglang.srt.managers.multi_tokenizer_mixin import MultiHttpWorkerDetokenizerMixin
@@ -135,6 +136,7 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
                 (BatchEmbeddingOutput, self.handle_batch_embedding_out),
                 (BatchTokenIDOutput, self.handle_batch_token_id_out),
                 (FreezeGCReq, self.handle_freeze_gc_req),
+                (ConfigureLoggingReq, self.handle_configure_logging_req),
             ]
         )
 
@@ -400,6 +402,10 @@ class DetokenizerManager(MultiHttpWorkerDetokenizerMixin):
         freeze_gc("Detokenizer Manager")
         return None
 
+    def handle_configure_logging_req(self, recv_req: ConfigureLoggingReq):
+        if recv_req.log_level is not None:
+            logging.getLogger().setLevel(recv_req.log_level.upper())
+
 
 def is_health_check_request(rid: Optional[str]) -> bool:
     return isinstance(rid, str) and rid.startswith(HEALTH_CHECK_RID_PREFIX)

python/sglang/srt/managers/io_struct.py

@@ -1773,6 +1773,7 @@ class ConfigureLoggingReq(BaseReq):
     log_requests: Optional[bool] = None
     log_requests_level: Optional[int] = None
     log_requests_format: Optional[str] = None
+    log_level: Optional[str] = None
     dump_requests_folder: Optional[str] = None
     dump_requests_threshold: Optional[int] = None
     crash_dump_folder: Optional[str] = None

python/sglang/srt/managers/scheduler.py

@@ -95,6 +95,7 @@ from sglang.srt.managers.io_struct import (
     ClearHiCacheReqInput,
     ClearHiCacheReqOutput,
     CloseSessionReqInput,
+    ConfigureLoggingReq,
     ContinueGenerationReqInput,
     DestroyWeightsUpdateGroupReqInput,
     DetachHiCacheStorageReqInput,
@@ -1343,6 +1344,7 @@ class Scheduler(
                 (GetLoadsReqInput, self.handle_get_loads_req),
                 (PauseGenerationReqInput, self.pause_generation),
                 (ContinueGenerationReqInput, self.continue_generation),
+                (ConfigureLoggingReq, self.configure_logging),
                 (DumperControlReqInput, self.handle_dumper_control),
                 (AddExternalCorpusReqInput, self.add_external_corpus),
                 (
@@ -3760,6 +3762,11 @@ class Scheduler(
         self.ipc_channels.send_to_detokenizer.send_output(recv_req, recv_req)
         return None
 
+    def configure_logging(self, recv_req: ConfigureLoggingReq):
+        if recv_req.log_level is not None:
+            logging.getLogger().setLevel(recv_req.log_level.upper())
+        self.ipc_channels.send_to_detokenizer.send_output(recv_req, recv_req)
+
     def handle_dumper_control(self, recv_req: DumperControlReqInput):
         from sglang.srt.debug_utils.dumper import dumper
 

python/sglang/srt/managers/tokenizer_manager.py

@@ -1651,6 +1651,12 @@ class TokenizerManager(TokenizerControlMixin, TokenizerManagerScoreMixin):
             )
         if obj.crash_dump_folder is not None:
             self.crash_dump_folder = obj.crash_dump_folder
+        if obj.log_level is not None:
+            # setLevel() may raise exception if obj.log_level is illegal string.
+            # Let the exception propagate to the caller.
+            # Only legal requests will be sent to scheduler.
+            logging.getLogger().setLevel(obj.log_level.upper())
+            self.send_to_scheduler.send_pyobj(obj)
         logging.info(f"Config logging: {obj=}")
 
     async def freeze_gc(self):