Clean up logging (#15919)

python/sglang/srt/distributed/device_communicators/custom_all_reduce.py

@@ -433,6 +433,25 @@ def dispatch_custom_allreduce():
     On AMD with 1-stage AR enabled, use sglang's CustomAllreduce (has deterministic_all_reduce method).
     Otherwise use AiterCustomAllreduce if available.
     """
+    if _is_cuda:
+        return CustomAllreduce
+
+    assert _is_hip
+
+    if envs.SGLANG_USE_1STAGE_ALLREDUCE.is_set():
+        if envs.SGLANG_USE_1STAGE_ALLREDUCE.get():
+            logger.debug(
+                "[AR] All-reduce: 1-stage kernel (SGLANG_USE_1STAGE_ALLREDUCE=1)"
+            )
+        else:
+            logger.debug("[AR] All-reduce: default (SGLANG_USE_1STAGE_ALLREDUCE=0)")
+    elif envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.get():
+        logger.debug(
+            "[AR] All-reduce: 1-stage kernel (deterministic inference enabled)"
+        )
+    else:
+        logger.debug("[AR] All-reduce: default")
+
     # Check if 1-stage AR should be used
     if envs.SGLANG_USE_1STAGE_ALLREDUCE.is_set():
         use_1stage = envs.SGLANG_USE_1STAGE_ALLREDUCE.get()
@@ -441,11 +460,10 @@ def dispatch_custom_allreduce():
 
     # On AMD with 1-stage AR, use sglang's CustomAllreduce
     # (AiterCustomAllreduce doesn't have deterministic_all_reduce method)
-    if is_hip() and use_1stage:
+    if use_1stage:
-        logger.info("[AR] Using sglang CustomAllreduce (1-stage kernel)")
         return CustomAllreduce
 
-    if is_hip() and get_bool_env_var("SGLANG_USE_AITER_AR", default="true"):
+    if get_bool_env_var("SGLANG_USE_AITER_AR", default="true"):
         try:
             from aiter.dist.device_communicators.custom_all_reduce import (
                 CustomAllreduce as AiterCustomAllreduce,
@@ -460,5 +478,5 @@ def dispatch_custom_allreduce():
                 e,
             )
             return CustomAllreduce
-    logger.info("[AR] Using sglang CustomAllreduce")
+
     return CustomAllreduce

python/sglang/srt/distributed/parallel_state.py

@@ -376,28 +376,12 @@ class GroupCoordinator:
                     group=self.cpu_group,
                     device=self.device,
                 )
-                # Log which all-reduce mode will be used
-                if is_hip():
-                    if envs.SGLANG_USE_1STAGE_ALLREDUCE.is_set():
-                        if envs.SGLANG_USE_1STAGE_ALLREDUCE.get():
-                            logger.info(
-                                "[AR] All-reduce: 1-stage kernel (SGLANG_USE_1STAGE_ALLREDUCE=1)"
-                            )
-                        else:
-                            logger.info(
-                                "[AR] All-reduce: default (SGLANG_USE_1STAGE_ALLREDUCE=0)"
-                            )
-                    elif envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.get():
-                        logger.info(
-                            "[AR] All-reduce: 1-stage kernel (deterministic inference enabled)"
-                        )
-                    else:
-                        logger.info("[AR] All-reduce: default")
             except Exception as e:
                 logger.warning(
                     f"Setup Custom allreduce failed with {e}. To silence this "
                     "warning, specify --disable-custom-all-reduce explicitly."
                 )
+
             if is_hip():
                 try:
                     # Initialize a custom quick all-reduce implementation for AMD

python/sglang/srt/managers/tokenizer_manager.py

@@ -185,27 +185,11 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
         self.preferred_sampling_params = server_args.preferred_sampling_params
         self.crash_dump_folder = server_args.crash_dump_folder
         self.enable_trace = server_args.enable_trace
-
-        # Read model args
-        self.model_path = server_args.model_path
-        self.served_model_name = server_args.served_model_name
-        self.model_config = ModelConfig.from_server_args(server_args)
-        self.is_generation = self.model_config.is_generation
-        self.is_image_gen = self.model_config.is_image_gen
-        self.context_len = self.model_config.context_len
-        self.image_token_id = self.model_config.image_token_id
-        self.max_req_input_len = None  # Will be set later in engine.py
-        speculative_algorithm = SpeculativeAlgorithm.from_string(
-            server_args.speculative_algorithm
-        )
-        self.reserve_input_token_num = (
-            0
-            if speculative_algorithm.is_none()
-            else server_args.speculative_num_draft_tokens
-        )
-
         set_global_server_args_for_tokenizer(server_args)
 
+        # Init model config
+        self.init_model_config()
+
         # Initialize tokenizer and multimodalprocessor
         self.init_tokenizer_and_processor()
 
@@ -233,6 +217,27 @@ class TokenizerManager(TokenizerCommunicatorMixin, TokenizerManagerMultiItemMixi
         # Init request dispatcher
         self.init_request_dispatcher()
 
+    def init_model_config(self):
+        server_args = self.server_args
+
+        # Read model args
+        self.model_path = server_args.model_path
+        self.served_model_name = server_args.served_model_name
+        self.model_config = ModelConfig.from_server_args(server_args)
+        self.is_generation = self.model_config.is_generation
+        self.is_image_gen = self.model_config.is_image_gen
+        self.context_len = self.model_config.context_len
+        self.image_token_id = self.model_config.image_token_id
+        self.max_req_input_len = None  # Will be set later in engine.py
+        speculative_algorithm = SpeculativeAlgorithm.from_string(
+            server_args.speculative_algorithm
+        )
+        self.reserve_input_token_num = (
+            0
+            if speculative_algorithm.is_none()
+            else server_args.speculative_num_draft_tokens
+        )
+
     def init_tokenizer_and_processor(self):
         server_args = self.server_args
 

python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py

@@ -140,7 +140,6 @@ class ModelRunnerKVCacheMixin:
         if self.mambaish_config is not None:
             rest_memory = self.handle_max_mamba_cache(rest_memory)
 
-        logger.info(f"The available memory for KV cache is {rest_memory:.2f} GB.")
         return int(rest_memory * (1 << 30)) // cell_size
 
     def handle_max_mamba_cache(self: ModelRunner, total_rest_memory):

python/sglang/srt/server_args.py

@@ -1540,7 +1540,7 @@ class ServerArgs:
                 else:
                     self.attention_backend = "triton"
 
-            logger.warning(
+            logger.info(
                 f"Attention backend not specified. Use {self.attention_backend} backend by default."
             )
 
@@ -1981,19 +1981,20 @@ class ServerArgs:
             ):
                 self.disable_overlap_schedule = False
                 logger.warning(
-                    "Beta spec is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on."
+                    "Spec v2 is enabled for eagle/eagle3 speculative decoding and overlap schedule is turned on."
                 )
                 if (
                     self.speculative_eagle_topk is not None
                     and self.speculative_eagle_topk > 1
                 ):
                     raise ValueError(
-                        "Beta spec currently only supports topk = 1 for speculative decoding."
+                        "Spec v2 currently only supports topk = 1 for speculative decoding."
                     )
             else:
                 self.disable_overlap_schedule = True
                 logger.warning(
-                    "Overlap scheduler is disabled when beta spec is off or using unsupported speculative algorithm."
+                    "Overlap scheduler is disabled when spec v2 is off or using unsupported speculative algorithm. "
+                    "You can set env SGLANG_ENABLE_SPEC_V2=True to enable the experimental overlap scheduler. "
                 )
 
             if self.enable_mixed_chunk:

python/sglang/srt/utils/hf_transformers_utils.py

@@ -125,7 +125,7 @@ def get_hf_text_config(config: PretrainedConfig):
             # read the wrong values from the unused default text_config.
             # NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as
             # `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`.
-            setattr(config, "torch_dtype", torch.float16)
+            setattr(config, "dtype", torch.float16)
             return config
 
     if hasattr(config, "text_config"):

python/sglang/test/test_utils.py

@@ -683,6 +683,7 @@ def popen_launch_server(
                 response = session.get(
                     f"{base_url}/health_generate",
                     headers=headers,
+                    timeout=5,
                 )
                 if response.status_code == 200:
                     return process

test/srt/models/test_nvidia_nemotron_nano_v2_vl.py

@@ -15,7 +15,7 @@ class TestNvidiaNemotronNanoV2VLTextOnly(GSM8KMixin, DefaultServerBase):
 
 
 class TestNvidiaNemotronNanoV2VLMMMU(MMMUVLMTestBase):
-    accuracy = 0.454
+    accuracy = 0.444
     model = MODEL
     other_args = ["--max-mamba-cache-size", "128", "--trust-remote-code"]
     mmmu_args = ["--limit=0.1"]

test/srt/test_vision_chunked_prefill.py

@@ -199,6 +199,7 @@ class TestVisionChunkedPrefill(CustomTestCase):
             logger.info(f"Killing chunked server pid={chunked_server_pid}")
             kill_process_tree(chunked_server_pid)
             logger.info("Chunked server killed")
+            time.sleep(4)
 
         # None-chunked
         logger.info("Phase 2: Testing with chunked_prefill_size=-1 (no chunking)")
@@ -218,6 +219,7 @@ class TestVisionChunkedPrefill(CustomTestCase):
             logger.info(f"Killing non-chunked server pid={no_chunked_server_pid}")
             kill_process_tree(no_chunked_server_pid)
             logger.info("Non-chunked server killed")
+            time.sleep(4)
 
         for output_chunked, output_no_chunked in zip(
             outputs_chunked, outputs_no_chunked