From 16d3b118a20e70f7bb93b57c9e6d532d6a6fece6 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Tue, 4 Aug 2026 02:19:56 -0700 Subject: [PATCH] Reduce startup log noise and fix Dynamo / CUDA-graph edge cases (#33428) --- python/sglang/kernels/jit/utils/compile.py | 6 +++ python/sglang/srt/arg_groups/overrides.py | 6 +++ python/sglang/srt/configs/model_config.py | 14 ------- .../srt/layers/moe/fused_moe_triton/layer.py | 20 +++++++--- .../breakable_cuda_graph_backend.py | 2 +- python/sglang/srt/server_args.py | 40 ++++--------------- .../srt/utils/hf_transformers/common.py | 13 +++--- test/registered/unit/test_model_overrides.py | 4 +- 8 files changed, 45 insertions(+), 60 deletions(-) diff --git a/python/sglang/kernels/jit/utils/compile.py b/python/sglang/kernels/jit/utils/compile.py index b045e731c..284ae3318 100644 --- a/python/sglang/kernels/jit/utils/compile.py +++ b/python/sglang/kernels/jit/utils/compile.py @@ -169,6 +169,12 @@ def _jit_build_dir_name(module_name: str) -> str: return f"{module_name}__arch_{arch}__tvmffi_{_tvm_ffi_version()}" +# JIT compilation is pure Python/filesystem plumbing (path `.resolve()` calls +# `os.lstat`, etc.) that Dynamo cannot trace. When a lazily-loaded kernel is +# first reached from inside a `@torch.compile`d region, tracing into it produces +# spurious "Dynamo does not know how to trace the builtin `posix.lstat`" graph +# breaks. The load happens once and is memoized, so keep it out of the graph. +@torch.compiler.disable def load_jit( *args: str, cpp_files: List[str] | None = None, diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index f1290b9c6..3ab05bcc4 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -2235,6 +2235,12 @@ def _a2a_backend_overrides(view: Any) -> dict: @register_post_process def _a2a_ep_size(view: Any) -> dict: if view.moe_a2a_backend in _A2A_EP_SPANNING_BACKENDS: + if view.ep_size != view.tp_size: + logger.info( + f"{view.moe_a2a_backend} MoE is enabled. The expert parallel size " + f"is adjusted from {view.ep_size} to the tensor parallel size " + f"[{view.tp_size}]." + ) return {"ep_size": view.tp_size} return {} diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 96daa3490..97719a069 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -1503,20 +1503,6 @@ class ModelConfig: f"({self.quantization})." ) - # Warn if DeepGemm is enabled for a non-ue8m0 checkpoint on Blackwell. - # MXFP8 stores E8M0 block scales that DeepGemm consumes losslessly, so skip the warning there. - self.use_scale_ue8m0 = quant_cfg.get("scale_fmt", None) == "ue8m0" - from sglang.srt.layers import deep_gemm_wrapper - - if ( - not self.use_scale_ue8m0 - and deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 - and self.quantization != "mxfp8" - ): - logger.warning( - "DeepGemm is enabled but the scale_fmt of checkpoint is not ue8m0. This might cause accuracy degradation on Blackwell." - ) - if self.quantization is not None: if self.quantization not in supported_quantization: raise ValueError( diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 8755afc3a..ee82f29bc 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -93,6 +93,11 @@ _is_cpu = is_cpu() _is_npu = is_npu() _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip +# Log the deferred-finalize config at most once per process (rank). Different MoE +# layers can resolve to different quant methods, so print_info_once (keyed on the +# full message) would otherwise fire once per distinct quant method. +_deferred_finalize_info_logged = False + def _copy_weight_view_before_h2d(loaded_weight: torch.Tensor) -> torch.Tensor: """Copy a CPU tensor view into independent contiguous storage.""" @@ -379,12 +384,15 @@ class FusedMoE(torch.nn.Module): and get_moe_runner_backend().is_flashinfer_trtllm() and isinstance(self.quant_method, ModelOptNvFp4FusedMoEMethod) ) - print_info_once( - "FlashInfer TRTLLM MoE deferred finalize is " - f"{'enabled' if self.supports_deferred_finalize else 'disabled'} " - f"(moe_runner_backend={get_exec().moe.moe_runner_backend}, " - f"quant_method={type(self.quant_method).__name__})." - ) + global _deferred_finalize_info_logged + if not _deferred_finalize_info_logged: + _deferred_finalize_info_logged = True + logging.getLogger(__name__).info( + "FlashInfer TRTLLM MoE deferred finalize is " + f"{'enabled' if self.supports_deferred_finalize else 'disabled'} " + f"(moe_runner_backend={get_exec().moe.moe_runner_backend}, " + f"quant_method={type(self.quant_method).__name__})." + ) self.quant_method.create_weights( layer=self, diff --git a/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py b/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py index 55f8eabe2..c4fc86532 100644 --- a/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py +++ b/python/sglang/srt/model_executor/runner_backend/breakable_cuda_graph_backend.py @@ -119,7 +119,7 @@ class BreakableCudaGraphBackend(DedupedCudaGraphMixin, BaseCudaGraphBackend): if post_warmup_hook is not None: post_warmup_hook() - graph = BreakableCUDAGraph() + graph = BreakableCUDAGraph(self.deduped_cuda_graph) captured_fn = ( eager_on_graph(True)(forward_fn) if self._debug_eager else forward_fn ) diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 9c2f3895b..b882cc7fc 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -4785,9 +4785,15 @@ class ServerArgs: ) # Multimodal models need more memory for the image processing, - # so we adjust the mem_fraction_static accordingly. + # so we adjust the mem_fraction_static accordingly. The VLM encoder + # only runs on the prefill stage, so PD decode engines do not need + # this headroom; prefill engines and normal (non-PD) engines do. model_config = self.get_model_config() - if model_config.is_multimodal and not self.language_only: + if ( + model_config.is_multimodal + and not self.language_only + and self.disaggregation_mode != "decode" + ): self.adjust_mem_fraction_for_vlm(model_config) # If symm mem is enabled and prealloc size is not set, set it to 4GB @@ -6611,10 +6617,6 @@ class ServerArgs: if a2a_backend == "megamoe": if not envs.SGLANG_OPT_FIX_MEGA_MOE_MEMORY.is_set(): envs.SGLANG_OPT_FIX_MEGA_MOE_MEMORY.set(True) - logger.info( - f"Mega MoE is enabled. The expert parallel size is adjusted " - f"to be the same as the tensor parallel size[{self.tp_size}]." - ) if a2a_backend == "deepep": if self.moe_runner_backend == "flashinfer_cutedsl": @@ -6637,19 +6639,6 @@ class ServerArgs: logger.warning("Cuda graph is disabled because deepep_mode=`normal`") self.cuda_graph_config.decode.backend = Backend.DISABLED self.cuda_graph_config.prefill.backend = Backend.DISABLED - logger.warning( - f"DeepEP MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]." - ) - - if a2a_backend == "mooncake": - logger.warning( - f"Mooncake MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]." - ) - - if a2a_backend == "nixl": - logger.warning( - f"Nixl MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]." - ) if ( self.moe_a2a_backend == "none" and is_npu() @@ -6657,17 +6646,10 @@ class ServerArgs: # FIXME (OrangeRedeng): for some reasons if pass "ascend_tp" accuracy drops to zero self.moe_a2a_backend = "none" - if self.moe_a2a_backend == "ascend_fuseep": - logger.warning( - f"Ascend fused EP MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]." - ) if self.moe_a2a_backend == "flashinfer": assert ( resolved_view(self).enable_dp_attention and self.dp_size == self.tp_size ), "Flashinfer MoE A2A is only supported with dp_size == tp_size and --enable-dp-attention" - logger.warning( - f"Flashinfer MoE A2A is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]." - ) if self.deepep_mode != "auto": logger.warning("--deepep-mode is ignored for Flashinfer MoE A2A") if not envs.SGLANG_MOE_NVFP4_DISPATCH.is_set() and ( @@ -6688,9 +6670,6 @@ class ServerArgs: if self.deepep_mode == "auto": self.deepep_mode = "normal" logger.warning("auto set deepep_mode=`normal` for MORI EP") - logger.warning( - f"MoRI MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]." - ) # Check chunked prefill for mori # Skip validation if chunked prefill is disabled (i.e., size <= 0). @@ -6731,9 +6710,6 @@ class ServerArgs: if self.moe_runner_backend == "auto": self.moe_runner_backend = "deep_gemm" logger.warning("auto set moe_runner_backend=`deep_gemm` for PPLX EP") - logger.warning( - f"PPLX MoE is enabled. The expert parallel size is adjusted to be the same as the tensor parallel size[{self.tp_size}]." - ) # Check per-rank dispatch tokens for pplx # Skip validation if chunked prefill is disabled (i.e., size <= 0) diff --git a/python/sglang/srt/utils/hf_transformers/common.py b/python/sglang/srt/utils/hf_transformers/common.py index b6763950b..cd84b2885 100644 --- a/python/sglang/srt/utils/hf_transformers/common.py +++ b/python/sglang/srt/utils/hf_transformers/common.py @@ -489,12 +489,13 @@ def get_generation_config( return GenerationConfig.from_pretrained( model, trust_remote_code=trust_remote_code, revision=revision, **kwargs ) - except FileNotFoundError: - return None - except OSError as e: - logger.warning( - "Failed to load generation config for %s: %s. " - "Proceeding without generation config.", + except (FileNotFoundError, OSError) as e: + # A missing generation_config.json is normal for many checkpoints and + # is surfaced by HF as a generic OSError (not FileNotFoundError). Treat + # it as benign — proceed without a generation config, at DEBUG level so + # normal startup logs stay quiet. + logger.debug( + "No generation config for %s: %s. Proceeding without it.", model, e, ) diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index b9e7c8e89..3c3e2630e 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -2010,7 +2010,9 @@ class TestGoldenModelOverrides(_IsolatedPublish): self.assertEqual( _a2a_ep_size( - ResolvedView(SimpleNamespace(moe_a2a_backend="deepep", tp_size=8)) + ResolvedView( + SimpleNamespace(moe_a2a_backend="deepep", ep_size=1, tp_size=8) + ) ), {"ep_size": 8}, )