From de61ab53d47f406aadcd8771bf829bd66d2d1c84 Mon Sep 17 00:00:00 2001 From: Richard Wang Date: Sun, 18 Oct 2026 00:31:10 -0700 Subject: [PATCH] [Perf] diffusion: Defer full GC collections while importing the runtime (#44606) Co-authored-by: Mick Qian --- python/sglang/multimodal_gen/registry.py | 89 ++++++++++--------- .../runtime/managers/worker_bootstrap.py | 56 ++++++------ .../multimodal_gen/runtime/utils/process.py | 58 ++++++++++++ .../multimodal_gen/test/unit/test_process.py | 77 ++++++++++++++++ 4 files changed, 213 insertions(+), 67 deletions(-) diff --git a/python/sglang/multimodal_gen/registry.py b/python/sglang/multimodal_gen/registry.py index d608946238..c06eadb178 100644 --- a/python/sglang/multimodal_gen/registry.py +++ b/python/sglang/multimodal_gen/registry.py @@ -52,6 +52,7 @@ from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import ( maybe_download_model_index, ) from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger +from sglang.multimodal_gen.runtime.utils.process import defer_full_gc_for_imports logger = init_logger(__name__) @@ -76,51 +77,54 @@ def _discover_and_register_pipelines(): if _BUILTIN_PIPELINES_DISCOVERED: return - package_name = "sglang.multimodal_gen.runtime.pipelines" - package = importlib.import_module(package_name) - _BUILTIN_PIPELINES_DISCOVERED = True + with defer_full_gc_for_imports(): + package_name = "sglang.multimodal_gen.runtime.pipelines" + package = importlib.import_module(package_name) + _BUILTIN_PIPELINES_DISCOVERED = True - for _, module_name, ispkg in pkgutil.walk_packages( - package.__path__, package.__name__ + "." - ): - if not ispkg: - try: - pipeline_module = importlib.import_module(module_name) - except Exception as exc: - logger.warning( - "Skipping pipeline module %s during discovery due to import failure: %s", - module_name, - exc, - ) - logger.debug( - "Pipeline import failure details for %s", module_name, exc_info=True - ) - continue - if hasattr(pipeline_module, "EntryClass"): - entry_cls = pipeline_module.EntryClass - entry_cls_list = ( - [entry_cls] if not isinstance(entry_cls, list) else entry_cls - ) + for _, module_name, ispkg in pkgutil.walk_packages( + package.__path__, package.__name__ + "." + ): + if not ispkg: + try: + pipeline_module = importlib.import_module(module_name) + except Exception as exc: + logger.warning( + "Skipping pipeline module %s during discovery due to import failure: %s", + module_name, + exc, + ) + logger.debug( + "Pipeline import failure details for %s", + module_name, + exc_info=True, + ) + continue + if hasattr(pipeline_module, "EntryClass"): + entry_cls = pipeline_module.EntryClass + entry_cls_list = ( + [entry_cls] if not isinstance(entry_cls, list) else entry_cls + ) - for cls in entry_cls_list: - if not issubclass(cls, ComposedPipelineBase): - continue - if cls.pipeline_name in _PIPELINE_REGISTRY: - logger.warning( - f"Duplicate pipeline name '{cls.pipeline_name}' found. Overwriting." - ) - _PIPELINE_REGISTRY[cls.pipeline_name] = cls + for cls in entry_cls_list: + if not issubclass(cls, ComposedPipelineBase): + continue + if cls.pipeline_name in _PIPELINE_REGISTRY: + logger.warning( + f"Duplicate pipeline name '{cls.pipeline_name}' found. Overwriting." + ) + _PIPELINE_REGISTRY[cls.pipeline_name] = cls - # Special handling for ComfyUI Pipelines: - # Auto-register config classes if Pipeline class has them defined - # since comfyui get model from a single weight file, so we need to register the config classes here - if hasattr(cls, "pipeline_config_cls") and hasattr( - cls, "sampling_params_cls" - ): - _PIPELINE_CONFIG_REGISTRY[cls.pipeline_name] = ( - cls.pipeline_config_cls, - cls.sampling_params_cls, - ) + # Special handling for ComfyUI Pipelines: + # Auto-register config classes if Pipeline class has them defined + # since comfyui get model from a single weight file, so we need to register the config classes here + if hasattr(cls, "pipeline_config_cls") and hasattr( + cls, "sampling_params_cls" + ): + _PIPELINE_CONFIG_REGISTRY[cls.pipeline_name] = ( + cls.pipeline_config_cls, + cls.sampling_params_cls, + ) logger.debug( f"Registering pipelines complete, {len(_PIPELINE_REGISTRY)} pipelines registered" ) @@ -305,6 +309,7 @@ _configs_discovered: bool = False _CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video2", "sana_video") +@defer_full_gc_for_imports() def _discover_and_register_configs() -> None: global _configs_discovered if _configs_discovered: diff --git a/python/sglang/multimodal_gen/runtime/managers/worker_bootstrap.py b/python/sglang/multimodal_gen/runtime/managers/worker_bootstrap.py index 0c59a60ace..57132e55c9 100644 --- a/python/sglang/multimodal_gen/runtime/managers/worker_bootstrap.py +++ b/python/sglang/multimodal_gen/runtime/managers/worker_bootstrap.py @@ -114,6 +114,7 @@ def bootstrap_scheduler_process(spec: SchedulerProcessSpec) -> None: # Arm PDEATHSIG before any vendor code runs: everything below can block, # and a child that hangs there would outlive a dead launcher. from sglang.multimodal_gen.runtime.utils.process import ( + defer_full_gc_for_imports, kill_itself_when_parent_died, ) @@ -123,26 +124,27 @@ def bootstrap_scheduler_process(spec: SchedulerProcessSpec) -> None: if spec.rank == 0: _warn_if_runtime_imported_early() - # Platform initialization is the first extensible runtime action. In - # particular it precedes plugin callbacks and hook target resolution, both - # of which may import arbitrary runtime modules. - from sglang.multimodal_gen.runtime.platforms import initialize_current_platform + with defer_full_gc_for_imports(): + # Platform initialization is the first extensible runtime action. In + # particular it precedes plugin callbacks and hook target resolution, both + # of which may import arbitrary runtime modules. + from sglang.multimodal_gen.runtime.platforms import initialize_current_platform - initialize_current_platform() + initialize_current_platform() - from sglang.multimodal_gen.runtime.platforms.plugins import ( - apply_plugin_hooks, - load_plugins, - ) + from sglang.multimodal_gen.runtime.platforms.plugins import ( + apply_plugin_hooks, + load_plugins, + ) - load_plugins() - apply_plugin_hooks() + load_plugins() + apply_plugin_hooks() - server_args = spec.server_args.materialize() + server_args = spec.server_args.materialize() - # Resolve the function from its module after hook application. A ``from`` - # binding created earlier would retain the unpatched callable. - from sglang.multimodal_gen.runtime.managers import gpu_worker + # Resolve the function from its module after hook application. A ``from`` + # binding created earlier would retain the unpatched callable. + from sglang.multimodal_gen.runtime.managers import gpu_worker gpu_worker.run_scheduler_process( local_rank=spec.local_rank, @@ -154,6 +156,7 @@ def bootstrap_scheduler_process(spec: SchedulerProcessSpec) -> None: def bootstrap_http_server_process(server_args: ServerArgsPayload) -> None: from sglang.multimodal_gen.runtime.utils.process import ( + defer_full_gc_for_imports, kill_itself_when_parent_died, ) @@ -161,16 +164,19 @@ def bootstrap_http_server_process(server_args: ServerArgsPayload) -> None: _warn_if_runtime_imported_early() - # No initialize_current_platform() here: this child serves HTTP and never - # touches the device, so it has no reason to bring up a vendor backend. - from sglang.multimodal_gen.runtime.platforms.plugins import ( - apply_plugin_hooks, - load_plugins, - ) + with defer_full_gc_for_imports(): + # No initialize_current_platform() here: this child serves HTTP and never + # touches the device, so it has no reason to bring up a vendor backend. + from sglang.multimodal_gen.runtime.platforms.plugins import ( + apply_plugin_hooks, + load_plugins, + ) - load_plugins() - apply_plugin_hooks() + load_plugins() + apply_plugin_hooks() - from sglang.multimodal_gen.runtime import launch_server + from sglang.multimodal_gen.runtime import launch_server - launch_server.launch_http_server_only(server_args.materialize()) + materialized = server_args.materialize() + + launch_server.launch_http_server_only(materialized) diff --git a/python/sglang/multimodal_gen/runtime/utils/process.py b/python/sglang/multimodal_gen/runtime/utils/process.py index 8ec8ba02ff..2a85e4a350 100644 --- a/python/sglang/multimodal_gen/runtime/utils/process.py +++ b/python/sglang/multimodal_gen/runtime/utils/process.py @@ -1,11 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # Adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/utils.py +import contextlib import ctypes +import gc import os import signal import sys import threading +from collections.abc import Iterator import psutil @@ -30,6 +33,61 @@ def kill_itself_when_parent_died() -> None: os.kill(os.getpid(), signal.SIGKILL) +_full_gc_lock = threading.Lock() +_full_gc_deferrals = 0 +_gc_thresholds = gc.get_threshold() +# The largest third threshold the collector accepts, a C int, so never reached. +_NO_FULL_GC = 2**31 - 1 + + +@contextlib.contextmanager +def defer_full_gc_for_imports() -> Iterator[None]: + """Run no automatic full garbage collection while a process imports its runtime. + + Imports create long-lived modules, classes and functions and little garbage, + yet each full collection traces the whole growing heap, and how many of them + fall inside an import depends on the import order rather than on the work + done. Young collections still run, so short-lived cyclic garbage is freed as + usual. Restoring the entry thresholds lets automatic full collections resume + after the block; nested or concurrent blocks restore them when the last one + exits. + """ + global _full_gc_deferrals, _gc_thresholds + # Python 3.14.0-3.14.4 uses incremental GC and ignores threshold2 + if (3, 14) <= sys.version_info < (3, 14, 5): + yield + return + + pid = os.getpid() + with _full_gc_lock: + if _full_gc_deferrals == 0: + _gc_thresholds = gc.get_threshold() + gc.set_threshold(*_gc_thresholds[:2], _NO_FULL_GC) + _full_gc_deferrals += 1 + try: + yield + finally: + # A child forked inside the block restored the thresholds at fork. + if os.getpid() == pid: + with _full_gc_lock: + _full_gc_deferrals -= 1 + if _full_gc_deferrals == 0: + gc.set_threshold(*_gc_thresholds) + + +def _restore_gc_thresholds_in_forked_child() -> None: + # The parent restores the thresholds in the parent only, so a child forked + # inside the block would otherwise never run a full collection. + global _full_gc_lock, _full_gc_deferrals + _full_gc_lock = threading.Lock() + if _full_gc_deferrals: + _full_gc_deferrals = 0 + gc.set_threshold(*_gc_thresholds) + + +os.register_at_fork(after_in_child=_restore_gc_thresholds_in_forked_child) + + def kill_process_tree( parent_pid, include_parent: bool = True, diff --git a/python/sglang/multimodal_gen/test/unit/test_process.py b/python/sglang/multimodal_gen/test/unit/test_process.py index 5e2b1224b3..c47c5e7ef8 100644 --- a/python/sglang/multimodal_gen/test/unit/test_process.py +++ b/python/sglang/multimodal_gen/test/unit/test_process.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 +import contextlib +import gc import importlib.util import os import subprocess @@ -7,6 +9,7 @@ import sys import textwrap import unittest from pathlib import Path +from unittest.mock import patch import psutil @@ -16,6 +19,7 @@ assert SPEC is not None and SPEC.loader is not None PROCESS = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(PROCESS) kill_process_tree = PROCESS.kill_process_tree +defer_full_gc_for_imports = PROCESS.defer_full_gc_for_imports @unittest.skipUnless(os.name == "posix", "requires POSIX process semantics") @@ -91,5 +95,78 @@ class TestKillProcessTree(unittest.TestCase): kill_process_tree(0x7FFFFFFF) +class TestDeferFullGcForImports(unittest.TestCase): + def full_collections_while_allocating(self, block): + gc.collect() + threshold0, threshold1, threshold2 = gc.get_threshold() + # Enough survivors that the collector's own rule makes a full one due. + count = len(gc.get_objects()) + 2 * threshold0 * threshold1 * threshold2 + before = gc.get_stats()[2]["collections"] + with block(): + kept = [[] for _ in range(count)] + full = gc.get_stats()[2]["collections"] - before + del kept + return full + + @unittest.skipIf( + (3, 14) <= sys.version_info < (3, 14, 5), + "incremental GC does not use threshold2", + ) + def test_no_full_collection_inside_and_thresholds_restored(self): + thresholds = gc.get_threshold() + + self.assertGreater( + self.full_collections_while_allocating(contextlib.nullcontext), 0 + ) + self.assertEqual( + self.full_collections_while_allocating(defer_full_gc_for_imports), 0 + ) + with self.assertRaises(ImportError): + with defer_full_gc_for_imports(): + raise ImportError + self.assertEqual(gc.get_threshold(), thresholds) + + def test_incremental_gc_is_unchanged(self): + thresholds = gc.get_threshold() + for version in ((3, 14, 0), (3, 14, 4)): + with ( + self.subTest(version=version), + patch.object(PROCESS.sys, "version_info", version), + ): + with defer_full_gc_for_imports(): + self.assertEqual(gc.get_threshold(), thresholds) + self.assertEqual(gc.get_threshold(), thresholds) + + def test_nested_exception_preserves_disabled_collector(self): + thresholds = gc.get_threshold() + self.addCleanup(gc.enable if gc.isenabled() else gc.disable) + self.addCleanup(gc.set_threshold, *thresholds) + gc.disable() + with defer_full_gc_for_imports(): + deferred = gc.get_threshold() + with self.assertRaises(ImportError): + with defer_full_gc_for_imports(): + raise ImportError + self.assertEqual(gc.get_threshold(), deferred) + self.assertFalse(gc.isenabled()) + self.assertEqual(gc.get_threshold(), thresholds) + self.assertFalse(gc.isenabled()) + + @unittest.skipUnless(os.name == "posix", "requires fork") + def test_fork_restores_child_without_changing_parent(self): + thresholds = gc.get_threshold() + with defer_full_gc_for_imports(): + deferred = gc.get_threshold() + pid = os.fork() + if pid == 0: + restored = gc.get_threshold() == thresholds + if pid == 0: + with defer_full_gc_for_imports(): + restored &= gc.get_threshold() == deferred + os._exit(int(not (restored and gc.get_threshold() == thresholds))) + self.assertEqual(os.waitstatus_to_exitcode(os.waitpid(pid, 0)[1]), 0) + self.assertEqual(gc.get_threshold(), thresholds) + + if __name__ == "__main__": unittest.main()