[Perf] diffusion: Defer full GC collections while importing the runtime (#44606)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Richard Wang
2026-10-18 15:31:10 +08:00
committed by GitHub
co-authored by Mick Qian
parent f1cf2e1005
commit de61ab53d4
4 changed files with 213 additions and 67 deletions
+47 -42
View File
@@ -52,6 +52,7 @@ from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
maybe_download_model_index,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.runtime.utils.process import defer_full_gc_for_imports
logger = init_logger(__name__)
@@ -76,51 +77,54 @@ def _discover_and_register_pipelines():
if _BUILTIN_PIPELINES_DISCOVERED:
return
package_name = "sglang.multimodal_gen.runtime.pipelines"
package = importlib.import_module(package_name)
_BUILTIN_PIPELINES_DISCOVERED = True
with defer_full_gc_for_imports():
package_name = "sglang.multimodal_gen.runtime.pipelines"
package = importlib.import_module(package_name)
_BUILTIN_PIPELINES_DISCOVERED = True
for _, module_name, ispkg in pkgutil.walk_packages(
package.__path__, package.__name__ + "."
):
if not ispkg:
try:
pipeline_module = importlib.import_module(module_name)
except Exception as exc:
logger.warning(
"Skipping pipeline module %s during discovery due to import failure: %s",
module_name,
exc,
)
logger.debug(
"Pipeline import failure details for %s", module_name, exc_info=True
)
continue
if hasattr(pipeline_module, "EntryClass"):
entry_cls = pipeline_module.EntryClass
entry_cls_list = (
[entry_cls] if not isinstance(entry_cls, list) else entry_cls
)
for _, module_name, ispkg in pkgutil.walk_packages(
package.__path__, package.__name__ + "."
):
if not ispkg:
try:
pipeline_module = importlib.import_module(module_name)
except Exception as exc:
logger.warning(
"Skipping pipeline module %s during discovery due to import failure: %s",
module_name,
exc,
)
logger.debug(
"Pipeline import failure details for %s",
module_name,
exc_info=True,
)
continue
if hasattr(pipeline_module, "EntryClass"):
entry_cls = pipeline_module.EntryClass
entry_cls_list = (
[entry_cls] if not isinstance(entry_cls, list) else entry_cls
)
for cls in entry_cls_list:
if not issubclass(cls, ComposedPipelineBase):
continue
if cls.pipeline_name in _PIPELINE_REGISTRY:
logger.warning(
f"Duplicate pipeline name '{cls.pipeline_name}' found. Overwriting."
)
_PIPELINE_REGISTRY[cls.pipeline_name] = cls
for cls in entry_cls_list:
if not issubclass(cls, ComposedPipelineBase):
continue
if cls.pipeline_name in _PIPELINE_REGISTRY:
logger.warning(
f"Duplicate pipeline name '{cls.pipeline_name}' found. Overwriting."
)
_PIPELINE_REGISTRY[cls.pipeline_name] = cls
# Special handling for ComfyUI Pipelines:
# Auto-register config classes if Pipeline class has them defined
# since comfyui get model from a single weight file, so we need to register the config classes here
if hasattr(cls, "pipeline_config_cls") and hasattr(
cls, "sampling_params_cls"
):
_PIPELINE_CONFIG_REGISTRY[cls.pipeline_name] = (
cls.pipeline_config_cls,
cls.sampling_params_cls,
)
# Special handling for ComfyUI Pipelines:
# Auto-register config classes if Pipeline class has them defined
# since comfyui get model from a single weight file, so we need to register the config classes here
if hasattr(cls, "pipeline_config_cls") and hasattr(
cls, "sampling_params_cls"
):
_PIPELINE_CONFIG_REGISTRY[cls.pipeline_name] = (
cls.pipeline_config_cls,
cls.sampling_params_cls,
)
logger.debug(
f"Registering pipelines complete, {len(_PIPELINE_REGISTRY)} pipelines registered"
)
@@ -305,6 +309,7 @@ _configs_discovered: bool = False
_CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video2", "sana_video")
@defer_full_gc_for_imports()
def _discover_and_register_configs() -> None:
global _configs_discovered
if _configs_discovered:
@@ -114,6 +114,7 @@ def bootstrap_scheduler_process(spec: SchedulerProcessSpec) -> None:
# Arm PDEATHSIG before any vendor code runs: everything below can block,
# and a child that hangs there would outlive a dead launcher.
from sglang.multimodal_gen.runtime.utils.process import (
defer_full_gc_for_imports,
kill_itself_when_parent_died,
)
@@ -123,26 +124,27 @@ def bootstrap_scheduler_process(spec: SchedulerProcessSpec) -> None:
if spec.rank == 0:
_warn_if_runtime_imported_early()
# Platform initialization is the first extensible runtime action. In
# particular it precedes plugin callbacks and hook target resolution, both
# of which may import arbitrary runtime modules.
from sglang.multimodal_gen.runtime.platforms import initialize_current_platform
with defer_full_gc_for_imports():
# Platform initialization is the first extensible runtime action. In
# particular it precedes plugin callbacks and hook target resolution, both
# of which may import arbitrary runtime modules.
from sglang.multimodal_gen.runtime.platforms import initialize_current_platform
initialize_current_platform()
initialize_current_platform()
from sglang.multimodal_gen.runtime.platforms.plugins import (
apply_plugin_hooks,
load_plugins,
)
from sglang.multimodal_gen.runtime.platforms.plugins import (
apply_plugin_hooks,
load_plugins,
)
load_plugins()
apply_plugin_hooks()
load_plugins()
apply_plugin_hooks()
server_args = spec.server_args.materialize()
server_args = spec.server_args.materialize()
# Resolve the function from its module after hook application. A ``from``
# binding created earlier would retain the unpatched callable.
from sglang.multimodal_gen.runtime.managers import gpu_worker
# Resolve the function from its module after hook application. A ``from``
# binding created earlier would retain the unpatched callable.
from sglang.multimodal_gen.runtime.managers import gpu_worker
gpu_worker.run_scheduler_process(
local_rank=spec.local_rank,
@@ -154,6 +156,7 @@ def bootstrap_scheduler_process(spec: SchedulerProcessSpec) -> None:
def bootstrap_http_server_process(server_args: ServerArgsPayload) -> None:
from sglang.multimodal_gen.runtime.utils.process import (
defer_full_gc_for_imports,
kill_itself_when_parent_died,
)
@@ -161,16 +164,19 @@ def bootstrap_http_server_process(server_args: ServerArgsPayload) -> None:
_warn_if_runtime_imported_early()
# No initialize_current_platform() here: this child serves HTTP and never
# touches the device, so it has no reason to bring up a vendor backend.
from sglang.multimodal_gen.runtime.platforms.plugins import (
apply_plugin_hooks,
load_plugins,
)
with defer_full_gc_for_imports():
# No initialize_current_platform() here: this child serves HTTP and never
# touches the device, so it has no reason to bring up a vendor backend.
from sglang.multimodal_gen.runtime.platforms.plugins import (
apply_plugin_hooks,
load_plugins,
)
load_plugins()
apply_plugin_hooks()
load_plugins()
apply_plugin_hooks()
from sglang.multimodal_gen.runtime import launch_server
from sglang.multimodal_gen.runtime import launch_server
launch_server.launch_http_server_only(server_args.materialize())
materialized = server_args.materialize()
launch_server.launch_http_server_only(materialized)
@@ -1,11 +1,14 @@
# SPDX-License-Identifier: Apache-2.0
# Adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/utils.py
import contextlib
import ctypes
import gc
import os
import signal
import sys
import threading
from collections.abc import Iterator
import psutil
@@ -30,6 +33,61 @@ def kill_itself_when_parent_died() -> None:
os.kill(os.getpid(), signal.SIGKILL)
_full_gc_lock = threading.Lock()
_full_gc_deferrals = 0
_gc_thresholds = gc.get_threshold()
# The largest third threshold the collector accepts, a C int, so never reached.
_NO_FULL_GC = 2**31 - 1
@contextlib.contextmanager
def defer_full_gc_for_imports() -> Iterator[None]:
"""Run no automatic full garbage collection while a process imports its runtime.
Imports create long-lived modules, classes and functions and little garbage,
yet each full collection traces the whole growing heap, and how many of them
fall inside an import depends on the import order rather than on the work
done. Young collections still run, so short-lived cyclic garbage is freed as
usual. Restoring the entry thresholds lets automatic full collections resume
after the block; nested or concurrent blocks restore them when the last one
exits.
"""
global _full_gc_deferrals, _gc_thresholds
# Python 3.14.0-3.14.4 uses incremental GC and ignores threshold2
if (3, 14) <= sys.version_info < (3, 14, 5):
yield
return
pid = os.getpid()
with _full_gc_lock:
if _full_gc_deferrals == 0:
_gc_thresholds = gc.get_threshold()
gc.set_threshold(*_gc_thresholds[:2], _NO_FULL_GC)
_full_gc_deferrals += 1
try:
yield
finally:
# A child forked inside the block restored the thresholds at fork.
if os.getpid() == pid:
with _full_gc_lock:
_full_gc_deferrals -= 1
if _full_gc_deferrals == 0:
gc.set_threshold(*_gc_thresholds)
def _restore_gc_thresholds_in_forked_child() -> None:
# The parent restores the thresholds in the parent only, so a child forked
# inside the block would otherwise never run a full collection.
global _full_gc_lock, _full_gc_deferrals
_full_gc_lock = threading.Lock()
if _full_gc_deferrals:
_full_gc_deferrals = 0
gc.set_threshold(*_gc_thresholds)
os.register_at_fork(after_in_child=_restore_gc_thresholds_in_forked_child)
def kill_process_tree(
parent_pid,
include_parent: bool = True,
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
import contextlib
import gc
import importlib.util
import os
import subprocess
@@ -7,6 +9,7 @@ import sys
import textwrap
import unittest
from pathlib import Path
from unittest.mock import patch
import psutil
@@ -16,6 +19,7 @@ assert SPEC is not None and SPEC.loader is not None
PROCESS = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(PROCESS)
kill_process_tree = PROCESS.kill_process_tree
defer_full_gc_for_imports = PROCESS.defer_full_gc_for_imports
@unittest.skipUnless(os.name == "posix", "requires POSIX process semantics")
@@ -91,5 +95,78 @@ class TestKillProcessTree(unittest.TestCase):
kill_process_tree(0x7FFFFFFF)
class TestDeferFullGcForImports(unittest.TestCase):
def full_collections_while_allocating(self, block):
gc.collect()
threshold0, threshold1, threshold2 = gc.get_threshold()
# Enough survivors that the collector's own rule makes a full one due.
count = len(gc.get_objects()) + 2 * threshold0 * threshold1 * threshold2
before = gc.get_stats()[2]["collections"]
with block():
kept = [[] for _ in range(count)]
full = gc.get_stats()[2]["collections"] - before
del kept
return full
@unittest.skipIf(
(3, 14) <= sys.version_info < (3, 14, 5),
"incremental GC does not use threshold2",
)
def test_no_full_collection_inside_and_thresholds_restored(self):
thresholds = gc.get_threshold()
self.assertGreater(
self.full_collections_while_allocating(contextlib.nullcontext), 0
)
self.assertEqual(
self.full_collections_while_allocating(defer_full_gc_for_imports), 0
)
with self.assertRaises(ImportError):
with defer_full_gc_for_imports():
raise ImportError
self.assertEqual(gc.get_threshold(), thresholds)
def test_incremental_gc_is_unchanged(self):
thresholds = gc.get_threshold()
for version in ((3, 14, 0), (3, 14, 4)):
with (
self.subTest(version=version),
patch.object(PROCESS.sys, "version_info", version),
):
with defer_full_gc_for_imports():
self.assertEqual(gc.get_threshold(), thresholds)
self.assertEqual(gc.get_threshold(), thresholds)
def test_nested_exception_preserves_disabled_collector(self):
thresholds = gc.get_threshold()
self.addCleanup(gc.enable if gc.isenabled() else gc.disable)
self.addCleanup(gc.set_threshold, *thresholds)
gc.disable()
with defer_full_gc_for_imports():
deferred = gc.get_threshold()
with self.assertRaises(ImportError):
with defer_full_gc_for_imports():
raise ImportError
self.assertEqual(gc.get_threshold(), deferred)
self.assertFalse(gc.isenabled())
self.assertEqual(gc.get_threshold(), thresholds)
self.assertFalse(gc.isenabled())
@unittest.skipUnless(os.name == "posix", "requires fork")
def test_fork_restores_child_without_changing_parent(self):
thresholds = gc.get_threshold()
with defer_full_gc_for_imports():
deferred = gc.get_threshold()
pid = os.fork()
if pid == 0:
restored = gc.get_threshold() == thresholds
if pid == 0:
with defer_full_gc_for_imports():
restored &= gc.get_threshold() == deferred
os._exit(int(not (restored and gc.get_threshold() == thresholds)))
self.assertEqual(os.waitstatus_to_exitcode(os.waitpid(pid, 0)[1]), 0)
self.assertEqual(gc.get_threshold(), thresholds)
if __name__ == "__main__":
unittest.main()