[Perf] diffusion: Defer full GC collections while importing the runtime (#44606)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
co-authored by
Mick Qian
parent
f1cf2e1005
commit
de61ab53d4
@@ -52,6 +52,7 @@ from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
|
||||
maybe_download_model_index,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.runtime.utils.process import defer_full_gc_for_imports
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -76,51 +77,54 @@ def _discover_and_register_pipelines():
|
||||
if _BUILTIN_PIPELINES_DISCOVERED:
|
||||
return
|
||||
|
||||
package_name = "sglang.multimodal_gen.runtime.pipelines"
|
||||
package = importlib.import_module(package_name)
|
||||
_BUILTIN_PIPELINES_DISCOVERED = True
|
||||
with defer_full_gc_for_imports():
|
||||
package_name = "sglang.multimodal_gen.runtime.pipelines"
|
||||
package = importlib.import_module(package_name)
|
||||
_BUILTIN_PIPELINES_DISCOVERED = True
|
||||
|
||||
for _, module_name, ispkg in pkgutil.walk_packages(
|
||||
package.__path__, package.__name__ + "."
|
||||
):
|
||||
if not ispkg:
|
||||
try:
|
||||
pipeline_module = importlib.import_module(module_name)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Skipping pipeline module %s during discovery due to import failure: %s",
|
||||
module_name,
|
||||
exc,
|
||||
)
|
||||
logger.debug(
|
||||
"Pipeline import failure details for %s", module_name, exc_info=True
|
||||
)
|
||||
continue
|
||||
if hasattr(pipeline_module, "EntryClass"):
|
||||
entry_cls = pipeline_module.EntryClass
|
||||
entry_cls_list = (
|
||||
[entry_cls] if not isinstance(entry_cls, list) else entry_cls
|
||||
)
|
||||
for _, module_name, ispkg in pkgutil.walk_packages(
|
||||
package.__path__, package.__name__ + "."
|
||||
):
|
||||
if not ispkg:
|
||||
try:
|
||||
pipeline_module = importlib.import_module(module_name)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Skipping pipeline module %s during discovery due to import failure: %s",
|
||||
module_name,
|
||||
exc,
|
||||
)
|
||||
logger.debug(
|
||||
"Pipeline import failure details for %s",
|
||||
module_name,
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
if hasattr(pipeline_module, "EntryClass"):
|
||||
entry_cls = pipeline_module.EntryClass
|
||||
entry_cls_list = (
|
||||
[entry_cls] if not isinstance(entry_cls, list) else entry_cls
|
||||
)
|
||||
|
||||
for cls in entry_cls_list:
|
||||
if not issubclass(cls, ComposedPipelineBase):
|
||||
continue
|
||||
if cls.pipeline_name in _PIPELINE_REGISTRY:
|
||||
logger.warning(
|
||||
f"Duplicate pipeline name '{cls.pipeline_name}' found. Overwriting."
|
||||
)
|
||||
_PIPELINE_REGISTRY[cls.pipeline_name] = cls
|
||||
for cls in entry_cls_list:
|
||||
if not issubclass(cls, ComposedPipelineBase):
|
||||
continue
|
||||
if cls.pipeline_name in _PIPELINE_REGISTRY:
|
||||
logger.warning(
|
||||
f"Duplicate pipeline name '{cls.pipeline_name}' found. Overwriting."
|
||||
)
|
||||
_PIPELINE_REGISTRY[cls.pipeline_name] = cls
|
||||
|
||||
# Special handling for ComfyUI Pipelines:
|
||||
# Auto-register config classes if Pipeline class has them defined
|
||||
# since comfyui get model from a single weight file, so we need to register the config classes here
|
||||
if hasattr(cls, "pipeline_config_cls") and hasattr(
|
||||
cls, "sampling_params_cls"
|
||||
):
|
||||
_PIPELINE_CONFIG_REGISTRY[cls.pipeline_name] = (
|
||||
cls.pipeline_config_cls,
|
||||
cls.sampling_params_cls,
|
||||
)
|
||||
# Special handling for ComfyUI Pipelines:
|
||||
# Auto-register config classes if Pipeline class has them defined
|
||||
# since comfyui get model from a single weight file, so we need to register the config classes here
|
||||
if hasattr(cls, "pipeline_config_cls") and hasattr(
|
||||
cls, "sampling_params_cls"
|
||||
):
|
||||
_PIPELINE_CONFIG_REGISTRY[cls.pipeline_name] = (
|
||||
cls.pipeline_config_cls,
|
||||
cls.sampling_params_cls,
|
||||
)
|
||||
logger.debug(
|
||||
f"Registering pipelines complete, {len(_PIPELINE_REGISTRY)} pipelines registered"
|
||||
)
|
||||
@@ -305,6 +309,7 @@ _configs_discovered: bool = False
|
||||
_CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video2", "sana_video")
|
||||
|
||||
|
||||
@defer_full_gc_for_imports()
|
||||
def _discover_and_register_configs() -> None:
|
||||
global _configs_discovered
|
||||
if _configs_discovered:
|
||||
|
||||
@@ -114,6 +114,7 @@ def bootstrap_scheduler_process(spec: SchedulerProcessSpec) -> None:
|
||||
# Arm PDEATHSIG before any vendor code runs: everything below can block,
|
||||
# and a child that hangs there would outlive a dead launcher.
|
||||
from sglang.multimodal_gen.runtime.utils.process import (
|
||||
defer_full_gc_for_imports,
|
||||
kill_itself_when_parent_died,
|
||||
)
|
||||
|
||||
@@ -123,26 +124,27 @@ def bootstrap_scheduler_process(spec: SchedulerProcessSpec) -> None:
|
||||
if spec.rank == 0:
|
||||
_warn_if_runtime_imported_early()
|
||||
|
||||
# Platform initialization is the first extensible runtime action. In
|
||||
# particular it precedes plugin callbacks and hook target resolution, both
|
||||
# of which may import arbitrary runtime modules.
|
||||
from sglang.multimodal_gen.runtime.platforms import initialize_current_platform
|
||||
with defer_full_gc_for_imports():
|
||||
# Platform initialization is the first extensible runtime action. In
|
||||
# particular it precedes plugin callbacks and hook target resolution, both
|
||||
# of which may import arbitrary runtime modules.
|
||||
from sglang.multimodal_gen.runtime.platforms import initialize_current_platform
|
||||
|
||||
initialize_current_platform()
|
||||
initialize_current_platform()
|
||||
|
||||
from sglang.multimodal_gen.runtime.platforms.plugins import (
|
||||
apply_plugin_hooks,
|
||||
load_plugins,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms.plugins import (
|
||||
apply_plugin_hooks,
|
||||
load_plugins,
|
||||
)
|
||||
|
||||
load_plugins()
|
||||
apply_plugin_hooks()
|
||||
load_plugins()
|
||||
apply_plugin_hooks()
|
||||
|
||||
server_args = spec.server_args.materialize()
|
||||
server_args = spec.server_args.materialize()
|
||||
|
||||
# Resolve the function from its module after hook application. A ``from``
|
||||
# binding created earlier would retain the unpatched callable.
|
||||
from sglang.multimodal_gen.runtime.managers import gpu_worker
|
||||
# Resolve the function from its module after hook application. A ``from``
|
||||
# binding created earlier would retain the unpatched callable.
|
||||
from sglang.multimodal_gen.runtime.managers import gpu_worker
|
||||
|
||||
gpu_worker.run_scheduler_process(
|
||||
local_rank=spec.local_rank,
|
||||
@@ -154,6 +156,7 @@ def bootstrap_scheduler_process(spec: SchedulerProcessSpec) -> None:
|
||||
|
||||
def bootstrap_http_server_process(server_args: ServerArgsPayload) -> None:
|
||||
from sglang.multimodal_gen.runtime.utils.process import (
|
||||
defer_full_gc_for_imports,
|
||||
kill_itself_when_parent_died,
|
||||
)
|
||||
|
||||
@@ -161,16 +164,19 @@ def bootstrap_http_server_process(server_args: ServerArgsPayload) -> None:
|
||||
|
||||
_warn_if_runtime_imported_early()
|
||||
|
||||
# No initialize_current_platform() here: this child serves HTTP and never
|
||||
# touches the device, so it has no reason to bring up a vendor backend.
|
||||
from sglang.multimodal_gen.runtime.platforms.plugins import (
|
||||
apply_plugin_hooks,
|
||||
load_plugins,
|
||||
)
|
||||
with defer_full_gc_for_imports():
|
||||
# No initialize_current_platform() here: this child serves HTTP and never
|
||||
# touches the device, so it has no reason to bring up a vendor backend.
|
||||
from sglang.multimodal_gen.runtime.platforms.plugins import (
|
||||
apply_plugin_hooks,
|
||||
load_plugins,
|
||||
)
|
||||
|
||||
load_plugins()
|
||||
apply_plugin_hooks()
|
||||
load_plugins()
|
||||
apply_plugin_hooks()
|
||||
|
||||
from sglang.multimodal_gen.runtime import launch_server
|
||||
from sglang.multimodal_gen.runtime import launch_server
|
||||
|
||||
launch_server.launch_http_server_only(server_args.materialize())
|
||||
materialized = server_args.materialize()
|
||||
|
||||
launch_server.launch_http_server_only(materialized)
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Adapted from https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/utils.py
|
||||
|
||||
import contextlib
|
||||
import ctypes
|
||||
import gc
|
||||
import os
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
|
||||
import psutil
|
||||
|
||||
@@ -30,6 +33,61 @@ def kill_itself_when_parent_died() -> None:
|
||||
os.kill(os.getpid(), signal.SIGKILL)
|
||||
|
||||
|
||||
_full_gc_lock = threading.Lock()
|
||||
_full_gc_deferrals = 0
|
||||
_gc_thresholds = gc.get_threshold()
|
||||
# The largest third threshold the collector accepts, a C int, so never reached.
|
||||
_NO_FULL_GC = 2**31 - 1
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def defer_full_gc_for_imports() -> Iterator[None]:
|
||||
"""Run no automatic full garbage collection while a process imports its runtime.
|
||||
|
||||
Imports create long-lived modules, classes and functions and little garbage,
|
||||
yet each full collection traces the whole growing heap, and how many of them
|
||||
fall inside an import depends on the import order rather than on the work
|
||||
done. Young collections still run, so short-lived cyclic garbage is freed as
|
||||
usual. Restoring the entry thresholds lets automatic full collections resume
|
||||
after the block; nested or concurrent blocks restore them when the last one
|
||||
exits.
|
||||
"""
|
||||
global _full_gc_deferrals, _gc_thresholds
|
||||
# Python 3.14.0-3.14.4 uses incremental GC and ignores threshold2
|
||||
if (3, 14) <= sys.version_info < (3, 14, 5):
|
||||
yield
|
||||
return
|
||||
|
||||
pid = os.getpid()
|
||||
with _full_gc_lock:
|
||||
if _full_gc_deferrals == 0:
|
||||
_gc_thresholds = gc.get_threshold()
|
||||
gc.set_threshold(*_gc_thresholds[:2], _NO_FULL_GC)
|
||||
_full_gc_deferrals += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# A child forked inside the block restored the thresholds at fork.
|
||||
if os.getpid() == pid:
|
||||
with _full_gc_lock:
|
||||
_full_gc_deferrals -= 1
|
||||
if _full_gc_deferrals == 0:
|
||||
gc.set_threshold(*_gc_thresholds)
|
||||
|
||||
|
||||
def _restore_gc_thresholds_in_forked_child() -> None:
|
||||
# The parent restores the thresholds in the parent only, so a child forked
|
||||
# inside the block would otherwise never run a full collection.
|
||||
global _full_gc_lock, _full_gc_deferrals
|
||||
_full_gc_lock = threading.Lock()
|
||||
if _full_gc_deferrals:
|
||||
_full_gc_deferrals = 0
|
||||
gc.set_threshold(*_gc_thresholds)
|
||||
|
||||
|
||||
os.register_at_fork(after_in_child=_restore_gc_thresholds_in_forked_child)
|
||||
|
||||
|
||||
def kill_process_tree(
|
||||
parent_pid,
|
||||
include_parent: bool = True,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import contextlib
|
||||
import gc
|
||||
import importlib.util
|
||||
import os
|
||||
import subprocess
|
||||
@@ -7,6 +9,7 @@ import sys
|
||||
import textwrap
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import psutil
|
||||
|
||||
@@ -16,6 +19,7 @@ assert SPEC is not None and SPEC.loader is not None
|
||||
PROCESS = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(PROCESS)
|
||||
kill_process_tree = PROCESS.kill_process_tree
|
||||
defer_full_gc_for_imports = PROCESS.defer_full_gc_for_imports
|
||||
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "requires POSIX process semantics")
|
||||
@@ -91,5 +95,78 @@ class TestKillProcessTree(unittest.TestCase):
|
||||
kill_process_tree(0x7FFFFFFF)
|
||||
|
||||
|
||||
class TestDeferFullGcForImports(unittest.TestCase):
|
||||
def full_collections_while_allocating(self, block):
|
||||
gc.collect()
|
||||
threshold0, threshold1, threshold2 = gc.get_threshold()
|
||||
# Enough survivors that the collector's own rule makes a full one due.
|
||||
count = len(gc.get_objects()) + 2 * threshold0 * threshold1 * threshold2
|
||||
before = gc.get_stats()[2]["collections"]
|
||||
with block():
|
||||
kept = [[] for _ in range(count)]
|
||||
full = gc.get_stats()[2]["collections"] - before
|
||||
del kept
|
||||
return full
|
||||
|
||||
@unittest.skipIf(
|
||||
(3, 14) <= sys.version_info < (3, 14, 5),
|
||||
"incremental GC does not use threshold2",
|
||||
)
|
||||
def test_no_full_collection_inside_and_thresholds_restored(self):
|
||||
thresholds = gc.get_threshold()
|
||||
|
||||
self.assertGreater(
|
||||
self.full_collections_while_allocating(contextlib.nullcontext), 0
|
||||
)
|
||||
self.assertEqual(
|
||||
self.full_collections_while_allocating(defer_full_gc_for_imports), 0
|
||||
)
|
||||
with self.assertRaises(ImportError):
|
||||
with defer_full_gc_for_imports():
|
||||
raise ImportError
|
||||
self.assertEqual(gc.get_threshold(), thresholds)
|
||||
|
||||
def test_incremental_gc_is_unchanged(self):
|
||||
thresholds = gc.get_threshold()
|
||||
for version in ((3, 14, 0), (3, 14, 4)):
|
||||
with (
|
||||
self.subTest(version=version),
|
||||
patch.object(PROCESS.sys, "version_info", version),
|
||||
):
|
||||
with defer_full_gc_for_imports():
|
||||
self.assertEqual(gc.get_threshold(), thresholds)
|
||||
self.assertEqual(gc.get_threshold(), thresholds)
|
||||
|
||||
def test_nested_exception_preserves_disabled_collector(self):
|
||||
thresholds = gc.get_threshold()
|
||||
self.addCleanup(gc.enable if gc.isenabled() else gc.disable)
|
||||
self.addCleanup(gc.set_threshold, *thresholds)
|
||||
gc.disable()
|
||||
with defer_full_gc_for_imports():
|
||||
deferred = gc.get_threshold()
|
||||
with self.assertRaises(ImportError):
|
||||
with defer_full_gc_for_imports():
|
||||
raise ImportError
|
||||
self.assertEqual(gc.get_threshold(), deferred)
|
||||
self.assertFalse(gc.isenabled())
|
||||
self.assertEqual(gc.get_threshold(), thresholds)
|
||||
self.assertFalse(gc.isenabled())
|
||||
|
||||
@unittest.skipUnless(os.name == "posix", "requires fork")
|
||||
def test_fork_restores_child_without_changing_parent(self):
|
||||
thresholds = gc.get_threshold()
|
||||
with defer_full_gc_for_imports():
|
||||
deferred = gc.get_threshold()
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
restored = gc.get_threshold() == thresholds
|
||||
if pid == 0:
|
||||
with defer_full_gc_for_imports():
|
||||
restored &= gc.get_threshold() == deferred
|
||||
os._exit(int(not (restored and gc.get_threshold() == thresholds)))
|
||||
self.assertEqual(os.waitstatus_to_exitcode(os.waitpid(pid, 0)[1]), 0)
|
||||
self.assertEqual(gc.get_threshold(), thresholds)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user