[diffusion] fix: skip warmup preferred preload when it would OOM (#40761)
Co-authored-by: cipheraxat <cipheraxat@users.noreply.github.com>
This commit is contained in:
co-authored by
cipheraxat
parent
8663e3b670
commit
860ebe720e
@@ -346,6 +346,7 @@ class ComponentResidencyManager:
|
||||
def end_stage(self) -> None:
|
||||
"""Close the component interval owned by the current stage."""
|
||||
self._record_warmup_phase_peak()
|
||||
self._release_unconsumed_stage_prefetches()
|
||||
if self._active_use is None:
|
||||
return
|
||||
if self._active_use.stage_name != self.state.stage_name:
|
||||
@@ -522,7 +523,10 @@ class ComponentResidencyManager:
|
||||
self.state.current_use = None
|
||||
self._begin_warmup_between_uses()
|
||||
if prefetch_next:
|
||||
self._prefetch_next_memory_intensive_use()
|
||||
# Stages call this when done with their components, so a later use
|
||||
# declared by the same stage (e.g. transformer_2 on a warmup that
|
||||
# only ran the high-noise expert) will not run in this request.
|
||||
self._prefetch_next_memory_intensive_use(skip_stage=self.state.stage_name)
|
||||
|
||||
def _prepare_forward_use(
|
||||
self, use: ComponentUse, module: nn.Module | None = None
|
||||
@@ -906,15 +910,31 @@ class ComponentResidencyManager:
|
||||
return index
|
||||
return None
|
||||
|
||||
def _prefetch_next_memory_intensive_use(self) -> None:
|
||||
def _prefetch_next_memory_intensive_use(
|
||||
self, *, skip_stage: str | None = None
|
||||
) -> None:
|
||||
for use in self._ordered_uses[self._current_use_index + 1 :]:
|
||||
if not use.memory_intensive:
|
||||
if not use.memory_intensive or use.stage_name == skip_stage:
|
||||
continue
|
||||
if self._use_key(use) in self._prefetched_use_keys:
|
||||
return
|
||||
self._prefetch_use(use)
|
||||
return
|
||||
|
||||
def _release_unconsumed_stage_prefetches(self) -> None:
|
||||
"""Release prefetched uses of the ending stage that never began."""
|
||||
for use in self.state.future_uses:
|
||||
key = self._use_key(use)
|
||||
if use.stage_name != self.state.stage_name or (
|
||||
key not in self._prefetched_use_keys
|
||||
):
|
||||
continue
|
||||
self._prefetched_use_keys.discard(key)
|
||||
if self._active_use is None or (
|
||||
self._active_use.component_name != use.component_name
|
||||
):
|
||||
self._finish_use(use, keep_on_warmup=False, force=True)
|
||||
|
||||
def _should_keep_after_use(self, use: ComponentUse) -> bool:
|
||||
if self.state.future_uses and self._same_use(use, self.state.future_uses[0]):
|
||||
return True
|
||||
|
||||
+85
-4
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
@@ -27,6 +28,8 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
# Device growth between a component's stages on a shared pool: activations and
|
||||
# the allocator's reserve (9.4 GiB measured for H3 at 1344x768x124f) plus margin.
|
||||
SHARED_POOL_NEXT_STAGE_HEADROOM_BYTES = 12 * 1024**3
|
||||
# Leave headroom for the allocator when deciding whether a warmup preload fits.
|
||||
_WARMUP_PRELOAD_MARGIN_BYTES = 1 * 1024**3
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -69,6 +72,51 @@ def _module_ready_on_local_device(
|
||||
return dtype is None or tensor.dtype == dtype
|
||||
|
||||
|
||||
def _cpu_module_nbytes(module: nn.Module, *, dtype: torch.dtype | None = None) -> int:
|
||||
"""Bytes the host tensors would occupy after an optional device cast.
|
||||
|
||||
``module.to(device, dtype=...)`` only casts floating-point tensors, so
|
||||
integer buffers keep their current nbytes.
|
||||
"""
|
||||
total = 0
|
||||
for tensor in (*module.parameters(), *module.buffers()):
|
||||
if tensor.device.type != "cpu":
|
||||
continue
|
||||
if (
|
||||
dtype is None
|
||||
or not tensor.is_floating_point()
|
||||
or dtype.itemsize == tensor.element_size()
|
||||
):
|
||||
total += tensor.nbytes
|
||||
else:
|
||||
total += tensor.numel() * dtype.itemsize
|
||||
return total
|
||||
|
||||
|
||||
def _device_free_bytes() -> int | None:
|
||||
device_module = torch.get_device_module()
|
||||
mem_get_info = getattr(device_module, "mem_get_info", None)
|
||||
if mem_get_info is None or not device_module.is_available():
|
||||
return None
|
||||
try:
|
||||
return int(mem_get_info()[0])
|
||||
except RuntimeError:
|
||||
# Some backends expose mem_get_info without implementing it.
|
||||
return None
|
||||
|
||||
|
||||
def _is_out_of_memory_error(error: BaseException) -> bool:
|
||||
return isinstance(error, torch.OutOfMemoryError) or (
|
||||
"out of memory" in str(error).lower()
|
||||
)
|
||||
|
||||
|
||||
def _empty_device_cache() -> None:
|
||||
empty_cache = getattr(torch.get_device_module(), "empty_cache", None)
|
||||
if empty_cache is not None:
|
||||
empty_cache()
|
||||
|
||||
|
||||
def is_fsdp_managed_module(module: nn.Module) -> bool:
|
||||
return isinstance(module, FSDPModule)
|
||||
|
||||
@@ -191,8 +239,11 @@ class ComponentOffloadStrategy(ComponentResidencyStrategy):
|
||||
state: ResidencyState,
|
||||
) -> None:
|
||||
self.wait_for_use(module, use, state)
|
||||
tensor = _module_reference_tensor(module)
|
||||
if tensor is not None and tensor.device.type != "cpu":
|
||||
# a failed H2D may leave children on device while the first parameter is on CPU
|
||||
if any(
|
||||
tensor.device.type != "cpu"
|
||||
for tensor in chain(module.parameters(), module.buffers())
|
||||
):
|
||||
# A non-blocking device->host move lands in pinned host memory the
|
||||
# size of the component. On a shared pool that pins a second copy
|
||||
# of the weights next to the device copy still being read from
|
||||
@@ -213,8 +264,38 @@ class ComponentOffloadStrategy(ComponentResidencyStrategy):
|
||||
preferred: bool,
|
||||
) -> None:
|
||||
if preferred and state.batch_is_warmup:
|
||||
self.prepare_for_use(module, use, state)
|
||||
self.wait_for_use(module, use, state)
|
||||
if _module_ready_on_local_device(module, dtype=use.target_dtype):
|
||||
self.wait_for_use(module, use, state)
|
||||
return
|
||||
required_bytes = _cpu_module_nbytes(module, dtype=use.target_dtype)
|
||||
free_bytes = _device_free_bytes()
|
||||
if free_bytes is not None and required_bytes > (
|
||||
free_bytes - _WARMUP_PRELOAD_MARGIN_BYTES
|
||||
):
|
||||
# reclaim unused allocator blocks only when driver-free memory is short
|
||||
_empty_device_cache()
|
||||
free_bytes = _device_free_bytes()
|
||||
preload_failed = False
|
||||
try:
|
||||
if free_bytes is None or required_bytes <= (
|
||||
free_bytes - _WARMUP_PRELOAD_MARGIN_BYTES
|
||||
):
|
||||
self.prepare_for_use(module, use, state)
|
||||
self.wait_for_use(module, use, state)
|
||||
return
|
||||
except RuntimeError as error:
|
||||
if not _is_out_of_memory_error(error):
|
||||
raise
|
||||
preload_failed = True
|
||||
# Warmup preload is optional; the next request loads it on demand.
|
||||
logger.warning(
|
||||
"Warmup could not keep %s resident after request finalization; "
|
||||
"leaving it offloaded until its next use.",
|
||||
use.component_name,
|
||||
)
|
||||
self.finish_use(module, use, state)
|
||||
if preload_failed:
|
||||
_empty_device_cache()
|
||||
return
|
||||
self.finish_use(module, use, state)
|
||||
|
||||
|
||||
@@ -11,6 +11,9 @@ from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import (
|
||||
QwenImagePipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import component_residency_bytes
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers import (
|
||||
component_residency_strategies as residency_strategies,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentResidencyManager,
|
||||
ComponentUse,
|
||||
@@ -208,8 +211,22 @@ def test_snapshot_offload_sleep_uses_existing_host_storage():
|
||||
assert module.weight.data_ptr() == host_pointer
|
||||
|
||||
|
||||
def test_component_offload_keeps_preferred_component_after_warmup():
|
||||
strategy = ComponentOffloadStrategy()
|
||||
@pytest.mark.parametrize(
|
||||
"strategy_cls", [ComponentOffloadStrategy, SnapshotOffloadStrategy]
|
||||
)
|
||||
@pytest.mark.parametrize("free_bytes", [None, 8 * 1024**3])
|
||||
def test_component_offload_keeps_preferred_component_after_warmup(
|
||||
monkeypatch, strategy_cls, free_bytes
|
||||
):
|
||||
empty_cache = Mock()
|
||||
monkeypatch.setattr(residency_strategies, "_empty_device_cache", empty_cache)
|
||||
monkeypatch.setattr(residency_strategies, "_device_free_bytes", lambda: free_bytes)
|
||||
monkeypatch.setattr(
|
||||
residency_strategies,
|
||||
"_module_ready_on_local_device",
|
||||
lambda *args, **kwargs: False,
|
||||
)
|
||||
strategy = strategy_cls()
|
||||
strategy.prepare_for_use = Mock()
|
||||
strategy.wait_for_use = Mock()
|
||||
strategy.finish_use = Mock()
|
||||
@@ -226,6 +243,216 @@ def test_component_offload_keeps_preferred_component_after_warmup():
|
||||
strategy.prepare_for_use.assert_called_once_with(module, use, state)
|
||||
strategy.wait_for_use.assert_called_once_with(module, use, state)
|
||||
strategy.finish_use.assert_not_called()
|
||||
empty_cache.assert_not_called()
|
||||
|
||||
|
||||
def test_component_offload_warmup_rechecks_memory_after_releasing_cache(monkeypatch):
|
||||
empty_cache = Mock()
|
||||
free_bytes = Mock(side_effect=[0, 8 * 1024**3])
|
||||
monkeypatch.setattr(residency_strategies, "_empty_device_cache", empty_cache)
|
||||
monkeypatch.setattr(residency_strategies, "_device_free_bytes", free_bytes)
|
||||
monkeypatch.setattr(
|
||||
residency_strategies,
|
||||
"_module_ready_on_local_device",
|
||||
lambda *args, **kwargs: False,
|
||||
)
|
||||
strategy = ComponentOffloadStrategy()
|
||||
strategy.prepare_for_use = Mock()
|
||||
strategy.wait_for_use = Mock()
|
||||
strategy.finish_use = Mock()
|
||||
module = torch.nn.Linear(2, 2)
|
||||
use = ComponentUse("denoise", "transformer")
|
||||
state = ResidencyState(batch_is_warmup=True)
|
||||
|
||||
strategy.finish_request(module, use, state, preferred=True)
|
||||
|
||||
empty_cache.assert_called_once_with()
|
||||
assert free_bytes.call_count == 2
|
||||
strategy.prepare_for_use.assert_called_once_with(module, use, state)
|
||||
strategy.wait_for_use.assert_called_once_with(module, use, state)
|
||||
strategy.finish_use.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_component_offload_warmup_keeps_ready_component_without_budget_check(
|
||||
monkeypatch,
|
||||
):
|
||||
empty_cache = Mock()
|
||||
free_bytes = Mock(return_value=0)
|
||||
monkeypatch.setattr(residency_strategies, "_empty_device_cache", empty_cache)
|
||||
monkeypatch.setattr(residency_strategies, "_device_free_bytes", free_bytes)
|
||||
strategy = ComponentOffloadStrategy()
|
||||
strategy.prepare_for_use = Mock()
|
||||
strategy.wait_for_use = Mock()
|
||||
strategy.finish_use = Mock()
|
||||
module = torch.nn.Linear(2, 2, device="cuda", dtype=torch.bfloat16)
|
||||
use = ComponentUse("denoise", "transformer", target_dtype=torch.bfloat16)
|
||||
state = ResidencyState(batch_is_warmup=True)
|
||||
|
||||
strategy.finish_request(module, use, state, preferred=True)
|
||||
|
||||
free_bytes.assert_not_called()
|
||||
empty_cache.assert_not_called()
|
||||
strategy.prepare_for_use.assert_not_called()
|
||||
strategy.wait_for_use.assert_called_once_with(module, use, state)
|
||||
strategy.finish_use.assert_not_called()
|
||||
|
||||
|
||||
def test_component_offload_warmup_propagates_non_oom_errors(monkeypatch):
|
||||
monkeypatch.setattr(residency_strategies, "_device_free_bytes", lambda: 8 * 1024**3)
|
||||
monkeypatch.setattr(
|
||||
residency_strategies,
|
||||
"_module_ready_on_local_device",
|
||||
lambda *args, **kwargs: False,
|
||||
)
|
||||
empty_cache = Mock()
|
||||
monkeypatch.setattr(residency_strategies, "_empty_device_cache", empty_cache)
|
||||
strategy = ComponentOffloadStrategy()
|
||||
strategy.prepare_for_use = Mock(side_effect=RuntimeError("invalid dtype"))
|
||||
strategy.finish_use = Mock()
|
||||
|
||||
with pytest.raises(RuntimeError, match="invalid dtype"):
|
||||
strategy.finish_request(
|
||||
torch.nn.Linear(2, 2),
|
||||
ComponentUse("denoise", "transformer"),
|
||||
ResidencyState(batch_is_warmup=True),
|
||||
preferred=True,
|
||||
)
|
||||
|
||||
strategy.finish_use.assert_not_called()
|
||||
empty_cache.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"free_bytes, prepare_error",
|
||||
[
|
||||
(64, None),
|
||||
(8 * 1024**3, torch.OutOfMemoryError("CUDA out of memory")),
|
||||
],
|
||||
ids=["does-not-fit", "oom"],
|
||||
)
|
||||
def test_component_offload_warmup_preload_failure_leaves_component_offloaded(
|
||||
monkeypatch, free_bytes, prepare_error
|
||||
):
|
||||
device_module = SimpleNamespace(
|
||||
is_available=lambda: True,
|
||||
empty_cache=Mock(),
|
||||
mem_get_info=lambda: (free_bytes, 16 * 1024**3),
|
||||
)
|
||||
monkeypatch.setattr(torch, "get_device_module", lambda: device_module)
|
||||
monkeypatch.setattr(
|
||||
residency_strategies,
|
||||
"_module_ready_on_local_device",
|
||||
lambda *args, **kwargs: False,
|
||||
)
|
||||
strategy = ComponentOffloadStrategy()
|
||||
strategy.prepare_for_use = Mock(side_effect=prepare_error)
|
||||
strategy.wait_for_use = Mock()
|
||||
strategy.finish_use = Mock()
|
||||
module = torch.nn.Linear(64, 64)
|
||||
use = ComponentUse(
|
||||
stage_name="DenoisingStage",
|
||||
component_name="transformer",
|
||||
preferred_ready_after_request=True,
|
||||
)
|
||||
state = ResidencyState(batch_is_warmup=True)
|
||||
|
||||
strategy.finish_request(module, use, state, preferred=True)
|
||||
|
||||
assert strategy.prepare_for_use.call_count == (prepare_error is not None)
|
||||
strategy.wait_for_use.assert_not_called()
|
||||
strategy.finish_use.assert_called_once_with(module, use, state)
|
||||
device_module.empty_cache.assert_called_once_with()
|
||||
|
||||
|
||||
def test_component_offload_warmup_preload_sizes_cast_to_target_dtype(monkeypatch):
|
||||
"""A wider target dtype must count toward the preload budget, not host bytes."""
|
||||
module = torch.nn.Linear(64, 64, bias=False).to(dtype=torch.float16)
|
||||
host_bytes = sum(tensor.nbytes for tensor in module.parameters())
|
||||
cast_bytes = sum(
|
||||
tensor.numel() * torch.float32.itemsize for tensor in module.parameters()
|
||||
)
|
||||
assert cast_bytes == 2 * host_bytes
|
||||
# Enough free memory for the host copy + margin, but not for the cast copy.
|
||||
free_bytes = host_bytes + (1 * 1024**3) + 1
|
||||
device_module = SimpleNamespace(
|
||||
is_available=lambda: True,
|
||||
empty_cache=Mock(),
|
||||
mem_get_info=lambda: (free_bytes, 16 * 1024**3),
|
||||
)
|
||||
monkeypatch.setattr(torch, "get_device_module", lambda: device_module)
|
||||
monkeypatch.setattr(
|
||||
residency_strategies,
|
||||
"_module_ready_on_local_device",
|
||||
lambda *args, **kwargs: False,
|
||||
)
|
||||
strategy = ComponentOffloadStrategy()
|
||||
strategy.prepare_for_use = Mock()
|
||||
strategy.wait_for_use = Mock()
|
||||
strategy.finish_use = Mock()
|
||||
use = ComponentUse(
|
||||
stage_name="DenoisingStage",
|
||||
component_name="transformer",
|
||||
preferred_ready_after_request=True,
|
||||
target_dtype=torch.float32,
|
||||
)
|
||||
state = ResidencyState(batch_is_warmup=True)
|
||||
|
||||
strategy.finish_request(module, use, state, preferred=True)
|
||||
|
||||
strategy.prepare_for_use.assert_not_called()
|
||||
strategy.finish_use.assert_called_once_with(module, use, state)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"strategy_cls", [ComponentOffloadStrategy, SnapshotOffloadStrategy]
|
||||
)
|
||||
@pytest.mark.parametrize("root_parameter", [False, True])
|
||||
def test_component_offload_warmup_preload_partial_oom_moves_module_to_cpu(
|
||||
monkeypatch, strategy_cls, root_parameter
|
||||
):
|
||||
strategy = strategy_cls()
|
||||
module = torch.nn.Sequential(torch.nn.Linear(4, 4), torch.nn.Linear(4, 4))
|
||||
module[0].register_buffer("counter", torch.ones((), dtype=torch.int64))
|
||||
if root_parameter:
|
||||
module.register_parameter("root", torch.nn.Parameter(torch.zeros(1)))
|
||||
expected = {name: tensor.clone() for name, tensor in module.state_dict().items()}
|
||||
use = ComponentUse(
|
||||
stage_name="DenoisingStage",
|
||||
component_name="transformer",
|
||||
preferred_ready_after_request=True,
|
||||
)
|
||||
state = ResidencyState(batch_is_warmup=True)
|
||||
|
||||
# Module.to visits children before parent parameters; parameters() does not
|
||||
failing_tensor = module.root if root_parameter else module[1].bias
|
||||
original_to = torch.Tensor.to
|
||||
|
||||
def fail_partial_move(tensor, *args, **kwargs):
|
||||
if tensor is failing_tensor and torch.device(args[0]).type == "cuda":
|
||||
assert module[0].weight.device.type == "cuda"
|
||||
assert module[0].counter.device.type == "cuda"
|
||||
raise torch.OutOfMemoryError("CUDA out of memory")
|
||||
return original_to(tensor, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(torch.Tensor, "to", fail_partial_move)
|
||||
|
||||
strategy.finish_request(module, use, state, preferred=True)
|
||||
|
||||
for name, tensor in module.state_dict().items():
|
||||
assert tensor.device.type == "cpu", name
|
||||
torch.testing.assert_close(tensor, expected[name], rtol=0, atol=0)
|
||||
|
||||
monkeypatch.setattr(torch.Tensor, "to", original_to)
|
||||
state.batch_is_warmup = False
|
||||
x = torch.randn(2, 4)
|
||||
expected_output = module(x)
|
||||
strategy.prepare_for_use(module, use, state)
|
||||
strategy.wait_for_use(module, use, state)
|
||||
torch.testing.assert_close(module(x.cuda()).cpu(), expected_output)
|
||||
strategy.finish_request(module, use, state, preferred=False)
|
||||
assert all(tensor.device.type == "cpu" for tensor in module.state_dict().values())
|
||||
|
||||
|
||||
def test_request_tail_uses_dynamic_component_instance():
|
||||
@@ -1047,3 +1274,58 @@ def test_component_is_not_kept_across_another_component_use():
|
||||
manager.end_use(text_use)
|
||||
|
||||
strategy.finish_use.assert_called_once_with(module, text_use, manager.state)
|
||||
|
||||
|
||||
def _dual_expert_manager():
|
||||
names = ("transformer", "transformer_2")
|
||||
uses = {
|
||||
name: ComponentUse("denoise", name, phase=name, memory_intensive=True)
|
||||
for name in names
|
||||
}
|
||||
denoise = _Stage(*uses.values())
|
||||
decode = _Stage(ComponentUse("decode", "vae", memory_intensive=True))
|
||||
pipeline = SimpleNamespace(
|
||||
modules={name: torch.nn.Linear(2, 2) for name in (*names, "vae")},
|
||||
_stage_name_mapping={"denoise": denoise, "decode": decode},
|
||||
component_residency_strategies={},
|
||||
)
|
||||
server_args = SimpleNamespace(enable_layerwise_nvtx_marker=False)
|
||||
manager = ComponentResidencyManager(pipeline, server_args)
|
||||
manager.refresh_pipeline(pipeline)
|
||||
batch = SimpleNamespace(is_warmup=False)
|
||||
manager.begin_request([denoise, decode], batch, server_args)
|
||||
manager.before_stage(denoise, 0, batch, server_args)
|
||||
return manager, uses
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"offload, experts_run, expected_finished",
|
||||
[
|
||||
(True, ("transformer",), ["transformer"]),
|
||||
# Non-offload strategies prefetch transformer_2 while transformer runs.
|
||||
(False, ("transformer",), ["transformer", "transformer_2"]),
|
||||
(True, ("transformer", "transformer_2"), ["transformer", "transformer_2"]),
|
||||
],
|
||||
ids=["one-expert-offload", "one-expert-prefetch-while-busy", "both-experts"],
|
||||
)
|
||||
def test_stage_exit_does_not_keep_unused_same_stage_expert(
|
||||
offload, experts_run, expected_finished
|
||||
):
|
||||
manager, uses = _dual_expert_manager()
|
||||
strategy = Mock(spec=ComponentOffloadStrategy) if offload else Mock()
|
||||
strategy.prefetch_for_use.return_value = True
|
||||
manager.strategy_for = Mock(return_value=strategy)
|
||||
|
||||
for name in experts_run:
|
||||
manager.begin_use(uses[name])
|
||||
# DenoisingStage finishes its DiT inside the stage; the executor then
|
||||
# calls end_stage.
|
||||
manager.finish_active_use()
|
||||
manager.end_stage()
|
||||
|
||||
finished = [
|
||||
call.args[1].component_name for call in strategy.finish_use.call_args_list
|
||||
]
|
||||
assert finished == expected_finished
|
||||
assert strategy.prefetch_for_use.call_args.args[1].component_name == "vae"
|
||||
assert manager._use_key(uses["transformer_2"]) not in manager._prefetched_use_keys
|
||||
|
||||
Reference in New Issue
Block a user