From 860ebe720efe458b8752bbb1d4b20b7cbf16e629 Mon Sep 17 00:00:00 2001 From: Akshat Anand Date: Mon, 5 Oct 2026 17:29:35 +0530 Subject: [PATCH] [diffusion] fix: skip warmup preferred preload when it would OOM (#40761) Co-authored-by: cipheraxat --- .../memory_managers/component_manager.py | 26 +- .../component_residency_strategies.py | 89 +++++- .../test/unit/test_component_residency.py | 286 +++++++++++++++++- 3 files changed, 392 insertions(+), 9 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py index 3b0fdf7f63..d5c813e14f 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py @@ -346,6 +346,7 @@ class ComponentResidencyManager: def end_stage(self) -> None: """Close the component interval owned by the current stage.""" self._record_warmup_phase_peak() + self._release_unconsumed_stage_prefetches() if self._active_use is None: return if self._active_use.stage_name != self.state.stage_name: @@ -522,7 +523,10 @@ class ComponentResidencyManager: self.state.current_use = None self._begin_warmup_between_uses() if prefetch_next: - self._prefetch_next_memory_intensive_use() + # Stages call this when done with their components, so a later use + # declared by the same stage (e.g. transformer_2 on a warmup that + # only ran the high-noise expert) will not run in this request. + self._prefetch_next_memory_intensive_use(skip_stage=self.state.stage_name) def _prepare_forward_use( self, use: ComponentUse, module: nn.Module | None = None @@ -906,15 +910,31 @@ class ComponentResidencyManager: return index return None - def _prefetch_next_memory_intensive_use(self) -> None: + def _prefetch_next_memory_intensive_use( + self, *, skip_stage: str | None = None + ) -> None: for use in self._ordered_uses[self._current_use_index + 1 :]: - if not use.memory_intensive: + if not use.memory_intensive or use.stage_name == skip_stage: continue if self._use_key(use) in self._prefetched_use_keys: return self._prefetch_use(use) return + def _release_unconsumed_stage_prefetches(self) -> None: + """Release prefetched uses of the ending stage that never began.""" + for use in self.state.future_uses: + key = self._use_key(use) + if use.stage_name != self.state.stage_name or ( + key not in self._prefetched_use_keys + ): + continue + self._prefetched_use_keys.discard(key) + if self._active_use is None or ( + self._active_use.component_name != use.component_name + ): + self._finish_use(use, keep_on_warmup=False, force=True) + def _should_keep_after_use(self, use: ComponentUse) -> bool: if self.state.future_uses and self._same_use(use, self.state.future_uses[0]): return True diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_residency_strategies.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_residency_strategies.py index 63a5d52939..be1450d0d0 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_residency_strategies.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_residency_strategies.py @@ -2,6 +2,7 @@ from __future__ import annotations +from itertools import chain from typing import TYPE_CHECKING import torch @@ -27,6 +28,8 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger # Device growth between a component's stages on a shared pool: activations and # the allocator's reserve (9.4 GiB measured for H3 at 1344x768x124f) plus margin. SHARED_POOL_NEXT_STAGE_HEADROOM_BYTES = 12 * 1024**3 +# Leave headroom for the allocator when deciding whether a warmup preload fits. +_WARMUP_PRELOAD_MARGIN_BYTES = 1 * 1024**3 logger = init_logger(__name__) @@ -69,6 +72,51 @@ def _module_ready_on_local_device( return dtype is None or tensor.dtype == dtype +def _cpu_module_nbytes(module: nn.Module, *, dtype: torch.dtype | None = None) -> int: + """Bytes the host tensors would occupy after an optional device cast. + + ``module.to(device, dtype=...)`` only casts floating-point tensors, so + integer buffers keep their current nbytes. + """ + total = 0 + for tensor in (*module.parameters(), *module.buffers()): + if tensor.device.type != "cpu": + continue + if ( + dtype is None + or not tensor.is_floating_point() + or dtype.itemsize == tensor.element_size() + ): + total += tensor.nbytes + else: + total += tensor.numel() * dtype.itemsize + return total + + +def _device_free_bytes() -> int | None: + device_module = torch.get_device_module() + mem_get_info = getattr(device_module, "mem_get_info", None) + if mem_get_info is None or not device_module.is_available(): + return None + try: + return int(mem_get_info()[0]) + except RuntimeError: + # Some backends expose mem_get_info without implementing it. + return None + + +def _is_out_of_memory_error(error: BaseException) -> bool: + return isinstance(error, torch.OutOfMemoryError) or ( + "out of memory" in str(error).lower() + ) + + +def _empty_device_cache() -> None: + empty_cache = getattr(torch.get_device_module(), "empty_cache", None) + if empty_cache is not None: + empty_cache() + + def is_fsdp_managed_module(module: nn.Module) -> bool: return isinstance(module, FSDPModule) @@ -191,8 +239,11 @@ class ComponentOffloadStrategy(ComponentResidencyStrategy): state: ResidencyState, ) -> None: self.wait_for_use(module, use, state) - tensor = _module_reference_tensor(module) - if tensor is not None and tensor.device.type != "cpu": + # a failed H2D may leave children on device while the first parameter is on CPU + if any( + tensor.device.type != "cpu" + for tensor in chain(module.parameters(), module.buffers()) + ): # A non-blocking device->host move lands in pinned host memory the # size of the component. On a shared pool that pins a second copy # of the weights next to the device copy still being read from @@ -213,8 +264,38 @@ class ComponentOffloadStrategy(ComponentResidencyStrategy): preferred: bool, ) -> None: if preferred and state.batch_is_warmup: - self.prepare_for_use(module, use, state) - self.wait_for_use(module, use, state) + if _module_ready_on_local_device(module, dtype=use.target_dtype): + self.wait_for_use(module, use, state) + return + required_bytes = _cpu_module_nbytes(module, dtype=use.target_dtype) + free_bytes = _device_free_bytes() + if free_bytes is not None and required_bytes > ( + free_bytes - _WARMUP_PRELOAD_MARGIN_BYTES + ): + # reclaim unused allocator blocks only when driver-free memory is short + _empty_device_cache() + free_bytes = _device_free_bytes() + preload_failed = False + try: + if free_bytes is None or required_bytes <= ( + free_bytes - _WARMUP_PRELOAD_MARGIN_BYTES + ): + self.prepare_for_use(module, use, state) + self.wait_for_use(module, use, state) + return + except RuntimeError as error: + if not _is_out_of_memory_error(error): + raise + preload_failed = True + # Warmup preload is optional; the next request loads it on demand. + logger.warning( + "Warmup could not keep %s resident after request finalization; " + "leaving it offloaded until its next use.", + use.component_name, + ) + self.finish_use(module, use, state) + if preload_failed: + _empty_device_cache() return self.finish_use(module, use, state) diff --git a/python/sglang/multimodal_gen/test/unit/test_component_residency.py b/python/sglang/multimodal_gen/test/unit/test_component_residency.py index af70919a48..b54f1ea7a4 100644 --- a/python/sglang/multimodal_gen/test/unit/test_component_residency.py +++ b/python/sglang/multimodal_gen/test/unit/test_component_residency.py @@ -11,6 +11,9 @@ from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import ( QwenImagePipelineConfig, ) from sglang.multimodal_gen.runtime.loader.utils import component_residency_bytes +from sglang.multimodal_gen.runtime.managers.memory_managers import ( + component_residency_strategies as residency_strategies, +) from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentResidencyManager, ComponentUse, @@ -208,8 +211,22 @@ def test_snapshot_offload_sleep_uses_existing_host_storage(): assert module.weight.data_ptr() == host_pointer -def test_component_offload_keeps_preferred_component_after_warmup(): - strategy = ComponentOffloadStrategy() +@pytest.mark.parametrize( + "strategy_cls", [ComponentOffloadStrategy, SnapshotOffloadStrategy] +) +@pytest.mark.parametrize("free_bytes", [None, 8 * 1024**3]) +def test_component_offload_keeps_preferred_component_after_warmup( + monkeypatch, strategy_cls, free_bytes +): + empty_cache = Mock() + monkeypatch.setattr(residency_strategies, "_empty_device_cache", empty_cache) + monkeypatch.setattr(residency_strategies, "_device_free_bytes", lambda: free_bytes) + monkeypatch.setattr( + residency_strategies, + "_module_ready_on_local_device", + lambda *args, **kwargs: False, + ) + strategy = strategy_cls() strategy.prepare_for_use = Mock() strategy.wait_for_use = Mock() strategy.finish_use = Mock() @@ -226,6 +243,216 @@ def test_component_offload_keeps_preferred_component_after_warmup(): strategy.prepare_for_use.assert_called_once_with(module, use, state) strategy.wait_for_use.assert_called_once_with(module, use, state) strategy.finish_use.assert_not_called() + empty_cache.assert_not_called() + + +def test_component_offload_warmup_rechecks_memory_after_releasing_cache(monkeypatch): + empty_cache = Mock() + free_bytes = Mock(side_effect=[0, 8 * 1024**3]) + monkeypatch.setattr(residency_strategies, "_empty_device_cache", empty_cache) + monkeypatch.setattr(residency_strategies, "_device_free_bytes", free_bytes) + monkeypatch.setattr( + residency_strategies, + "_module_ready_on_local_device", + lambda *args, **kwargs: False, + ) + strategy = ComponentOffloadStrategy() + strategy.prepare_for_use = Mock() + strategy.wait_for_use = Mock() + strategy.finish_use = Mock() + module = torch.nn.Linear(2, 2) + use = ComponentUse("denoise", "transformer") + state = ResidencyState(batch_is_warmup=True) + + strategy.finish_request(module, use, state, preferred=True) + + empty_cache.assert_called_once_with() + assert free_bytes.call_count == 2 + strategy.prepare_for_use.assert_called_once_with(module, use, state) + strategy.wait_for_use.assert_called_once_with(module, use, state) + strategy.finish_use.assert_not_called() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_component_offload_warmup_keeps_ready_component_without_budget_check( + monkeypatch, +): + empty_cache = Mock() + free_bytes = Mock(return_value=0) + monkeypatch.setattr(residency_strategies, "_empty_device_cache", empty_cache) + monkeypatch.setattr(residency_strategies, "_device_free_bytes", free_bytes) + strategy = ComponentOffloadStrategy() + strategy.prepare_for_use = Mock() + strategy.wait_for_use = Mock() + strategy.finish_use = Mock() + module = torch.nn.Linear(2, 2, device="cuda", dtype=torch.bfloat16) + use = ComponentUse("denoise", "transformer", target_dtype=torch.bfloat16) + state = ResidencyState(batch_is_warmup=True) + + strategy.finish_request(module, use, state, preferred=True) + + free_bytes.assert_not_called() + empty_cache.assert_not_called() + strategy.prepare_for_use.assert_not_called() + strategy.wait_for_use.assert_called_once_with(module, use, state) + strategy.finish_use.assert_not_called() + + +def test_component_offload_warmup_propagates_non_oom_errors(monkeypatch): + monkeypatch.setattr(residency_strategies, "_device_free_bytes", lambda: 8 * 1024**3) + monkeypatch.setattr( + residency_strategies, + "_module_ready_on_local_device", + lambda *args, **kwargs: False, + ) + empty_cache = Mock() + monkeypatch.setattr(residency_strategies, "_empty_device_cache", empty_cache) + strategy = ComponentOffloadStrategy() + strategy.prepare_for_use = Mock(side_effect=RuntimeError("invalid dtype")) + strategy.finish_use = Mock() + + with pytest.raises(RuntimeError, match="invalid dtype"): + strategy.finish_request( + torch.nn.Linear(2, 2), + ComponentUse("denoise", "transformer"), + ResidencyState(batch_is_warmup=True), + preferred=True, + ) + + strategy.finish_use.assert_not_called() + empty_cache.assert_not_called() + + +@pytest.mark.parametrize( + "free_bytes, prepare_error", + [ + (64, None), + (8 * 1024**3, torch.OutOfMemoryError("CUDA out of memory")), + ], + ids=["does-not-fit", "oom"], +) +def test_component_offload_warmup_preload_failure_leaves_component_offloaded( + monkeypatch, free_bytes, prepare_error +): + device_module = SimpleNamespace( + is_available=lambda: True, + empty_cache=Mock(), + mem_get_info=lambda: (free_bytes, 16 * 1024**3), + ) + monkeypatch.setattr(torch, "get_device_module", lambda: device_module) + monkeypatch.setattr( + residency_strategies, + "_module_ready_on_local_device", + lambda *args, **kwargs: False, + ) + strategy = ComponentOffloadStrategy() + strategy.prepare_for_use = Mock(side_effect=prepare_error) + strategy.wait_for_use = Mock() + strategy.finish_use = Mock() + module = torch.nn.Linear(64, 64) + use = ComponentUse( + stage_name="DenoisingStage", + component_name="transformer", + preferred_ready_after_request=True, + ) + state = ResidencyState(batch_is_warmup=True) + + strategy.finish_request(module, use, state, preferred=True) + + assert strategy.prepare_for_use.call_count == (prepare_error is not None) + strategy.wait_for_use.assert_not_called() + strategy.finish_use.assert_called_once_with(module, use, state) + device_module.empty_cache.assert_called_once_with() + + +def test_component_offload_warmup_preload_sizes_cast_to_target_dtype(monkeypatch): + """A wider target dtype must count toward the preload budget, not host bytes.""" + module = torch.nn.Linear(64, 64, bias=False).to(dtype=torch.float16) + host_bytes = sum(tensor.nbytes for tensor in module.parameters()) + cast_bytes = sum( + tensor.numel() * torch.float32.itemsize for tensor in module.parameters() + ) + assert cast_bytes == 2 * host_bytes + # Enough free memory for the host copy + margin, but not for the cast copy. + free_bytes = host_bytes + (1 * 1024**3) + 1 + device_module = SimpleNamespace( + is_available=lambda: True, + empty_cache=Mock(), + mem_get_info=lambda: (free_bytes, 16 * 1024**3), + ) + monkeypatch.setattr(torch, "get_device_module", lambda: device_module) + monkeypatch.setattr( + residency_strategies, + "_module_ready_on_local_device", + lambda *args, **kwargs: False, + ) + strategy = ComponentOffloadStrategy() + strategy.prepare_for_use = Mock() + strategy.wait_for_use = Mock() + strategy.finish_use = Mock() + use = ComponentUse( + stage_name="DenoisingStage", + component_name="transformer", + preferred_ready_after_request=True, + target_dtype=torch.float32, + ) + state = ResidencyState(batch_is_warmup=True) + + strategy.finish_request(module, use, state, preferred=True) + + strategy.prepare_for_use.assert_not_called() + strategy.finish_use.assert_called_once_with(module, use, state) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize( + "strategy_cls", [ComponentOffloadStrategy, SnapshotOffloadStrategy] +) +@pytest.mark.parametrize("root_parameter", [False, True]) +def test_component_offload_warmup_preload_partial_oom_moves_module_to_cpu( + monkeypatch, strategy_cls, root_parameter +): + strategy = strategy_cls() + module = torch.nn.Sequential(torch.nn.Linear(4, 4), torch.nn.Linear(4, 4)) + module[0].register_buffer("counter", torch.ones((), dtype=torch.int64)) + if root_parameter: + module.register_parameter("root", torch.nn.Parameter(torch.zeros(1))) + expected = {name: tensor.clone() for name, tensor in module.state_dict().items()} + use = ComponentUse( + stage_name="DenoisingStage", + component_name="transformer", + preferred_ready_after_request=True, + ) + state = ResidencyState(batch_is_warmup=True) + + # Module.to visits children before parent parameters; parameters() does not + failing_tensor = module.root if root_parameter else module[1].bias + original_to = torch.Tensor.to + + def fail_partial_move(tensor, *args, **kwargs): + if tensor is failing_tensor and torch.device(args[0]).type == "cuda": + assert module[0].weight.device.type == "cuda" + assert module[0].counter.device.type == "cuda" + raise torch.OutOfMemoryError("CUDA out of memory") + return original_to(tensor, *args, **kwargs) + + monkeypatch.setattr(torch.Tensor, "to", fail_partial_move) + + strategy.finish_request(module, use, state, preferred=True) + + for name, tensor in module.state_dict().items(): + assert tensor.device.type == "cpu", name + torch.testing.assert_close(tensor, expected[name], rtol=0, atol=0) + + monkeypatch.setattr(torch.Tensor, "to", original_to) + state.batch_is_warmup = False + x = torch.randn(2, 4) + expected_output = module(x) + strategy.prepare_for_use(module, use, state) + strategy.wait_for_use(module, use, state) + torch.testing.assert_close(module(x.cuda()).cpu(), expected_output) + strategy.finish_request(module, use, state, preferred=False) + assert all(tensor.device.type == "cpu" for tensor in module.state_dict().values()) def test_request_tail_uses_dynamic_component_instance(): @@ -1047,3 +1274,58 @@ def test_component_is_not_kept_across_another_component_use(): manager.end_use(text_use) strategy.finish_use.assert_called_once_with(module, text_use, manager.state) + + +def _dual_expert_manager(): + names = ("transformer", "transformer_2") + uses = { + name: ComponentUse("denoise", name, phase=name, memory_intensive=True) + for name in names + } + denoise = _Stage(*uses.values()) + decode = _Stage(ComponentUse("decode", "vae", memory_intensive=True)) + pipeline = SimpleNamespace( + modules={name: torch.nn.Linear(2, 2) for name in (*names, "vae")}, + _stage_name_mapping={"denoise": denoise, "decode": decode}, + component_residency_strategies={}, + ) + server_args = SimpleNamespace(enable_layerwise_nvtx_marker=False) + manager = ComponentResidencyManager(pipeline, server_args) + manager.refresh_pipeline(pipeline) + batch = SimpleNamespace(is_warmup=False) + manager.begin_request([denoise, decode], batch, server_args) + manager.before_stage(denoise, 0, batch, server_args) + return manager, uses + + +@pytest.mark.parametrize( + "offload, experts_run, expected_finished", + [ + (True, ("transformer",), ["transformer"]), + # Non-offload strategies prefetch transformer_2 while transformer runs. + (False, ("transformer",), ["transformer", "transformer_2"]), + (True, ("transformer", "transformer_2"), ["transformer", "transformer_2"]), + ], + ids=["one-expert-offload", "one-expert-prefetch-while-busy", "both-experts"], +) +def test_stage_exit_does_not_keep_unused_same_stage_expert( + offload, experts_run, expected_finished +): + manager, uses = _dual_expert_manager() + strategy = Mock(spec=ComponentOffloadStrategy) if offload else Mock() + strategy.prefetch_for_use.return_value = True + manager.strategy_for = Mock(return_value=strategy) + + for name in experts_run: + manager.begin_use(uses[name]) + # DenoisingStage finishes its DiT inside the stage; the executor then + # calls end_stage. + manager.finish_active_use() + manager.end_stage() + + finished = [ + call.args[1].component_name for call in strategy.finish_use.call_args_list + ] + assert finished == expected_finished + assert strategy.prefetch_for_use.call_args.args[1].component_name == "vae" + assert manager._use_key(uses["transformer_2"]) not in manager._prefetched_use_keys