diff --git a/docs/docs/sglang-diffusion/api/cli.mdx b/docs/docs/sglang-diffusion/api/cli.mdx index 120835fb72..56a61d2768 100644 --- a/docs/docs/sglang-diffusion/api/cli.mdx +++ b/docs/docs/sglang-diffusion/api/cli.mdx @@ -333,6 +333,16 @@ Selectors match exact loaded component keys from `model_index.json`, including n The existing `--dit-cpu-offload`, `--text-encoder-cpu-offload`, `--image-encoder-cpu-offload`, `--vae-cpu-offload`, and `--cpu-offload-components` options remain supported. New and legacy options may be mixed: `--component-residency` wins only for components it matches, while unmatched legacy settings remain effective. Legacy layerwise selectors take precedence over legacy component-offload selectors for the same component. Explicit `--dit-layerwise-offload false` makes the DiT resident unless another explicit DiT selector, such as `--dit-cpu-offload true` or `--component-residency dit=component-offload`, selects a different mode. +Explicit component placement also applies during warmup and request cleanup. Pipeline +preferences cannot keep or reload an explicitly offloaded component merely to +prepare for the next request. An explicitly `resident` component remains resident. +This applies to matching exact, group, and `all` selectors, as well as legacy +offload options. Components without an explicit placement may still use the +pipeline's warmup retention and request-end preload preferences. Selecting one component does +not disable those preferences for other components. On-demand loading and +prefetch for upcoming uses within a request are unchanged; explicit offload may +therefore add weight-transfer latency to the first real request after warmup. + Layerwise selection is strict. A native weighted component selected for `layerwise-offload` must declare its layer structure; otherwise startup fails with the unsupported component name instead of silently changing modes. Explicit non-resident placement also requires a request-time component-use declaration, so it cannot silently select a module that the pipeline does not manage. FSDP applies only to resident components. The Diffusers backend supports only pipeline-wide `all=resident` and `all=component-offload`. #### Snapshot offload diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py index a83aab6195..3b0fdf7f63 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/component_manager.py @@ -639,7 +639,9 @@ class ComponentResidencyManager: return if not force: should_keep = ( - keep_on_warmup and self.state.batch_is_warmup + keep_on_warmup + and self.state.batch_is_warmup + and self.server_args.explicit_residency_mode(use.component_name) is None ) or self._should_keep_after_use(use) if should_keep: return @@ -659,9 +661,15 @@ class ComponentResidencyManager: module = self.get_module(component_name) if module is None: continue - if self.state.batch_is_warmup and use.keep_ready_after_warmup: + # pipeline hints must not override an explicit placement policy + explicit_mode = self.server_args.explicit_residency_mode(component_name) + preferred = component_name in preferred_uses and explicit_mode is None + if ( + self.state.batch_is_warmup + and use.keep_ready_after_warmup + and explicit_mode is None + ): continue - preferred = component_name in preferred_uses if is_resident_layerwise_module(module): preferred = False keep_single_dit = self._should_keep_single_dit(component_name, module) diff --git a/python/sglang/multimodal_gen/test/unit/test_component_residency.py b/python/sglang/multimodal_gen/test/unit/test_component_residency.py index 3ea25d49af..903b89a2a1 100644 --- a/python/sglang/multimodal_gen/test/unit/test_component_residency.py +++ b/python/sglang/multimodal_gen/test/unit/test_component_residency.py @@ -1,3 +1,4 @@ +import sys from types import SimpleNamespace from unittest.mock import Mock @@ -5,6 +6,10 @@ import pytest import torch from safetensors.torch import load_file, save_file +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig +from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import ( + QwenImagePipelineConfig, +) from sglang.multimodal_gen.runtime.loader.utils import component_residency_bytes from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentResidencyManager, @@ -39,11 +44,13 @@ from sglang.multimodal_gen.runtime.post_training.weights_updater import ( _load_weights_into_module, ) from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.argparse import FlexibleArgumentParser def _server_args(*, supports_auto_residency=True): return SimpleNamespace( enable_layerwise_nvtx_marker=False, + explicit_residency_mode=lambda _: None, pipeline_config=SimpleNamespace( supports_auto_residency=supports_auto_residency, ), @@ -229,7 +236,7 @@ def test_request_tail_uses_dynamic_component_instance(): ) manager = ComponentResidencyManager( pipeline, - SimpleNamespace(enable_layerwise_nvtx_marker=False), + _server_args(), ) strategy = Mock() strategy.prefetch_for_use.return_value = False @@ -326,6 +333,134 @@ class _Stage: return self.uses +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("hint", ["preferred", "fallback", "keep"]) +@pytest.mark.parametrize( + "component_name, cli_args, explicit_mode", + [ + ("text_encoder_2", [], None), + ("text_encoder_2", ["--component-residency", "vae=component-offload"], None), + ("text_encoder_2", ["--cpu-offload-components", "vae"], None), + ( + "text_encoder_2", + ["--component-residency", "text_encoder_2=component-offload"], + "component-offload", + ), + ( + "text_encoder_2", + ["--component-residency", "text_encoder=component-offload"], + "component-offload", + ), + ( + "text_encoder_2", + ["--component-residency", "all=component-offload"], + "component-offload", + ), + ( + "text_encoder_2", + ["--component-residency", "text_encoder=snapshot-offload"], + "snapshot-offload", + ), + ("text_encoder_2", ["--text-encoder-cpu-offload", "true"], "component-offload"), + ("text_encoder_2", ["--text-encoder-cpu-offload", "false"], "resident"), + ( + "text_encoder_2", + ["--cpu-offload-components", "text_encoder"], + "component-offload", + ), + ( + "text_encoder_2", + ["--component-residency", "all=component-offload", "text_encoder=resident"], + "resident", + ), + ( + "text_encoder_2", + [ + "--component-residency", + "text_encoder=component-offload", + "text_encoder_2=resident", + ], + "resident", + ), + ( + "aux_encoder", + ["--component-residency", "aux_encoder=component-offload"], + "component-offload", + ), + ("vae", ["--vae-cpu-offload", "true"], "component-offload"), + ], +) +def test_warmup_hints_respect_explicit_residency( + monkeypatch, component_name, cli_args, explicit_mode, hint +): + monkeypatch.setattr( + PipelineConfig, "from_kwargs", lambda _: QwenImagePipelineConfig() + ) + parser = FlexibleArgumentParser() + ServerArgs.add_cli_args(parser) + argv = ["--model-path", "/unused/model", "--performance-mode", "manual", *cli_args] + monkeypatch.setattr(sys, "argv", ["sglang", *argv]) + parsed, unknown = parser.parse_known_args(argv) + args = ServerArgs.from_cli_args( + parsed, + unknown, + default_args={"text_encoder_cpu_offload": True, "vae_cpu_offload": True}, + ) + assert args.explicit_residency_mode(component_name) == explicit_mode + mode = explicit_mode or "component-offload" + assert args.residency_mode(component_name) == mode + + module = torch.nn.Linear(4, 4) + expected_weights = { + name: tensor.clone() for name, tensor in module.state_dict().items() + } + inputs = torch.randn(2, 4) + expected_output = module(inputs) + use = ComponentUse( + "encode", + component_name, + preferred_ready_after_request=hint == "preferred", + keep_ready_after_warmup=hint == "keep", + ) + stage = _Stage(use) + pipeline = SimpleNamespace( + modules={component_name: module}, + _stage_name_mapping={"encode": stage}, + component_residency_strategies={}, + ) + manager = ComponentResidencyManager(pipeline, args) + manager.refresh_pipeline(pipeline) + strategy = manager.strategy_for(component_name, module) + prepare = Mock(wraps=strategy.prepare_for_use) + monkeypatch.setattr(strategy, "prepare_for_use", prepare) + + for is_warmup in (True, False, False): + batch = SimpleNamespace(is_warmup=is_warmup) + manager.begin_request([stage], batch, args) + manager.before_stage(stage, 0, batch, args) + manager.begin_stage() + torch.testing.assert_close(module(inputs.cuda()).cpu(), expected_output) + manager.end_stage() + keep_on_warmup = is_warmup and explicit_mode is None + expected_stage_device = ( + "cuda" + if mode == "resident" or (keep_on_warmup and hint == "keep") + else "cpu" + ) + assert module.weight.device.type == expected_stage_device + prepare.reset_mock() + manager.finish_request() + if explicit_mode is not None: + prepare.assert_not_called() + expected_device = "cuda" if mode == "resident" or keep_on_warmup else "cpu" + assert module.weight.device.type == expected_device + torch.cuda.synchronize() + for name, tensor in module.state_dict().items(): + torch.testing.assert_close( + tensor.cpu(), expected_weights[name], rtol=0, atol=0 + ) + + def test_warmup_records_use_and_transition_peaks(monkeypatch): device_module = SimpleNamespace( is_available=lambda: True, diff --git a/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py b/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py index 27cfdaa64c..2bc093718b 100644 --- a/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py +++ b/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py @@ -3,6 +3,7 @@ import os import pathlib from contextlib import nullcontext from types import SimpleNamespace +from unittest.mock import Mock import pytest import torch @@ -519,6 +520,65 @@ def test_snapshot_and_layerwise_share_the_residency_managers_pin_budget(): assert budget.committed_bytes == 64 +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("explicit", [None, "canonical", "legacy"]) +@pytest.mark.parametrize("keep_after_warmup", [False, True]) +def test_explicit_layerwise_offload_disables_request_end_preload( + monkeypatch, explicit, keep_after_warmup +): + args = _server_args( + component_residency=( + {"text_encoder": "layerwise-offload"} if explicit == "canonical" else None + ), + layerwise_offload_components=["text_encoder"], + _explicit_arg_names=( + {"layerwise_offload_components"} if explicit == "legacy" else set() + ), + enable_layerwise_nvtx_marker=False, + pipeline_config=SimpleNamespace(supports_auto_residency=False), + ) + module = _NestedDummyModel() + use = ComponentUse( + "encode", + "text_encoder", + preferred_ready_after_request=True, + keep_ready_after_warmup=keep_after_warmup, + ) + stage = SimpleNamespace(component_uses=lambda *_: [use]) + pipeline = SimpleNamespace( + modules={"text_encoder": module}, + _stage_name_mapping={"encode": stage}, + component_residency_strategies={}, + ) + configure_layerwise_offload_modules(pipeline.modules, args) + manager = ComponentResidencyManager(pipeline, args) + manager.refresh_pipeline(pipeline) + strategy = manager.strategy_for("text_encoder", module) + assert isinstance(strategy, LayerwiseOffloadStrategy) + prepare = Mock(wraps=strategy.prepare_for_use) + monkeypatch.setattr(strategy, "prepare_for_use", prepare) + release = Mock(wraps=module.layerwise_offload_managers[0].release_after_use) + monkeypatch.setattr( + module.layerwise_offload_managers[0], "release_after_use", release + ) + + for is_warmup in (True, False, False): + batch = SimpleNamespace(is_warmup=is_warmup) + manager.begin_request([stage], batch, args) + manager.before_stage(stage, 0, batch, args) + manager.begin_stage() + prepare.reset_mock() + release.reset_mock() + manager.end_stage() + manager.finish_request() + if explicit is not None: + prepare.assert_not_called() + release.assert_called() + elif not (is_warmup and keep_after_warmup): + prepare.assert_called_once() + module.disable_offload() + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_layerwise_pin_lease_includes_alignment_and_survives_host_aliases(): budget = host_memory_budget.HostPinBudget( diff --git a/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py b/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py index 1bc92c02b4..f881252c2a 100644 --- a/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py +++ b/python/sglang/multimodal_gen/test/unit/test_text_encoding_cache.py @@ -232,6 +232,7 @@ def make_server_args(**kwargs): "model_path": "dummy-model", "backend": "auto", "model_id": None, + "explicit_residency_mode": lambda _: None, "pipeline_config": SimpleNamespace(text_encoder_configs=[]), } defaults.update(kwargs)