[diffusion] fix: respect explicit residency over pipeline preload hints (#41622)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-03 09:54:21 +08:00
committed by GitHub
co-authored by Mick Qian
parent a8b54f138e
commit 65c7425a56
5 changed files with 218 additions and 4 deletions
@@ -639,7 +639,9 @@ class ComponentResidencyManager:
return
if not force:
should_keep = (
keep_on_warmup and self.state.batch_is_warmup
keep_on_warmup
and self.state.batch_is_warmup
and self.server_args.explicit_residency_mode(use.component_name) is None
) or self._should_keep_after_use(use)
if should_keep:
return
@@ -659,9 +661,15 @@ class ComponentResidencyManager:
module = self.get_module(component_name)
if module is None:
continue
if self.state.batch_is_warmup and use.keep_ready_after_warmup:
# pipeline hints must not override an explicit placement policy
explicit_mode = self.server_args.explicit_residency_mode(component_name)
preferred = component_name in preferred_uses and explicit_mode is None
if (
self.state.batch_is_warmup
and use.keep_ready_after_warmup
and explicit_mode is None
):
continue
preferred = component_name in preferred_uses
if is_resident_layerwise_module(module):
preferred = False
keep_single_dit = self._should_keep_single_dit(component_name, module)
@@ -1,3 +1,4 @@
import sys
from types import SimpleNamespace
from unittest.mock import Mock
@@ -5,6 +6,10 @@ import pytest
import torch
from safetensors.torch import load_file, save_file
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import (
QwenImagePipelineConfig,
)
from sglang.multimodal_gen.runtime.loader.utils import component_residency_bytes
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentResidencyManager,
@@ -39,11 +44,13 @@ from sglang.multimodal_gen.runtime.post_training.weights_updater import (
_load_weights_into_module,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.argparse import FlexibleArgumentParser
def _server_args(*, supports_auto_residency=True):
return SimpleNamespace(
enable_layerwise_nvtx_marker=False,
explicit_residency_mode=lambda _: None,
pipeline_config=SimpleNamespace(
supports_auto_residency=supports_auto_residency,
),
@@ -229,7 +236,7 @@ def test_request_tail_uses_dynamic_component_instance():
)
manager = ComponentResidencyManager(
pipeline,
SimpleNamespace(enable_layerwise_nvtx_marker=False),
_server_args(),
)
strategy = Mock()
strategy.prefetch_for_use.return_value = False
@@ -326,6 +333,134 @@ class _Stage:
return self.uses
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("hint", ["preferred", "fallback", "keep"])
@pytest.mark.parametrize(
"component_name, cli_args, explicit_mode",
[
("text_encoder_2", [], None),
("text_encoder_2", ["--component-residency", "vae=component-offload"], None),
("text_encoder_2", ["--cpu-offload-components", "vae"], None),
(
"text_encoder_2",
["--component-residency", "text_encoder_2=component-offload"],
"component-offload",
),
(
"text_encoder_2",
["--component-residency", "text_encoder=component-offload"],
"component-offload",
),
(
"text_encoder_2",
["--component-residency", "all=component-offload"],
"component-offload",
),
(
"text_encoder_2",
["--component-residency", "text_encoder=snapshot-offload"],
"snapshot-offload",
),
("text_encoder_2", ["--text-encoder-cpu-offload", "true"], "component-offload"),
("text_encoder_2", ["--text-encoder-cpu-offload", "false"], "resident"),
(
"text_encoder_2",
["--cpu-offload-components", "text_encoder"],
"component-offload",
),
(
"text_encoder_2",
["--component-residency", "all=component-offload", "text_encoder=resident"],
"resident",
),
(
"text_encoder_2",
[
"--component-residency",
"text_encoder=component-offload",
"text_encoder_2=resident",
],
"resident",
),
(
"aux_encoder",
["--component-residency", "aux_encoder=component-offload"],
"component-offload",
),
("vae", ["--vae-cpu-offload", "true"], "component-offload"),
],
)
def test_warmup_hints_respect_explicit_residency(
monkeypatch, component_name, cli_args, explicit_mode, hint
):
monkeypatch.setattr(
PipelineConfig, "from_kwargs", lambda _: QwenImagePipelineConfig()
)
parser = FlexibleArgumentParser()
ServerArgs.add_cli_args(parser)
argv = ["--model-path", "/unused/model", "--performance-mode", "manual", *cli_args]
monkeypatch.setattr(sys, "argv", ["sglang", *argv])
parsed, unknown = parser.parse_known_args(argv)
args = ServerArgs.from_cli_args(
parsed,
unknown,
default_args={"text_encoder_cpu_offload": True, "vae_cpu_offload": True},
)
assert args.explicit_residency_mode(component_name) == explicit_mode
mode = explicit_mode or "component-offload"
assert args.residency_mode(component_name) == mode
module = torch.nn.Linear(4, 4)
expected_weights = {
name: tensor.clone() for name, tensor in module.state_dict().items()
}
inputs = torch.randn(2, 4)
expected_output = module(inputs)
use = ComponentUse(
"encode",
component_name,
preferred_ready_after_request=hint == "preferred",
keep_ready_after_warmup=hint == "keep",
)
stage = _Stage(use)
pipeline = SimpleNamespace(
modules={component_name: module},
_stage_name_mapping={"encode": stage},
component_residency_strategies={},
)
manager = ComponentResidencyManager(pipeline, args)
manager.refresh_pipeline(pipeline)
strategy = manager.strategy_for(component_name, module)
prepare = Mock(wraps=strategy.prepare_for_use)
monkeypatch.setattr(strategy, "prepare_for_use", prepare)
for is_warmup in (True, False, False):
batch = SimpleNamespace(is_warmup=is_warmup)
manager.begin_request([stage], batch, args)
manager.before_stage(stage, 0, batch, args)
manager.begin_stage()
torch.testing.assert_close(module(inputs.cuda()).cpu(), expected_output)
manager.end_stage()
keep_on_warmup = is_warmup and explicit_mode is None
expected_stage_device = (
"cuda"
if mode == "resident" or (keep_on_warmup and hint == "keep")
else "cpu"
)
assert module.weight.device.type == expected_stage_device
prepare.reset_mock()
manager.finish_request()
if explicit_mode is not None:
prepare.assert_not_called()
expected_device = "cuda" if mode == "resident" or keep_on_warmup else "cpu"
assert module.weight.device.type == expected_device
torch.cuda.synchronize()
for name, tensor in module.state_dict().items():
torch.testing.assert_close(
tensor.cpu(), expected_weights[name], rtol=0, atol=0
)
def test_warmup_records_use_and_transition_peaks(monkeypatch):
device_module = SimpleNamespace(
is_available=lambda: True,
@@ -3,6 +3,7 @@ import os
import pathlib
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
@@ -519,6 +520,65 @@ def test_snapshot_and_layerwise_share_the_residency_managers_pin_budget():
assert budget.committed_bytes == 64
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("explicit", [None, "canonical", "legacy"])
@pytest.mark.parametrize("keep_after_warmup", [False, True])
def test_explicit_layerwise_offload_disables_request_end_preload(
monkeypatch, explicit, keep_after_warmup
):
args = _server_args(
component_residency=(
{"text_encoder": "layerwise-offload"} if explicit == "canonical" else None
),
layerwise_offload_components=["text_encoder"],
_explicit_arg_names=(
{"layerwise_offload_components"} if explicit == "legacy" else set()
),
enable_layerwise_nvtx_marker=False,
pipeline_config=SimpleNamespace(supports_auto_residency=False),
)
module = _NestedDummyModel()
use = ComponentUse(
"encode",
"text_encoder",
preferred_ready_after_request=True,
keep_ready_after_warmup=keep_after_warmup,
)
stage = SimpleNamespace(component_uses=lambda *_: [use])
pipeline = SimpleNamespace(
modules={"text_encoder": module},
_stage_name_mapping={"encode": stage},
component_residency_strategies={},
)
configure_layerwise_offload_modules(pipeline.modules, args)
manager = ComponentResidencyManager(pipeline, args)
manager.refresh_pipeline(pipeline)
strategy = manager.strategy_for("text_encoder", module)
assert isinstance(strategy, LayerwiseOffloadStrategy)
prepare = Mock(wraps=strategy.prepare_for_use)
monkeypatch.setattr(strategy, "prepare_for_use", prepare)
release = Mock(wraps=module.layerwise_offload_managers[0].release_after_use)
monkeypatch.setattr(
module.layerwise_offload_managers[0], "release_after_use", release
)
for is_warmup in (True, False, False):
batch = SimpleNamespace(is_warmup=is_warmup)
manager.begin_request([stage], batch, args)
manager.before_stage(stage, 0, batch, args)
manager.begin_stage()
prepare.reset_mock()
release.reset_mock()
manager.end_stage()
manager.finish_request()
if explicit is not None:
prepare.assert_not_called()
release.assert_called()
elif not (is_warmup and keep_after_warmup):
prepare.assert_called_once()
module.disable_offload()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_layerwise_pin_lease_includes_alignment_and_survives_host_aliases():
budget = host_memory_budget.HostPinBudget(
@@ -232,6 +232,7 @@ def make_server_args(**kwargs):
"model_path": "dummy-model",
"backend": "auto",
"model_id": None,
"explicit_residency_mode": lambda _: None,
"pipeline_config": SimpleNamespace(text_encoder_configs=[]),
}
defaults.update(kwargs)