[diffusion] feat: support permanent lifetime for layerwise resident layers (#40599)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-09-23 13:49:09 +08:00
committed by GitHub
co-authored by Mick Qian Claude Fable 5.1
parent 28be39f72e
commit a3d11f9d95
10 changed files with 639 additions and 28 deletions
@@ -47,6 +47,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.host_memory_budget i
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
LAYERWISE_OFFLOAD_ALL_COMPONENTS,
LAYERWISE_OFFLOAD_DIT_GROUP,
RESIDENCY_LIFETIME_FORWARD,
RESIDENCY_LIFETIME_PERMANENT,
RESIDENCY_LIFETIMES,
RESIDENCY_POLICIES,
RESIDENCY_POLICY_LEADING,
RESIDENCY_POLICY_STRIDED,
@@ -1008,6 +1011,7 @@ class LayerwiseOffloadManager:
resident_layers: int = 0,
initialize: bool = True,
residency_policy: str = RESIDENCY_POLICY_LEADING,
residency_lifetime: str = RESIDENCY_LIFETIME_FORWARD,
pin_budget: HostPinBudget | None = None,
pin_component_name: str = "layerwise offload",
) -> None:
@@ -1045,9 +1049,34 @@ class LayerwiseOffloadManager:
self._resident_set = frozenset(range(self.num_layers)) - set(
self._streamed_order
)
# Armed on the first denoise forward, so that the load-time prefetch below
# does not pin the whole resident set before the DiT is the active component.
self._residency_active = False
if residency_lifetime not in RESIDENCY_LIFETIMES:
raise ValueError(
f"unknown residency lifetime {residency_lifetime!r}, expected one "
f"of {RESIDENCY_LIFETIMES}"
)
self.residency_lifetime = residency_lifetime
# Placed on the device at load and never released: no host copy, no
# per-request transfer. Empty under `forward`, where the resident set is
# pinned when the component starts running and released when it finishes.
self._permanent_set = (
self._resident_set
if residency_lifetime == RESIDENCY_LIFETIME_PERMANENT
else frozenset()
)
if (
residency_lifetime == RESIDENCY_LIFETIME_PERMANENT
and not self._resident_set
):
logger.warning(
"Layerwise offload: %s asks for permanent residency with no resident "
"layers, so nothing is kept and every layer streams.",
pin_component_name,
)
# Under `forward`, armed on the first denoise forward so that the
# load-time prefetch below does not pin the whole resident set before the
# DiT is the active component. Permanent layers are on the device from
# load, so that set is armed from the start.
self._residency_active = bool(self._permanent_set)
# True while load_all_layers materializes every layer for a resident
# placement; every mapped source is then read exactly once.
self._materializing_all = False
@@ -1270,6 +1299,9 @@ class LayerwiseOffloadManager:
first, in streamed order, which is also deterministic. Unpinning a
resident layer costs one possibly-faulting arming copy per request and
buys a whole layer's worth of per-step overlap.
Permanent resident layers are not in ``layer_groups``: they live on the
device with no host copy, so there is nothing here to host.
"""
totals, mapped = self._layer_byte_totals(layer_groups)
if host_copies_are_redundant():
@@ -1380,6 +1412,19 @@ class LayerwiseOffloadManager:
local_tensor.dtype, []
).append((name, tensor))
permanent_groups = {
layer_idx: groups
for layer_idx, groups in layer_groups.items()
if layer_idx in self._permanent_set
}
if permanent_groups:
self._place_permanent_layers(permanent_groups)
layer_groups = {
layer_idx: groups
for layer_idx, groups in layer_groups.items()
if layer_idx not in self._permanent_set
}
layer_hosting, untracked_bytes = self._plan_layer_hosting(layer_groups)
try:
for storage in self._initialize_host_stores(layer_groups, layer_hosting):
@@ -1389,6 +1434,54 @@ class LayerwiseOffloadManager:
# failed allocations have no storage finalizer to return their allowance
self._pin_budget.release(untracked_bytes)
def _copy_to_device_keeping_layout(
self, local_weight: torch.Tensor
) -> torch.Tensor:
"""One device copy of a weight, with a strided view's layout kept.
Same reason as the strided host store: a transposed FP8 view has to
keep its layout, and `.to()` does not promise that.
"""
if local_weight.is_contiguous():
return local_weight.to(self.device, non_blocking=False)
device_tensor = torch.empty_strided(
size=local_weight.shape,
stride=local_weight.stride(),
dtype=local_weight.dtype,
device=self.device,
)
device_tensor.copy_(local_weight, non_blocking=False)
return device_tensor
def _place_permanent_layers(self, layer_groups: Dict) -> None:
"""Move the permanent resident layers to the device, once, with no host store.
They are never released, so a host copy would be a reload source for a
reload that never comes; dropping it is the point. Doing this before
the hosting plan also keeps these bytes out of the pin budget, which is
for layers that stream.
"""
placed_bytes = 0
with torch.inference_mode(False), torch.no_grad():
for layer_idx, dtype_to_params in layer_groups.items():
for weights in dtype_to_params.values():
for _, weight in weights:
device_tensor = self._copy_to_device_keeping_layout(
self._to_local_tensor(weight)
)
placed_bytes += (
device_tensor.numel() * device_tensor.element_size()
)
weight.data = self._wrap_for_target(weight, device_tensor)
self._gpu_layers.add(layer_idx)
logger.info(
"Layerwise offload: %s placed %d permanent resident layers (%.2f GiB) "
"on the device at load; no host copy is kept.",
self._pin_component_name,
len(layer_groups),
placed_bytes / (1 << 30),
)
def _initialize_host_stores(
self, layer_groups: Dict, layer_hosting: Dict[int, str]
) -> Iterator[torch.UntypedStorage]:
@@ -1519,8 +1612,9 @@ class LayerwiseOffloadManager:
self._consolidated_cpu_weights[layer_idx][dtype] = cpu_buffer
def _finalize_initialization(self) -> None:
# prefetch the head of the stream for warm-up; residency is not armed
# yet, so this is layer 0 regardless of policy
# prefetch the head of the stream for warm-up. Under `forward` residency
# is not armed yet, so this is layer 0 regardless of policy; permanent
# layers are already on the device, so the head is the first streamed one.
self.prepare_for_next_req(non_blocking=False)
self.register_forward_hooks()
@@ -1554,6 +1648,14 @@ class LayerwiseOffloadManager:
layer_idx = self._match_layer_idx(name)
if layer_idx is None or layer_idx >= self.num_layers:
continue
if layer_idx in self._permanent_set:
# Placed once, no CPU copy kept.
tensor.data = self._wrap_for_target(
tensor,
self._copy_to_device_keeping_layout(self._to_local_tensor(tensor)),
)
self._gpu_layers.add(layer_idx)
continue
local_tensor = self._to_local_tensor(tensor).detach()
cpu_tensor = (
local_tensor
@@ -1583,8 +1685,8 @@ class LayerwiseOffloadManager:
self._release_unneeded_streamed_layers(keep=set(self._head_of_stream()))
# The resident set first: it has to be there for the whole step, and the
# caller decides whether to block on it.
for layer_idx in sorted(self._retained_set):
# caller decides whether to block on it. Permanent layers never left.
for layer_idx in sorted(self._retained_set - self._permanent_set):
self.prefetch_layer(layer_idx, non_blocking=non_blocking)
if not non_blocking and self.copy_stream is not None:
torch.get_device_module().current_stream().wait_stream(self.copy_stream)
@@ -1603,7 +1705,8 @@ class LayerwiseOffloadManager:
@property
def holds_residents(self) -> bool:
"""True if this manager keeps a resident layer set beyond the streaming
prefetch window, so it must be denoise-stage-scoped."""
prefetch window. Under `forward` that set is released when the component
finishes running; under `permanent` it never is."""
return self.enabled and self.resident_layers > 0
@property
@@ -2096,6 +2199,10 @@ class LayerwiseOffloadManager:
"""
if not self.enabled or self.device is None:
return
# No host store to fall back to. `force` ends a forward-lifetime set;
# it does not apply here.
if layer_idx in self._permanent_set:
return
if not force and layer_idx in self._retained_set:
return
@@ -2725,7 +2832,7 @@ class LayerwiseOffloadableModuleMixin:
named_modules = dict(self.named_modules())
layer_specs = []
# `--dit-*` is the group default these fall back to, not a scope.
prefetch_value, resident_value, residency_policy = (
prefetch_value, resident_value, residency_policy, residency_lifetime = (
server_args.layerwise_tuning_for(
component_name,
dit_group=self.layerwise_offload_dit_group_enabled,
@@ -2799,6 +2906,7 @@ class LayerwiseOffloadableModuleMixin:
resident_layers=resident_layers,
initialize=False,
residency_policy=residency_policy,
residency_lifetime=residency_lifetime,
)
self.layerwise_offload_managers.append(manager)
@@ -2843,6 +2951,9 @@ class LayerwiseOffloadableModuleMixin:
for value in sorted({manager.prefetch_size for manager in managers})
)
policies = ", ".join(sorted({manager.residency_policy for manager in managers}))
lifetimes = ", ".join(
sorted({manager.residency_lifetime for manager in managers})
)
total_layers = sum(manager.num_layers for manager in managers)
resident_layers = sum(manager.resident_layers for manager in managers)
if envs.SGLANG_DIFFUSION_DEBUG_HOST_MEMORY:
@@ -2853,7 +2964,7 @@ class LayerwiseOffloadableModuleMixin:
log_anon_vmas(f"layerwise offload ready for {component_name}")
logger.info(
"Layerwise offload ready for %s in %.2fs: groups=%d, layers=%d, "
"prefetch/group=%s, resident=%d/%d (per request), policy=%s",
"prefetch/group=%s, resident=%d/%d (%s), policy=%s",
component_label,
perf_counter() - started_at,
len(managers),
@@ -2861,6 +2972,7 @@ class LayerwiseOffloadableModuleMixin:
prefetch_sizes,
resident_layers,
total_layers,
lifetimes,
policies,
)
@@ -14,6 +14,13 @@ RESIDENCY_POLICY_LEADING = "leading"
RESIDENCY_POLICY_STRIDED = "strided"
RESIDENCY_POLICIES = (RESIDENCY_POLICY_LEADING, RESIDENCY_POLICY_STRIDED)
# How long those layers stay. `forward`: placed when the component starts
# running its layers for a request, released when it finishes, host copy kept.
# `permanent`: placed at load, no host copy, never released.
RESIDENCY_LIFETIME_FORWARD = "forward"
RESIDENCY_LIFETIME_PERMANENT = "permanent"
RESIDENCY_LIFETIMES = (RESIDENCY_LIFETIME_FORWARD, RESIDENCY_LIFETIME_PERMANENT)
# Components whose layerwise policy has been validated as a better default than
# component-level CPU offload when the user has not pinned their placement.
LAYERWISE_OFFLOAD_DEFAULT_GROUP_COMPONENTS = (
@@ -48,6 +48,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_co
LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP,
LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP,
LAYERWISE_OFFLOAD_VAE_GROUP,
RESIDENCY_LIFETIME_FORWARD,
RESIDENCY_LIFETIME_PERMANENT,
RESIDENCY_LIFETIMES,
RESIDENCY_POLICIES,
RESIDENCY_POLICY_LEADING,
cpu_offload_component_matches,
@@ -418,7 +421,10 @@ class ServerArgs(DisaggServerArgsMixin):
dit_layerwise_resident_layers: float = 0.0
# Which layers those are: the leading ones, or spread evenly over the stack.
dit_layerwise_residency_policy: str = RESIDENCY_POLICY_LEADING
# Per-component overrides of the three knobs above; an entry wins for that
# How long they stay: while the DiT runs for a request, or for the life of
# the server.
dit_layerwise_residency_lifetime: str = RESIDENCY_LIFETIME_FORWARD
# Per-component overrides of the four knobs above; an entry wins for that
# component.
layerwise_prefetch_size: dict[str, float] | str | None = field(default_factory=dict)
layerwise_resident_layers: dict[str, float] | str | None = field(
@@ -427,6 +433,9 @@ class ServerArgs(DisaggServerArgsMixin):
layerwise_residency_policy: dict[str, str] | str | None = field(
default_factory=dict
)
layerwise_residency_lifetime: dict[str, str] | str | None = field(
default_factory=dict
)
offload_during_compile: bool = True
text_encoder_cpu_offload: bool | None = None
image_encoder_cpu_offload: bool | None = None
@@ -1209,8 +1218,8 @@ class ServerArgs(DisaggServerArgsMixin):
def layerwise_tuning_for(
self, component_name: str | None, *, dit_group: bool
) -> tuple[float, float, str]:
"""Prefetch size, resident layers and residency policy for one component."""
) -> tuple[float, float, str, str]:
"""Prefetch size, resident layers, residency policy and lifetime for one component."""
prefetch_map = self._parse_component_value_map(
self.layerwise_prefetch_size, option="--layerwise-prefetch-size"
)
@@ -1220,6 +1229,9 @@ class ServerArgs(DisaggServerArgsMixin):
policy_map = self._parse_component_value_map(
self.layerwise_residency_policy, option="--layerwise-residency-policy"
)
lifetime_map = self._parse_component_value_map(
self.layerwise_residency_lifetime, option="--layerwise-residency-lifetime"
)
def _pick(mapping: dict[str, str], group_default, aux_default):
if component_name is not None and component_name in mapping:
@@ -1240,7 +1252,19 @@ class ServerArgs(DisaggServerArgsMixin):
f"unknown residency policy {policy!r} for component "
f"{component_name!r}, expected one of {RESIDENCY_POLICIES}"
)
return prefetch, resident, policy
lifetime = str(
_pick(
lifetime_map,
self.dit_layerwise_residency_lifetime,
RESIDENCY_LIFETIME_FORWARD,
)
)
if lifetime not in RESIDENCY_LIFETIMES:
raise ValueError(
f"unknown residency lifetime {lifetime!r} for component "
f"{component_name!r}, expected one of {RESIDENCY_LIFETIMES}"
)
return prefetch, resident, policy, lifetime
@staticmethod
def _parse_component_attention_backend_map(
@@ -2551,8 +2575,9 @@ class ServerArgs(DisaggServerArgsMixin):
"streaming). Between 0.0 and 1.0 = ratio of layers; >= 1 = absolute "
"count. Unlike raising the prefetch size, resident layers are transferred "
"once per request rather than once per step, so this trades VRAM for "
"lower denoise latency when memory is available. They are released when "
"the request finishes, not kept for the life of the server.",
"lower denoise latency when memory is available. How long they stay on "
"the GPU is --dit-layerwise-residency-lifetime: released when the "
"request finishes by default, or kept for the life of the server.",
)
parser.add_argument(
"--layerwise-prefetch-size",
@@ -2577,7 +2602,8 @@ class ServerArgs(DisaggServerArgsMixin):
"a DiT across the denoise steps, a video VAE across the latent chunks. "
"A component that runs its layers once per request, such as a text "
"encoder, transfers the whole set again every request, so setting "
"this for one has no effect.",
"this for one has no effect unless --layerwise-residency-lifetime "
"keeps the layers for the life of the server.",
)
parser.add_argument(
"--layerwise-residency-policy",
@@ -2587,6 +2613,17 @@ class ServerArgs(DisaggServerArgsMixin):
"component=value entries, e.g. --layerwise-residency-policy "
"text_encoder=strided.",
)
parser.add_argument(
"--layerwise-residency-lifetime",
type=str,
default=None,
help="Per-component override of --dit-layerwise-residency-lifetime, as "
"component=value entries, e.g. --layerwise-residency-lifetime "
"transformer=permanent,video_vae=forward. 'forward' releases a "
"component's resident layers when it finishes running for a request; "
"'permanent' places them on the GPU at load time and never releases "
"them.",
)
parser.add_argument(
"--dit-layerwise-residency-policy",
type=str,
@@ -2601,6 +2638,21 @@ class ServerArgs(DisaggServerArgsMixin):
"memory-bound compute -- the transfers stop competing with it for "
"L2 and DRAM bandwidth, which is where the gain comes from.",
)
parser.add_argument(
"--dit-layerwise-residency-lifetime",
type=str,
choices=RESIDENCY_LIFETIMES,
default=ServerArgs.dit_layerwise_residency_lifetime,
help="How long the DiT layers kept by --dit-layerwise-resident-layers "
"stay on the GPU. 'forward' (default): placed when the DiT starts its "
"denoise steps for a request and released when they finish, so they "
"are transferred once per request and keep a copy in host memory. "
"'permanent': placed at load time, no host copy, never released -- "
"one transfer for the life of the server. Choose 'permanent' when the "
"resident layers of every component fit on the GPU together; with "
"'forward' only the running component holds its set, so the peak is "
"the largest set rather than their sum.",
)
# offload flags
parser.add_argument(
@@ -3625,6 +3677,27 @@ class ServerArgs(DisaggServerArgsMixin):
"so there is no resident set to place."
)
if self.dit_layerwise_residency_lifetime not in RESIDENCY_LIFETIMES:
raise ValueError(
f"Invalid --dit-layerwise-residency-lifetime "
f"{self.dit_layerwise_residency_lifetime!r}; expected one of "
f"{RESIDENCY_LIFETIMES}."
)
if self.dit_layerwise_residency_lifetime == RESIDENCY_LIFETIME_PERMANENT:
if not self.is_dit_layerwise_offload_selected:
logger.warning(
"--dit-layerwise-residency-lifetime has no effect because the "
"DiT is not layerwise-offloaded. It only applies together with "
"--dit-layerwise-offload (or 'dit' in "
"--layerwise-offload-components)."
)
elif self.dit_layerwise_resident_layers <= 0:
logger.warning(
"--dit-layerwise-residency-lifetime has no effect because "
"--dit-layerwise-resident-layers is 0: there is no resident "
"set to keep."
)
# validate layerwise offload conflicts
if envs.SGLANG_CACHE_DIT_ENABLED and self.use_fsdp_inference:
if self.is_arg_explicitly_set("use_fsdp_inference"):
@@ -303,6 +303,27 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
extras={"enable_teacache": True},
),
),
# The text encoder's resident set placed at load and never released. The
# guard is TextEncodingStage: with the set on the device the stage is
# compute only, and if the layers ever go back to being transferred per
# request the stage regains that whole transfer, well past the
# non_denoise_stage tolerance. load_peak_vram carries the placed set, so
# it also pins that the placement happens at load, not on the first use.
# The encoder is already layerwise under the default component set.
# No consistency check: the lifetime moves weights, not math, so the
# output is the base case's and that case already guards it.
DiffusionTestCase(
"wan2_1_t2v_1.3b_encoder_permanent_residents",
DiffusionServerArgs(
model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
extras=[
"--layerwise-resident-layers text_encoder=0.8 "
"--layerwise-residency-lifetime text_encoder=permanent",
],
),
DiffusionSamplingParams(prompt=T2V_PROMPT),
run_consistency_check=False,
),
# Frame interpolation (2× / exp=1)
# Uses the same 1.3B model already in the suite;
DiffusionTestCase(
@@ -1230,6 +1230,78 @@
"runtime_peak_allocated_mb": 9730.0,
"estimated_full_test_time_s": 126.0
},
"wan2_1_t2v_1.3b_encoder_permanent_residents": {
"expected_load_ms": 29198.13,
"stages_ms": {
"InputValidationStage": 0.04,
"TextEncodingStage": 171.56,
"LatentPreparationStage": 0.25,
"TimestepPreparationStage": 1.63,
"DenoisingStage": 7415.91,
"DecodingStage": 352.12
},
"denoise_step_ms": {
"0": 105.75,
"1": 126.42,
"2": 147.21,
"3": 146.06,
"4": 146.19,
"5": 147.25,
"6": 162.75,
"7": 150.95,
"8": 148.83,
"9": 148.53,
"10": 146.13,
"11": 146.61,
"12": 153.93,
"13": 154.16,
"14": 149.9,
"15": 149.11,
"16": 150.65,
"17": 147.48,
"18": 150.69,
"19": 152.47,
"20": 152.68,
"21": 147.5,
"22": 148.45,
"23": 146.57,
"24": 145.73,
"25": 148.87,
"26": 151.59,
"27": 150.64,
"28": 149.1,
"29": 148.96,
"30": 149.04,
"31": 148.03,
"32": 149.99,
"33": 151.77,
"34": 150.24,
"35": 149.52,
"36": 149.38,
"37": 149.0,
"38": 147.44,
"39": 150.58,
"40": 150.23,
"41": 151.2,
"42": 149.01,
"43": 150.49,
"44": 148.42,
"45": 147.62,
"46": 149.83,
"47": 150.52,
"48": 150.02,
"49": 148.03
},
"expected_e2e_ms": 8010.27,
"expected_avg_denoise_ms": 148.23,
"expected_median_denoise_ms": 149.07,
"load_peak_vram_mb": 17792.0,
"runtime_peak_vram_mb": 26930.0,
"warmup_peak_vram_mb": 27492.0,
"load_peak_allocated_mb": 17625.61,
"runtime_peak_allocated_mb": 23676.07,
"estimated_full_test_time_s": 130.0
},
"wan2_1_t2v_1.3b": {
"expected_load_ms": 30234.05,
"stages_ms": {
@@ -49,8 +49,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
get_layerwise_offload_component_names_for_pipeline,
is_layerwise_offloaded_module,
is_resident_layerwise_module,
iter_materialized_weights,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
RESIDENCY_LIFETIME_FORWARD,
RESIDENCY_LIFETIME_PERMANENT,
RESIDENCY_POLICY_LEADING,
RESIDENCY_POLICY_STRIDED,
)
@@ -224,9 +227,11 @@ def _server_args(**kwargs):
dit_offload_prefetch_size=1,
dit_layerwise_resident_layers=0.0,
dit_layerwise_residency_policy=RESIDENCY_POLICY_LEADING,
dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_FORWARD,
layerwise_prefetch_size={},
layerwise_resident_layers={},
layerwise_residency_policy={},
layerwise_residency_lifetime={},
pin_cpu_memory=False,
# the pin budget ranks candidates by bytes x steps, and reads the step
# count off the pipeline's sampling defaults
@@ -986,6 +991,7 @@ def _resident_manager(
prefetch_size=1,
resident_layers=0,
residency_policy=RESIDENCY_POLICY_LEADING,
residency_lifetime=RESIDENCY_LIFETIME_FORWARD,
):
return LayerwiseOffloadManager(
model=model,
@@ -996,6 +1002,7 @@ def _resident_manager(
prefetch_size=prefetch_size,
resident_layers=resident_layers,
residency_policy=residency_policy,
residency_lifetime=residency_lifetime,
)
@@ -1242,11 +1249,11 @@ def test_configure_logs_component_start_and_completion(monkeypatch):
"Configuring layerwise offload for transformer (_ResidentComponent): "
"blocks (8 layers)"
)
# "(per request)" is the point of this line: the set is re-established every
# request, not pinned for the server's lifetime.
# The lifetime in parentheses is the point of this line: `forward` says the
# set is re-established every request, not pinned for the server's lifetime.
assert logs[-1] == (
"Layerwise offload ready for transformer (_ResidentComponent) in 2.35s: "
"groups=1, layers=8, prefetch/group=2, resident=3/8 (per request), policy=leading"
"groups=1, layers=8, prefetch/group=2, resident=3/8 (forward), policy=leading"
)
@@ -2052,11 +2059,13 @@ def test_layerwise_tuning_defaults_match_the_group():
3.0,
20.0,
RESIDENCY_POLICY_STRIDED,
RESIDENCY_LIFETIME_FORWARD,
)
assert args.layerwise_tuning_for("text_encoder", dit_group=False) == (
0.0,
0.0,
RESIDENCY_POLICY_LEADING,
RESIDENCY_LIFETIME_FORWARD,
)
@@ -2073,12 +2082,14 @@ def test_layerwise_tuning_per_component_entry_wins():
2.0,
4.0,
RESIDENCY_POLICY_STRIDED,
RESIDENCY_LIFETIME_FORWARD,
)
# an entry for one component leaves every other component alone
assert args.layerwise_tuning_for("vae", dit_group=False) == (
0.0,
0.0,
RESIDENCY_POLICY_LEADING,
RESIDENCY_LIFETIME_FORWARD,
)
assert args.layerwise_tuning_for("transformer", dit_group=True)[:2] == (3.0, 20.0)
@@ -2582,3 +2593,230 @@ def test_small_pinned_stores_keep_using_the_pool(monkeypatch):
tensor = layerwise_offload_mod._pinned_empty(1024, dtype=torch.float32)
assert tensor.is_pinned()
assert len(pooled) == 1
# --- permanent residency lifetime -------------------------------------------
def _permanent_manager(model, *, num_layers, resident_layers, prefetch_size=1):
return _resident_manager(
model,
num_layers=num_layers,
prefetch_size=prefetch_size,
resident_layers=resident_layers,
residency_lifetime=RESIDENCY_LIFETIME_PERMANENT,
)
def test_permanent_residents_are_placed_at_load_without_a_host_store(monkeypatch):
"""`permanent` puts the resident set on the device in the constructor.
Not on the first forward, not during warmup: at load. And it keeps no host
copy of those layers -- there is nothing to reload them from because
nothing ever releases them.
"""
_patch_fake_device(monkeypatch)
model = _MultiBlockModel(6)
originals = [model.blocks[i].weight.detach().clone() for i in range(6)]
manager = _permanent_manager(model, num_layers=6, resident_layers=3)
assert manager._permanent_set == frozenset({0, 1, 2})
# Armed from load: the forward-lifetime set waits for the first forward.
assert manager._residency_active is True
assert manager._retained_layers == 3
assert {0, 1, 2} <= manager._gpu_layers
for i in range(3):
assert i not in manager._weight_metadata
assert not manager._consolidated_cpu_weights.get(i)
assert torch.equal(model.blocks[i].weight, originals[i])
# a strided view keeps its layout, as the strided host store would have
assert model.blocks[i].weight.stride() == originals[i].stride()
# Streamed layers take the ordinary path: a host store, and a placeholder
# until prefetched. The head of the stream is now layer 3, not layer 0.
assert 3 in manager._gpu_layers
for i in (4, 5):
assert i in manager._weight_metadata
assert tuple(model.blocks[i].weight.shape) == (1,)
def test_permanent_residents_survive_every_release_path(monkeypatch):
_patch_fake_device(monkeypatch)
manager = _permanent_manager(_MultiBlockModel(6), num_layers=6, resident_layers=3)
manager.prefetch_layer(4, non_blocking=False)
assert {0, 1, 2, 3, 4} <= manager._gpu_layers
manager.release_layer(0, force=True)
assert 0 in manager._gpu_layers
manager.release_all()
assert manager._gpu_layers == {0, 1, 2}
# Nothing to write back and nothing to write it to; must be a no-op.
manager.sync_all_layers_to_cpu()
assert manager._gpu_layers == {0, 1, 2}
def test_prepare_does_not_touch_permanent_residents(monkeypatch):
_patch_fake_device(monkeypatch)
model = _MultiBlockModel(6)
manager = _permanent_manager(model, num_layers=6, resident_layers=3)
manager.release_all()
before = [model.blocks[i].weight.data_ptr() for i in range(3)]
log = _record_prepare(manager, monkeypatch)
manager.prepare_for_next_req(non_blocking=False)
prefetched = [entry[1] for entry in log if entry[0] == "prefetch"]
assert not set(prefetched) & {0, 1, 2}
assert 3 in prefetched
assert [model.blocks[i].weight.data_ptr() for i in range(3)] == before
def test_permanent_residents_survive_disable_and_enable_offload(monkeypatch):
"""The compile-warmup and LoRA paths toggle offload off and on again.
Off loads every layer; on syncs the loaded layers back to their host
stores and releases them. Permanent layers have no host store and must
come out of that round trip untouched, with the streamed layer released
as before.
"""
_patch_fake_device(monkeypatch)
model = _MixinModel()
model.configure_layerwise_offload(
_server_args(
dit_layerwise_offload=True,
dit_layerwise_resident_layers=2,
dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT,
)
)
(manager,) = model.layerwise_offload_managers
assert manager._permanent_set == frozenset({0, 1})
expected = torch.arange(9, dtype=torch.float32).reshape(3, 3)
model.disable_offload()
assert not is_layerwise_offloaded_module(model)
model.enable_offload()
assert is_layerwise_offloaded_module(model)
assert {0, 1} <= manager._gpu_layers
for i in (0, 1):
assert torch.equal(model.blocks[i].weight, expected)
assert tuple(model.blocks[2].weight.shape) == (1,)
def test_iter_materialized_weights_reads_permanent_residents_from_the_module(
monkeypatch,
):
_patch_fake_device(monkeypatch)
model = _MixinModel()
model.configure_layerwise_offload(
_server_args(
dit_layerwise_offload=True,
dit_layerwise_resident_layers=2,
dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT,
)
)
expected = torch.arange(9, dtype=torch.float32).reshape(3, 3)
materialized = dict(iter_materialized_weights(model))
assert sorted(materialized) == sorted(name for name, _ in model.named_parameters())
# permanent: the module's own tensor; streamed: rebuilt from the host store
assert torch.equal(materialized["blocks.0.weight"], expected)
assert torch.equal(materialized["blocks.2.weight"], expected)
def test_permanent_residents_stay_out_of_the_host_pin_plan(monkeypatch):
_patch_fake_device(monkeypatch)
planned = {}
original = LayerwiseOffloadManager._plan_layer_hosting
def spy(self, layer_groups):
planned["layers"] = sorted(layer_groups)
return original(self, layer_groups)
monkeypatch.setattr(LayerwiseOffloadManager, "_plan_layer_hosting", spy)
_permanent_manager(_MultiBlockModel(6), num_layers=6, resident_layers=3)
assert planned["layers"] == [3, 4, 5]
def test_permanent_lifetime_with_no_resident_layers_warns_and_streams(monkeypatch):
_patch_fake_device(monkeypatch)
warnings = []
monkeypatch.setattr(
layerwise_offload_mod.logger,
"warning",
lambda message, *args: warnings.append(message % args),
)
manager = _permanent_manager(_MultiBlockModel(4), num_layers=4, resident_layers=0)
assert manager._permanent_set == frozenset()
assert manager._residency_active is False
assert any("permanent residency with no resident layers" in w for w in warnings)
def test_unknown_residency_lifetime_is_rejected(monkeypatch):
_patch_fake_device(monkeypatch)
with pytest.raises(ValueError, match="unknown residency lifetime"):
_resident_manager(
_MultiBlockModel(4),
num_layers=4,
resident_layers=2,
residency_lifetime="sometimes",
)
def test_forward_lifetime_is_the_default_and_unchanged(monkeypatch):
_patch_fake_device(monkeypatch)
manager = _resident_manager(
_MultiBlockModel(4), num_layers=4, prefetch_size=1, resident_layers=2
)
assert manager.residency_lifetime == RESIDENCY_LIFETIME_FORWARD
assert manager._permanent_set == frozenset()
assert manager._residency_active is False
for i in range(4):
assert i in manager._weight_metadata
def test_layerwise_tuning_returns_the_lifetime():
args = _server_args(
dit_layerwise_resident_layers=20,
dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT,
layerwise_residency_lifetime="video_vae=forward",
)
assert (
args.layerwise_tuning_for("transformer", dit_group=True)[3]
== RESIDENCY_LIFETIME_PERMANENT
)
assert (
args.layerwise_tuning_for("video_vae", dit_group=False)[3]
== RESIDENCY_LIFETIME_FORWARD
)
# the DiT-group default is not the auxiliary default
assert (
args.layerwise_tuning_for("text_encoder", dit_group=False)[3]
== RESIDENCY_LIFETIME_FORWARD
)
def test_layerwise_tuning_rejects_unknown_lifetime():
args = _server_args(layerwise_residency_lifetime="vae=sometimes")
with pytest.raises(ValueError, match="unknown residency lifetime"):
args.layerwise_tuning_for("vae", dit_group=False)
def test_configure_logs_a_permanent_lifetime(monkeypatch):
_patch_fake_device(monkeypatch)
logs = []
monkeypatch.setattr(
layerwise_offload_mod.logger,
"info",
lambda message, *args: logs.append(message % args),
)
comp = _ResidentComponent(8)
comp.configure_layerwise_offload(
_server_args(
dit_offload_prefetch_size=2,
dit_layerwise_resident_layers=3,
dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT,
),
component_name="transformer",
)
assert any("placed 3 permanent resident layers" in line for line in logs)
assert logs[-1].endswith("resident=3/8 (permanent), policy=leading")
@@ -3412,6 +3412,75 @@ class TestDirectGpuWeightLoading(unittest.TestCase):
tp_args._validate_direct_gpu_weight_loading()
class TestLayerwiseResidencyLifetime(unittest.TestCase):
def _help_by_option(self):
parser = FlexibleArgumentParser()
ServerArgs.add_cli_args(parser)
return {
action.option_strings[0]: (action.help or "")
for action in parser._actions
if action.option_strings
}
def test_cli_accepts_the_two_lifetimes_and_rejects_others(self):
parser = FlexibleArgumentParser()
ServerArgs.add_cli_args(parser)
args = parser.parse_args(
["--model-path", "/fake", "--dit-layerwise-residency-lifetime", "permanent"]
)
self.assertEqual(args.dit_layerwise_residency_lifetime, "permanent")
self.assertEqual(
parser.parse_args(
["--model-path", "/fake"]
).dit_layerwise_residency_lifetime,
"forward",
)
with self.assertRaises(SystemExit):
parser.parse_args(
[
"--model-path",
"/fake",
"--dit-layerwise-residency-lifetime",
"sometimes",
]
)
def test_python_api_rejects_an_unknown_lifetime(self):
# argparse choices cover the CLI only; ServerArgs is also built directly.
with self.assertRaisesRegex(ValueError, "dit-layerwise-residency-lifetime"):
_from_dict_without_model_resolution(
{
"model_path": "/data/my-model",
"dit_layerwise_residency_lifetime": "sometimes",
}
)
def test_help_says_when_layers_are_placed_and_released(self):
"""A user reads these to decide between the two; both answers must be there."""
help_by_option = self._help_by_option()
dit_help = help_by_option["--dit-layerwise-residency-lifetime"]
for claim in (
"'forward' (default)",
"'permanent'",
"load time",
"never released",
):
self.assertIn(claim, dit_help)
per_component_help = help_by_option["--layerwise-residency-lifetime"]
for claim in ("transformer=permanent", "never releases"):
self.assertIn(claim, per_component_help)
# The resident-layer flags point at the lifetime knob instead of
# describing a lifetime of their own.
self.assertIn(
"--dit-layerwise-residency-lifetime",
help_by_option["--dit-layerwise-resident-layers"],
)
self.assertIn(
"--layerwise-residency-lifetime",
help_by_option["--layerwise-resident-layers"],
)
if __name__ == "__main__":
unittest.main()