[diffusion] feat: support permanent lifetime for layerwise resident layers (#40599)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Fable 5.1
parent
28be39f72e
commit
a3d11f9d95
+122
-10
@@ -47,6 +47,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.host_memory_budget i
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
|
||||
LAYERWISE_OFFLOAD_ALL_COMPONENTS,
|
||||
LAYERWISE_OFFLOAD_DIT_GROUP,
|
||||
RESIDENCY_LIFETIME_FORWARD,
|
||||
RESIDENCY_LIFETIME_PERMANENT,
|
||||
RESIDENCY_LIFETIMES,
|
||||
RESIDENCY_POLICIES,
|
||||
RESIDENCY_POLICY_LEADING,
|
||||
RESIDENCY_POLICY_STRIDED,
|
||||
@@ -1008,6 +1011,7 @@ class LayerwiseOffloadManager:
|
||||
resident_layers: int = 0,
|
||||
initialize: bool = True,
|
||||
residency_policy: str = RESIDENCY_POLICY_LEADING,
|
||||
residency_lifetime: str = RESIDENCY_LIFETIME_FORWARD,
|
||||
pin_budget: HostPinBudget | None = None,
|
||||
pin_component_name: str = "layerwise offload",
|
||||
) -> None:
|
||||
@@ -1045,9 +1049,34 @@ class LayerwiseOffloadManager:
|
||||
self._resident_set = frozenset(range(self.num_layers)) - set(
|
||||
self._streamed_order
|
||||
)
|
||||
# Armed on the first denoise forward, so that the load-time prefetch below
|
||||
# does not pin the whole resident set before the DiT is the active component.
|
||||
self._residency_active = False
|
||||
if residency_lifetime not in RESIDENCY_LIFETIMES:
|
||||
raise ValueError(
|
||||
f"unknown residency lifetime {residency_lifetime!r}, expected one "
|
||||
f"of {RESIDENCY_LIFETIMES}"
|
||||
)
|
||||
self.residency_lifetime = residency_lifetime
|
||||
# Placed on the device at load and never released: no host copy, no
|
||||
# per-request transfer. Empty under `forward`, where the resident set is
|
||||
# pinned when the component starts running and released when it finishes.
|
||||
self._permanent_set = (
|
||||
self._resident_set
|
||||
if residency_lifetime == RESIDENCY_LIFETIME_PERMANENT
|
||||
else frozenset()
|
||||
)
|
||||
if (
|
||||
residency_lifetime == RESIDENCY_LIFETIME_PERMANENT
|
||||
and not self._resident_set
|
||||
):
|
||||
logger.warning(
|
||||
"Layerwise offload: %s asks for permanent residency with no resident "
|
||||
"layers, so nothing is kept and every layer streams.",
|
||||
pin_component_name,
|
||||
)
|
||||
# Under `forward`, armed on the first denoise forward so that the
|
||||
# load-time prefetch below does not pin the whole resident set before the
|
||||
# DiT is the active component. Permanent layers are on the device from
|
||||
# load, so that set is armed from the start.
|
||||
self._residency_active = bool(self._permanent_set)
|
||||
# True while load_all_layers materializes every layer for a resident
|
||||
# placement; every mapped source is then read exactly once.
|
||||
self._materializing_all = False
|
||||
@@ -1270,6 +1299,9 @@ class LayerwiseOffloadManager:
|
||||
first, in streamed order, which is also deterministic. Unpinning a
|
||||
resident layer costs one possibly-faulting arming copy per request and
|
||||
buys a whole layer's worth of per-step overlap.
|
||||
|
||||
Permanent resident layers are not in ``layer_groups``: they live on the
|
||||
device with no host copy, so there is nothing here to host.
|
||||
"""
|
||||
totals, mapped = self._layer_byte_totals(layer_groups)
|
||||
if host_copies_are_redundant():
|
||||
@@ -1380,6 +1412,19 @@ class LayerwiseOffloadManager:
|
||||
local_tensor.dtype, []
|
||||
).append((name, tensor))
|
||||
|
||||
permanent_groups = {
|
||||
layer_idx: groups
|
||||
for layer_idx, groups in layer_groups.items()
|
||||
if layer_idx in self._permanent_set
|
||||
}
|
||||
if permanent_groups:
|
||||
self._place_permanent_layers(permanent_groups)
|
||||
layer_groups = {
|
||||
layer_idx: groups
|
||||
for layer_idx, groups in layer_groups.items()
|
||||
if layer_idx not in self._permanent_set
|
||||
}
|
||||
|
||||
layer_hosting, untracked_bytes = self._plan_layer_hosting(layer_groups)
|
||||
try:
|
||||
for storage in self._initialize_host_stores(layer_groups, layer_hosting):
|
||||
@@ -1389,6 +1434,54 @@ class LayerwiseOffloadManager:
|
||||
# failed allocations have no storage finalizer to return their allowance
|
||||
self._pin_budget.release(untracked_bytes)
|
||||
|
||||
def _copy_to_device_keeping_layout(
|
||||
self, local_weight: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""One device copy of a weight, with a strided view's layout kept.
|
||||
|
||||
Same reason as the strided host store: a transposed FP8 view has to
|
||||
keep its layout, and `.to()` does not promise that.
|
||||
"""
|
||||
if local_weight.is_contiguous():
|
||||
return local_weight.to(self.device, non_blocking=False)
|
||||
device_tensor = torch.empty_strided(
|
||||
size=local_weight.shape,
|
||||
stride=local_weight.stride(),
|
||||
dtype=local_weight.dtype,
|
||||
device=self.device,
|
||||
)
|
||||
device_tensor.copy_(local_weight, non_blocking=False)
|
||||
return device_tensor
|
||||
|
||||
def _place_permanent_layers(self, layer_groups: Dict) -> None:
|
||||
"""Move the permanent resident layers to the device, once, with no host store.
|
||||
|
||||
They are never released, so a host copy would be a reload source for a
|
||||
reload that never comes; dropping it is the point. Doing this before
|
||||
the hosting plan also keeps these bytes out of the pin budget, which is
|
||||
for layers that stream.
|
||||
"""
|
||||
placed_bytes = 0
|
||||
with torch.inference_mode(False), torch.no_grad():
|
||||
for layer_idx, dtype_to_params in layer_groups.items():
|
||||
for weights in dtype_to_params.values():
|
||||
for _, weight in weights:
|
||||
device_tensor = self._copy_to_device_keeping_layout(
|
||||
self._to_local_tensor(weight)
|
||||
)
|
||||
placed_bytes += (
|
||||
device_tensor.numel() * device_tensor.element_size()
|
||||
)
|
||||
weight.data = self._wrap_for_target(weight, device_tensor)
|
||||
self._gpu_layers.add(layer_idx)
|
||||
logger.info(
|
||||
"Layerwise offload: %s placed %d permanent resident layers (%.2f GiB) "
|
||||
"on the device at load; no host copy is kept.",
|
||||
self._pin_component_name,
|
||||
len(layer_groups),
|
||||
placed_bytes / (1 << 30),
|
||||
)
|
||||
|
||||
def _initialize_host_stores(
|
||||
self, layer_groups: Dict, layer_hosting: Dict[int, str]
|
||||
) -> Iterator[torch.UntypedStorage]:
|
||||
@@ -1519,8 +1612,9 @@ class LayerwiseOffloadManager:
|
||||
self._consolidated_cpu_weights[layer_idx][dtype] = cpu_buffer
|
||||
|
||||
def _finalize_initialization(self) -> None:
|
||||
# prefetch the head of the stream for warm-up; residency is not armed
|
||||
# yet, so this is layer 0 regardless of policy
|
||||
# prefetch the head of the stream for warm-up. Under `forward` residency
|
||||
# is not armed yet, so this is layer 0 regardless of policy; permanent
|
||||
# layers are already on the device, so the head is the first streamed one.
|
||||
self.prepare_for_next_req(non_blocking=False)
|
||||
|
||||
self.register_forward_hooks()
|
||||
@@ -1554,6 +1648,14 @@ class LayerwiseOffloadManager:
|
||||
layer_idx = self._match_layer_idx(name)
|
||||
if layer_idx is None or layer_idx >= self.num_layers:
|
||||
continue
|
||||
if layer_idx in self._permanent_set:
|
||||
# Placed once, no CPU copy kept.
|
||||
tensor.data = self._wrap_for_target(
|
||||
tensor,
|
||||
self._copy_to_device_keeping_layout(self._to_local_tensor(tensor)),
|
||||
)
|
||||
self._gpu_layers.add(layer_idx)
|
||||
continue
|
||||
local_tensor = self._to_local_tensor(tensor).detach()
|
||||
cpu_tensor = (
|
||||
local_tensor
|
||||
@@ -1583,8 +1685,8 @@ class LayerwiseOffloadManager:
|
||||
self._release_unneeded_streamed_layers(keep=set(self._head_of_stream()))
|
||||
|
||||
# The resident set first: it has to be there for the whole step, and the
|
||||
# caller decides whether to block on it.
|
||||
for layer_idx in sorted(self._retained_set):
|
||||
# caller decides whether to block on it. Permanent layers never left.
|
||||
for layer_idx in sorted(self._retained_set - self._permanent_set):
|
||||
self.prefetch_layer(layer_idx, non_blocking=non_blocking)
|
||||
if not non_blocking and self.copy_stream is not None:
|
||||
torch.get_device_module().current_stream().wait_stream(self.copy_stream)
|
||||
@@ -1603,7 +1705,8 @@ class LayerwiseOffloadManager:
|
||||
@property
|
||||
def holds_residents(self) -> bool:
|
||||
"""True if this manager keeps a resident layer set beyond the streaming
|
||||
prefetch window, so it must be denoise-stage-scoped."""
|
||||
prefetch window. Under `forward` that set is released when the component
|
||||
finishes running; under `permanent` it never is."""
|
||||
return self.enabled and self.resident_layers > 0
|
||||
|
||||
@property
|
||||
@@ -2096,6 +2199,10 @@ class LayerwiseOffloadManager:
|
||||
"""
|
||||
if not self.enabled or self.device is None:
|
||||
return
|
||||
# No host store to fall back to. `force` ends a forward-lifetime set;
|
||||
# it does not apply here.
|
||||
if layer_idx in self._permanent_set:
|
||||
return
|
||||
|
||||
if not force and layer_idx in self._retained_set:
|
||||
return
|
||||
@@ -2725,7 +2832,7 @@ class LayerwiseOffloadableModuleMixin:
|
||||
named_modules = dict(self.named_modules())
|
||||
layer_specs = []
|
||||
# `--dit-*` is the group default these fall back to, not a scope.
|
||||
prefetch_value, resident_value, residency_policy = (
|
||||
prefetch_value, resident_value, residency_policy, residency_lifetime = (
|
||||
server_args.layerwise_tuning_for(
|
||||
component_name,
|
||||
dit_group=self.layerwise_offload_dit_group_enabled,
|
||||
@@ -2799,6 +2906,7 @@ class LayerwiseOffloadableModuleMixin:
|
||||
resident_layers=resident_layers,
|
||||
initialize=False,
|
||||
residency_policy=residency_policy,
|
||||
residency_lifetime=residency_lifetime,
|
||||
)
|
||||
self.layerwise_offload_managers.append(manager)
|
||||
|
||||
@@ -2843,6 +2951,9 @@ class LayerwiseOffloadableModuleMixin:
|
||||
for value in sorted({manager.prefetch_size for manager in managers})
|
||||
)
|
||||
policies = ", ".join(sorted({manager.residency_policy for manager in managers}))
|
||||
lifetimes = ", ".join(
|
||||
sorted({manager.residency_lifetime for manager in managers})
|
||||
)
|
||||
total_layers = sum(manager.num_layers for manager in managers)
|
||||
resident_layers = sum(manager.resident_layers for manager in managers)
|
||||
if envs.SGLANG_DIFFUSION_DEBUG_HOST_MEMORY:
|
||||
@@ -2853,7 +2964,7 @@ class LayerwiseOffloadableModuleMixin:
|
||||
log_anon_vmas(f"layerwise offload ready for {component_name}")
|
||||
logger.info(
|
||||
"Layerwise offload ready for %s in %.2fs: groups=%d, layers=%d, "
|
||||
"prefetch/group=%s, resident=%d/%d (per request), policy=%s",
|
||||
"prefetch/group=%s, resident=%d/%d (%s), policy=%s",
|
||||
component_label,
|
||||
perf_counter() - started_at,
|
||||
len(managers),
|
||||
@@ -2861,6 +2972,7 @@ class LayerwiseOffloadableModuleMixin:
|
||||
prefetch_sizes,
|
||||
resident_layers,
|
||||
total_layers,
|
||||
lifetimes,
|
||||
policies,
|
||||
)
|
||||
|
||||
|
||||
+7
@@ -14,6 +14,13 @@ RESIDENCY_POLICY_LEADING = "leading"
|
||||
RESIDENCY_POLICY_STRIDED = "strided"
|
||||
RESIDENCY_POLICIES = (RESIDENCY_POLICY_LEADING, RESIDENCY_POLICY_STRIDED)
|
||||
|
||||
# How long those layers stay. `forward`: placed when the component starts
|
||||
# running its layers for a request, released when it finishes, host copy kept.
|
||||
# `permanent`: placed at load, no host copy, never released.
|
||||
RESIDENCY_LIFETIME_FORWARD = "forward"
|
||||
RESIDENCY_LIFETIME_PERMANENT = "permanent"
|
||||
RESIDENCY_LIFETIMES = (RESIDENCY_LIFETIME_FORWARD, RESIDENCY_LIFETIME_PERMANENT)
|
||||
|
||||
# Components whose layerwise policy has been validated as a better default than
|
||||
# component-level CPU offload when the user has not pinned their placement.
|
||||
LAYERWISE_OFFLOAD_DEFAULT_GROUP_COMPONENTS = (
|
||||
|
||||
@@ -48,6 +48,9 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_co
|
||||
LAYERWISE_OFFLOAD_IMAGE_ENCODER_GROUP,
|
||||
LAYERWISE_OFFLOAD_TEXT_ENCODER_GROUP,
|
||||
LAYERWISE_OFFLOAD_VAE_GROUP,
|
||||
RESIDENCY_LIFETIME_FORWARD,
|
||||
RESIDENCY_LIFETIME_PERMANENT,
|
||||
RESIDENCY_LIFETIMES,
|
||||
RESIDENCY_POLICIES,
|
||||
RESIDENCY_POLICY_LEADING,
|
||||
cpu_offload_component_matches,
|
||||
@@ -418,7 +421,10 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
dit_layerwise_resident_layers: float = 0.0
|
||||
# Which layers those are: the leading ones, or spread evenly over the stack.
|
||||
dit_layerwise_residency_policy: str = RESIDENCY_POLICY_LEADING
|
||||
# Per-component overrides of the three knobs above; an entry wins for that
|
||||
# How long they stay: while the DiT runs for a request, or for the life of
|
||||
# the server.
|
||||
dit_layerwise_residency_lifetime: str = RESIDENCY_LIFETIME_FORWARD
|
||||
# Per-component overrides of the four knobs above; an entry wins for that
|
||||
# component.
|
||||
layerwise_prefetch_size: dict[str, float] | str | None = field(default_factory=dict)
|
||||
layerwise_resident_layers: dict[str, float] | str | None = field(
|
||||
@@ -427,6 +433,9 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
layerwise_residency_policy: dict[str, str] | str | None = field(
|
||||
default_factory=dict
|
||||
)
|
||||
layerwise_residency_lifetime: dict[str, str] | str | None = field(
|
||||
default_factory=dict
|
||||
)
|
||||
offload_during_compile: bool = True
|
||||
text_encoder_cpu_offload: bool | None = None
|
||||
image_encoder_cpu_offload: bool | None = None
|
||||
@@ -1209,8 +1218,8 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
|
||||
def layerwise_tuning_for(
|
||||
self, component_name: str | None, *, dit_group: bool
|
||||
) -> tuple[float, float, str]:
|
||||
"""Prefetch size, resident layers and residency policy for one component."""
|
||||
) -> tuple[float, float, str, str]:
|
||||
"""Prefetch size, resident layers, residency policy and lifetime for one component."""
|
||||
prefetch_map = self._parse_component_value_map(
|
||||
self.layerwise_prefetch_size, option="--layerwise-prefetch-size"
|
||||
)
|
||||
@@ -1220,6 +1229,9 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
policy_map = self._parse_component_value_map(
|
||||
self.layerwise_residency_policy, option="--layerwise-residency-policy"
|
||||
)
|
||||
lifetime_map = self._parse_component_value_map(
|
||||
self.layerwise_residency_lifetime, option="--layerwise-residency-lifetime"
|
||||
)
|
||||
|
||||
def _pick(mapping: dict[str, str], group_default, aux_default):
|
||||
if component_name is not None and component_name in mapping:
|
||||
@@ -1240,7 +1252,19 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
f"unknown residency policy {policy!r} for component "
|
||||
f"{component_name!r}, expected one of {RESIDENCY_POLICIES}"
|
||||
)
|
||||
return prefetch, resident, policy
|
||||
lifetime = str(
|
||||
_pick(
|
||||
lifetime_map,
|
||||
self.dit_layerwise_residency_lifetime,
|
||||
RESIDENCY_LIFETIME_FORWARD,
|
||||
)
|
||||
)
|
||||
if lifetime not in RESIDENCY_LIFETIMES:
|
||||
raise ValueError(
|
||||
f"unknown residency lifetime {lifetime!r} for component "
|
||||
f"{component_name!r}, expected one of {RESIDENCY_LIFETIMES}"
|
||||
)
|
||||
return prefetch, resident, policy, lifetime
|
||||
|
||||
@staticmethod
|
||||
def _parse_component_attention_backend_map(
|
||||
@@ -2551,8 +2575,9 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
"streaming). Between 0.0 and 1.0 = ratio of layers; >= 1 = absolute "
|
||||
"count. Unlike raising the prefetch size, resident layers are transferred "
|
||||
"once per request rather than once per step, so this trades VRAM for "
|
||||
"lower denoise latency when memory is available. They are released when "
|
||||
"the request finishes, not kept for the life of the server.",
|
||||
"lower denoise latency when memory is available. How long they stay on "
|
||||
"the GPU is --dit-layerwise-residency-lifetime: released when the "
|
||||
"request finishes by default, or kept for the life of the server.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--layerwise-prefetch-size",
|
||||
@@ -2577,7 +2602,8 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
"a DiT across the denoise steps, a video VAE across the latent chunks. "
|
||||
"A component that runs its layers once per request, such as a text "
|
||||
"encoder, transfers the whole set again every request, so setting "
|
||||
"this for one has no effect.",
|
||||
"this for one has no effect unless --layerwise-residency-lifetime "
|
||||
"keeps the layers for the life of the server.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--layerwise-residency-policy",
|
||||
@@ -2587,6 +2613,17 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
"component=value entries, e.g. --layerwise-residency-policy "
|
||||
"text_encoder=strided.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--layerwise-residency-lifetime",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Per-component override of --dit-layerwise-residency-lifetime, as "
|
||||
"component=value entries, e.g. --layerwise-residency-lifetime "
|
||||
"transformer=permanent,video_vae=forward. 'forward' releases a "
|
||||
"component's resident layers when it finishes running for a request; "
|
||||
"'permanent' places them on the GPU at load time and never releases "
|
||||
"them.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dit-layerwise-residency-policy",
|
||||
type=str,
|
||||
@@ -2601,6 +2638,21 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
"memory-bound compute -- the transfers stop competing with it for "
|
||||
"L2 and DRAM bandwidth, which is where the gain comes from.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dit-layerwise-residency-lifetime",
|
||||
type=str,
|
||||
choices=RESIDENCY_LIFETIMES,
|
||||
default=ServerArgs.dit_layerwise_residency_lifetime,
|
||||
help="How long the DiT layers kept by --dit-layerwise-resident-layers "
|
||||
"stay on the GPU. 'forward' (default): placed when the DiT starts its "
|
||||
"denoise steps for a request and released when they finish, so they "
|
||||
"are transferred once per request and keep a copy in host memory. "
|
||||
"'permanent': placed at load time, no host copy, never released -- "
|
||||
"one transfer for the life of the server. Choose 'permanent' when the "
|
||||
"resident layers of every component fit on the GPU together; with "
|
||||
"'forward' only the running component holds its set, so the peak is "
|
||||
"the largest set rather than their sum.",
|
||||
)
|
||||
|
||||
# offload flags
|
||||
parser.add_argument(
|
||||
@@ -3625,6 +3677,27 @@ class ServerArgs(DisaggServerArgsMixin):
|
||||
"so there is no resident set to place."
|
||||
)
|
||||
|
||||
if self.dit_layerwise_residency_lifetime not in RESIDENCY_LIFETIMES:
|
||||
raise ValueError(
|
||||
f"Invalid --dit-layerwise-residency-lifetime "
|
||||
f"{self.dit_layerwise_residency_lifetime!r}; expected one of "
|
||||
f"{RESIDENCY_LIFETIMES}."
|
||||
)
|
||||
if self.dit_layerwise_residency_lifetime == RESIDENCY_LIFETIME_PERMANENT:
|
||||
if not self.is_dit_layerwise_offload_selected:
|
||||
logger.warning(
|
||||
"--dit-layerwise-residency-lifetime has no effect because the "
|
||||
"DiT is not layerwise-offloaded. It only applies together with "
|
||||
"--dit-layerwise-offload (or 'dit' in "
|
||||
"--layerwise-offload-components)."
|
||||
)
|
||||
elif self.dit_layerwise_resident_layers <= 0:
|
||||
logger.warning(
|
||||
"--dit-layerwise-residency-lifetime has no effect because "
|
||||
"--dit-layerwise-resident-layers is 0: there is no resident "
|
||||
"set to keep."
|
||||
)
|
||||
|
||||
# validate layerwise offload conflicts
|
||||
if envs.SGLANG_CACHE_DIT_ENABLED and self.use_fsdp_inference:
|
||||
if self.is_arg_explicitly_set("use_fsdp_inference"):
|
||||
|
||||
@@ -303,6 +303,27 @@ ONE_GPU_CASES: list[DiffusionTestCase] = [
|
||||
extras={"enable_teacache": True},
|
||||
),
|
||||
),
|
||||
# The text encoder's resident set placed at load and never released. The
|
||||
# guard is TextEncodingStage: with the set on the device the stage is
|
||||
# compute only, and if the layers ever go back to being transferred per
|
||||
# request the stage regains that whole transfer, well past the
|
||||
# non_denoise_stage tolerance. load_peak_vram carries the placed set, so
|
||||
# it also pins that the placement happens at load, not on the first use.
|
||||
# The encoder is already layerwise under the default component set.
|
||||
# No consistency check: the lifetime moves weights, not math, so the
|
||||
# output is the base case's and that case already guards it.
|
||||
DiffusionTestCase(
|
||||
"wan2_1_t2v_1.3b_encoder_permanent_residents",
|
||||
DiffusionServerArgs(
|
||||
model_path=DEFAULT_WAN_2_1_T2V_1_3B_MODEL_NAME_FOR_TEST,
|
||||
extras=[
|
||||
"--layerwise-resident-layers text_encoder=0.8 "
|
||||
"--layerwise-residency-lifetime text_encoder=permanent",
|
||||
],
|
||||
),
|
||||
DiffusionSamplingParams(prompt=T2V_PROMPT),
|
||||
run_consistency_check=False,
|
||||
),
|
||||
# Frame interpolation (2× / exp=1)
|
||||
# Uses the same 1.3B model already in the suite;
|
||||
DiffusionTestCase(
|
||||
|
||||
@@ -1230,6 +1230,78 @@
|
||||
"runtime_peak_allocated_mb": 9730.0,
|
||||
"estimated_full_test_time_s": 126.0
|
||||
},
|
||||
"wan2_1_t2v_1.3b_encoder_permanent_residents": {
|
||||
"expected_load_ms": 29198.13,
|
||||
"stages_ms": {
|
||||
"InputValidationStage": 0.04,
|
||||
"TextEncodingStage": 171.56,
|
||||
"LatentPreparationStage": 0.25,
|
||||
"TimestepPreparationStage": 1.63,
|
||||
"DenoisingStage": 7415.91,
|
||||
"DecodingStage": 352.12
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 105.75,
|
||||
"1": 126.42,
|
||||
"2": 147.21,
|
||||
"3": 146.06,
|
||||
"4": 146.19,
|
||||
"5": 147.25,
|
||||
"6": 162.75,
|
||||
"7": 150.95,
|
||||
"8": 148.83,
|
||||
"9": 148.53,
|
||||
"10": 146.13,
|
||||
"11": 146.61,
|
||||
"12": 153.93,
|
||||
"13": 154.16,
|
||||
"14": 149.9,
|
||||
"15": 149.11,
|
||||
"16": 150.65,
|
||||
"17": 147.48,
|
||||
"18": 150.69,
|
||||
"19": 152.47,
|
||||
"20": 152.68,
|
||||
"21": 147.5,
|
||||
"22": 148.45,
|
||||
"23": 146.57,
|
||||
"24": 145.73,
|
||||
"25": 148.87,
|
||||
"26": 151.59,
|
||||
"27": 150.64,
|
||||
"28": 149.1,
|
||||
"29": 148.96,
|
||||
"30": 149.04,
|
||||
"31": 148.03,
|
||||
"32": 149.99,
|
||||
"33": 151.77,
|
||||
"34": 150.24,
|
||||
"35": 149.52,
|
||||
"36": 149.38,
|
||||
"37": 149.0,
|
||||
"38": 147.44,
|
||||
"39": 150.58,
|
||||
"40": 150.23,
|
||||
"41": 151.2,
|
||||
"42": 149.01,
|
||||
"43": 150.49,
|
||||
"44": 148.42,
|
||||
"45": 147.62,
|
||||
"46": 149.83,
|
||||
"47": 150.52,
|
||||
"48": 150.02,
|
||||
"49": 148.03
|
||||
},
|
||||
"expected_e2e_ms": 8010.27,
|
||||
"expected_avg_denoise_ms": 148.23,
|
||||
"expected_median_denoise_ms": 149.07,
|
||||
"load_peak_vram_mb": 17792.0,
|
||||
"runtime_peak_vram_mb": 26930.0,
|
||||
"warmup_peak_vram_mb": 27492.0,
|
||||
"load_peak_allocated_mb": 17625.61,
|
||||
"runtime_peak_allocated_mb": 23676.07,
|
||||
"estimated_full_test_time_s": 130.0
|
||||
},
|
||||
"wan2_1_t2v_1.3b": {
|
||||
"expected_load_ms": 30234.05,
|
||||
"stages_ms": {
|
||||
|
||||
@@ -49,8 +49,11 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
|
||||
get_layerwise_offload_component_names_for_pipeline,
|
||||
is_layerwise_offloaded_module,
|
||||
is_resident_layerwise_module,
|
||||
iter_materialized_weights,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload_components import (
|
||||
RESIDENCY_LIFETIME_FORWARD,
|
||||
RESIDENCY_LIFETIME_PERMANENT,
|
||||
RESIDENCY_POLICY_LEADING,
|
||||
RESIDENCY_POLICY_STRIDED,
|
||||
)
|
||||
@@ -224,9 +227,11 @@ def _server_args(**kwargs):
|
||||
dit_offload_prefetch_size=1,
|
||||
dit_layerwise_resident_layers=0.0,
|
||||
dit_layerwise_residency_policy=RESIDENCY_POLICY_LEADING,
|
||||
dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_FORWARD,
|
||||
layerwise_prefetch_size={},
|
||||
layerwise_resident_layers={},
|
||||
layerwise_residency_policy={},
|
||||
layerwise_residency_lifetime={},
|
||||
pin_cpu_memory=False,
|
||||
# the pin budget ranks candidates by bytes x steps, and reads the step
|
||||
# count off the pipeline's sampling defaults
|
||||
@@ -986,6 +991,7 @@ def _resident_manager(
|
||||
prefetch_size=1,
|
||||
resident_layers=0,
|
||||
residency_policy=RESIDENCY_POLICY_LEADING,
|
||||
residency_lifetime=RESIDENCY_LIFETIME_FORWARD,
|
||||
):
|
||||
return LayerwiseOffloadManager(
|
||||
model=model,
|
||||
@@ -996,6 +1002,7 @@ def _resident_manager(
|
||||
prefetch_size=prefetch_size,
|
||||
resident_layers=resident_layers,
|
||||
residency_policy=residency_policy,
|
||||
residency_lifetime=residency_lifetime,
|
||||
)
|
||||
|
||||
|
||||
@@ -1242,11 +1249,11 @@ def test_configure_logs_component_start_and_completion(monkeypatch):
|
||||
"Configuring layerwise offload for transformer (_ResidentComponent): "
|
||||
"blocks (8 layers)"
|
||||
)
|
||||
# "(per request)" is the point of this line: the set is re-established every
|
||||
# request, not pinned for the server's lifetime.
|
||||
# The lifetime in parentheses is the point of this line: `forward` says the
|
||||
# set is re-established every request, not pinned for the server's lifetime.
|
||||
assert logs[-1] == (
|
||||
"Layerwise offload ready for transformer (_ResidentComponent) in 2.35s: "
|
||||
"groups=1, layers=8, prefetch/group=2, resident=3/8 (per request), policy=leading"
|
||||
"groups=1, layers=8, prefetch/group=2, resident=3/8 (forward), policy=leading"
|
||||
)
|
||||
|
||||
|
||||
@@ -2052,11 +2059,13 @@ def test_layerwise_tuning_defaults_match_the_group():
|
||||
3.0,
|
||||
20.0,
|
||||
RESIDENCY_POLICY_STRIDED,
|
||||
RESIDENCY_LIFETIME_FORWARD,
|
||||
)
|
||||
assert args.layerwise_tuning_for("text_encoder", dit_group=False) == (
|
||||
0.0,
|
||||
0.0,
|
||||
RESIDENCY_POLICY_LEADING,
|
||||
RESIDENCY_LIFETIME_FORWARD,
|
||||
)
|
||||
|
||||
|
||||
@@ -2073,12 +2082,14 @@ def test_layerwise_tuning_per_component_entry_wins():
|
||||
2.0,
|
||||
4.0,
|
||||
RESIDENCY_POLICY_STRIDED,
|
||||
RESIDENCY_LIFETIME_FORWARD,
|
||||
)
|
||||
# an entry for one component leaves every other component alone
|
||||
assert args.layerwise_tuning_for("vae", dit_group=False) == (
|
||||
0.0,
|
||||
0.0,
|
||||
RESIDENCY_POLICY_LEADING,
|
||||
RESIDENCY_LIFETIME_FORWARD,
|
||||
)
|
||||
assert args.layerwise_tuning_for("transformer", dit_group=True)[:2] == (3.0, 20.0)
|
||||
|
||||
@@ -2582,3 +2593,230 @@ def test_small_pinned_stores_keep_using_the_pool(monkeypatch):
|
||||
tensor = layerwise_offload_mod._pinned_empty(1024, dtype=torch.float32)
|
||||
assert tensor.is_pinned()
|
||||
assert len(pooled) == 1
|
||||
|
||||
|
||||
# --- permanent residency lifetime -------------------------------------------
|
||||
|
||||
|
||||
def _permanent_manager(model, *, num_layers, resident_layers, prefetch_size=1):
|
||||
return _resident_manager(
|
||||
model,
|
||||
num_layers=num_layers,
|
||||
prefetch_size=prefetch_size,
|
||||
resident_layers=resident_layers,
|
||||
residency_lifetime=RESIDENCY_LIFETIME_PERMANENT,
|
||||
)
|
||||
|
||||
|
||||
def test_permanent_residents_are_placed_at_load_without_a_host_store(monkeypatch):
|
||||
"""`permanent` puts the resident set on the device in the constructor.
|
||||
|
||||
Not on the first forward, not during warmup: at load. And it keeps no host
|
||||
copy of those layers -- there is nothing to reload them from because
|
||||
nothing ever releases them.
|
||||
"""
|
||||
_patch_fake_device(monkeypatch)
|
||||
model = _MultiBlockModel(6)
|
||||
originals = [model.blocks[i].weight.detach().clone() for i in range(6)]
|
||||
manager = _permanent_manager(model, num_layers=6, resident_layers=3)
|
||||
|
||||
assert manager._permanent_set == frozenset({0, 1, 2})
|
||||
# Armed from load: the forward-lifetime set waits for the first forward.
|
||||
assert manager._residency_active is True
|
||||
assert manager._retained_layers == 3
|
||||
assert {0, 1, 2} <= manager._gpu_layers
|
||||
for i in range(3):
|
||||
assert i not in manager._weight_metadata
|
||||
assert not manager._consolidated_cpu_weights.get(i)
|
||||
assert torch.equal(model.blocks[i].weight, originals[i])
|
||||
# a strided view keeps its layout, as the strided host store would have
|
||||
assert model.blocks[i].weight.stride() == originals[i].stride()
|
||||
# Streamed layers take the ordinary path: a host store, and a placeholder
|
||||
# until prefetched. The head of the stream is now layer 3, not layer 0.
|
||||
assert 3 in manager._gpu_layers
|
||||
for i in (4, 5):
|
||||
assert i in manager._weight_metadata
|
||||
assert tuple(model.blocks[i].weight.shape) == (1,)
|
||||
|
||||
|
||||
def test_permanent_residents_survive_every_release_path(monkeypatch):
|
||||
_patch_fake_device(monkeypatch)
|
||||
manager = _permanent_manager(_MultiBlockModel(6), num_layers=6, resident_layers=3)
|
||||
manager.prefetch_layer(4, non_blocking=False)
|
||||
assert {0, 1, 2, 3, 4} <= manager._gpu_layers
|
||||
|
||||
manager.release_layer(0, force=True)
|
||||
assert 0 in manager._gpu_layers
|
||||
manager.release_all()
|
||||
assert manager._gpu_layers == {0, 1, 2}
|
||||
# Nothing to write back and nothing to write it to; must be a no-op.
|
||||
manager.sync_all_layers_to_cpu()
|
||||
assert manager._gpu_layers == {0, 1, 2}
|
||||
|
||||
|
||||
def test_prepare_does_not_touch_permanent_residents(monkeypatch):
|
||||
_patch_fake_device(monkeypatch)
|
||||
model = _MultiBlockModel(6)
|
||||
manager = _permanent_manager(model, num_layers=6, resident_layers=3)
|
||||
manager.release_all()
|
||||
before = [model.blocks[i].weight.data_ptr() for i in range(3)]
|
||||
|
||||
log = _record_prepare(manager, monkeypatch)
|
||||
manager.prepare_for_next_req(non_blocking=False)
|
||||
|
||||
prefetched = [entry[1] for entry in log if entry[0] == "prefetch"]
|
||||
assert not set(prefetched) & {0, 1, 2}
|
||||
assert 3 in prefetched
|
||||
assert [model.blocks[i].weight.data_ptr() for i in range(3)] == before
|
||||
|
||||
|
||||
def test_permanent_residents_survive_disable_and_enable_offload(monkeypatch):
|
||||
"""The compile-warmup and LoRA paths toggle offload off and on again.
|
||||
|
||||
Off loads every layer; on syncs the loaded layers back to their host
|
||||
stores and releases them. Permanent layers have no host store and must
|
||||
come out of that round trip untouched, with the streamed layer released
|
||||
as before.
|
||||
"""
|
||||
_patch_fake_device(monkeypatch)
|
||||
model = _MixinModel()
|
||||
model.configure_layerwise_offload(
|
||||
_server_args(
|
||||
dit_layerwise_offload=True,
|
||||
dit_layerwise_resident_layers=2,
|
||||
dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT,
|
||||
)
|
||||
)
|
||||
(manager,) = model.layerwise_offload_managers
|
||||
assert manager._permanent_set == frozenset({0, 1})
|
||||
expected = torch.arange(9, dtype=torch.float32).reshape(3, 3)
|
||||
|
||||
model.disable_offload()
|
||||
assert not is_layerwise_offloaded_module(model)
|
||||
model.enable_offload()
|
||||
assert is_layerwise_offloaded_module(model)
|
||||
|
||||
assert {0, 1} <= manager._gpu_layers
|
||||
for i in (0, 1):
|
||||
assert torch.equal(model.blocks[i].weight, expected)
|
||||
assert tuple(model.blocks[2].weight.shape) == (1,)
|
||||
|
||||
|
||||
def test_iter_materialized_weights_reads_permanent_residents_from_the_module(
|
||||
monkeypatch,
|
||||
):
|
||||
_patch_fake_device(monkeypatch)
|
||||
model = _MixinModel()
|
||||
model.configure_layerwise_offload(
|
||||
_server_args(
|
||||
dit_layerwise_offload=True,
|
||||
dit_layerwise_resident_layers=2,
|
||||
dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT,
|
||||
)
|
||||
)
|
||||
expected = torch.arange(9, dtype=torch.float32).reshape(3, 3)
|
||||
|
||||
materialized = dict(iter_materialized_weights(model))
|
||||
assert sorted(materialized) == sorted(name for name, _ in model.named_parameters())
|
||||
# permanent: the module's own tensor; streamed: rebuilt from the host store
|
||||
assert torch.equal(materialized["blocks.0.weight"], expected)
|
||||
assert torch.equal(materialized["blocks.2.weight"], expected)
|
||||
|
||||
|
||||
def test_permanent_residents_stay_out_of_the_host_pin_plan(monkeypatch):
|
||||
_patch_fake_device(monkeypatch)
|
||||
planned = {}
|
||||
original = LayerwiseOffloadManager._plan_layer_hosting
|
||||
|
||||
def spy(self, layer_groups):
|
||||
planned["layers"] = sorted(layer_groups)
|
||||
return original(self, layer_groups)
|
||||
|
||||
monkeypatch.setattr(LayerwiseOffloadManager, "_plan_layer_hosting", spy)
|
||||
_permanent_manager(_MultiBlockModel(6), num_layers=6, resident_layers=3)
|
||||
assert planned["layers"] == [3, 4, 5]
|
||||
|
||||
|
||||
def test_permanent_lifetime_with_no_resident_layers_warns_and_streams(monkeypatch):
|
||||
_patch_fake_device(monkeypatch)
|
||||
warnings = []
|
||||
monkeypatch.setattr(
|
||||
layerwise_offload_mod.logger,
|
||||
"warning",
|
||||
lambda message, *args: warnings.append(message % args),
|
||||
)
|
||||
manager = _permanent_manager(_MultiBlockModel(4), num_layers=4, resident_layers=0)
|
||||
assert manager._permanent_set == frozenset()
|
||||
assert manager._residency_active is False
|
||||
assert any("permanent residency with no resident layers" in w for w in warnings)
|
||||
|
||||
|
||||
def test_unknown_residency_lifetime_is_rejected(monkeypatch):
|
||||
_patch_fake_device(monkeypatch)
|
||||
with pytest.raises(ValueError, match="unknown residency lifetime"):
|
||||
_resident_manager(
|
||||
_MultiBlockModel(4),
|
||||
num_layers=4,
|
||||
resident_layers=2,
|
||||
residency_lifetime="sometimes",
|
||||
)
|
||||
|
||||
|
||||
def test_forward_lifetime_is_the_default_and_unchanged(monkeypatch):
|
||||
_patch_fake_device(monkeypatch)
|
||||
manager = _resident_manager(
|
||||
_MultiBlockModel(4), num_layers=4, prefetch_size=1, resident_layers=2
|
||||
)
|
||||
assert manager.residency_lifetime == RESIDENCY_LIFETIME_FORWARD
|
||||
assert manager._permanent_set == frozenset()
|
||||
assert manager._residency_active is False
|
||||
for i in range(4):
|
||||
assert i in manager._weight_metadata
|
||||
|
||||
|
||||
def test_layerwise_tuning_returns_the_lifetime():
|
||||
args = _server_args(
|
||||
dit_layerwise_resident_layers=20,
|
||||
dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT,
|
||||
layerwise_residency_lifetime="video_vae=forward",
|
||||
)
|
||||
assert (
|
||||
args.layerwise_tuning_for("transformer", dit_group=True)[3]
|
||||
== RESIDENCY_LIFETIME_PERMANENT
|
||||
)
|
||||
assert (
|
||||
args.layerwise_tuning_for("video_vae", dit_group=False)[3]
|
||||
== RESIDENCY_LIFETIME_FORWARD
|
||||
)
|
||||
# the DiT-group default is not the auxiliary default
|
||||
assert (
|
||||
args.layerwise_tuning_for("text_encoder", dit_group=False)[3]
|
||||
== RESIDENCY_LIFETIME_FORWARD
|
||||
)
|
||||
|
||||
|
||||
def test_layerwise_tuning_rejects_unknown_lifetime():
|
||||
args = _server_args(layerwise_residency_lifetime="vae=sometimes")
|
||||
with pytest.raises(ValueError, match="unknown residency lifetime"):
|
||||
args.layerwise_tuning_for("vae", dit_group=False)
|
||||
|
||||
|
||||
def test_configure_logs_a_permanent_lifetime(monkeypatch):
|
||||
_patch_fake_device(monkeypatch)
|
||||
logs = []
|
||||
monkeypatch.setattr(
|
||||
layerwise_offload_mod.logger,
|
||||
"info",
|
||||
lambda message, *args: logs.append(message % args),
|
||||
)
|
||||
comp = _ResidentComponent(8)
|
||||
comp.configure_layerwise_offload(
|
||||
_server_args(
|
||||
dit_offload_prefetch_size=2,
|
||||
dit_layerwise_resident_layers=3,
|
||||
dit_layerwise_residency_lifetime=RESIDENCY_LIFETIME_PERMANENT,
|
||||
),
|
||||
component_name="transformer",
|
||||
)
|
||||
assert any("placed 3 permanent resident layers" in line for line in logs)
|
||||
assert logs[-1].endswith("resident=3/8 (permanent), policy=leading")
|
||||
|
||||
@@ -3412,6 +3412,75 @@ class TestDirectGpuWeightLoading(unittest.TestCase):
|
||||
tp_args._validate_direct_gpu_weight_loading()
|
||||
|
||||
|
||||
class TestLayerwiseResidencyLifetime(unittest.TestCase):
|
||||
def _help_by_option(self):
|
||||
parser = FlexibleArgumentParser()
|
||||
ServerArgs.add_cli_args(parser)
|
||||
return {
|
||||
action.option_strings[0]: (action.help or "")
|
||||
for action in parser._actions
|
||||
if action.option_strings
|
||||
}
|
||||
|
||||
def test_cli_accepts_the_two_lifetimes_and_rejects_others(self):
|
||||
parser = FlexibleArgumentParser()
|
||||
ServerArgs.add_cli_args(parser)
|
||||
args = parser.parse_args(
|
||||
["--model-path", "/fake", "--dit-layerwise-residency-lifetime", "permanent"]
|
||||
)
|
||||
self.assertEqual(args.dit_layerwise_residency_lifetime, "permanent")
|
||||
self.assertEqual(
|
||||
parser.parse_args(
|
||||
["--model-path", "/fake"]
|
||||
).dit_layerwise_residency_lifetime,
|
||||
"forward",
|
||||
)
|
||||
with self.assertRaises(SystemExit):
|
||||
parser.parse_args(
|
||||
[
|
||||
"--model-path",
|
||||
"/fake",
|
||||
"--dit-layerwise-residency-lifetime",
|
||||
"sometimes",
|
||||
]
|
||||
)
|
||||
|
||||
def test_python_api_rejects_an_unknown_lifetime(self):
|
||||
# argparse choices cover the CLI only; ServerArgs is also built directly.
|
||||
with self.assertRaisesRegex(ValueError, "dit-layerwise-residency-lifetime"):
|
||||
_from_dict_without_model_resolution(
|
||||
{
|
||||
"model_path": "/data/my-model",
|
||||
"dit_layerwise_residency_lifetime": "sometimes",
|
||||
}
|
||||
)
|
||||
|
||||
def test_help_says_when_layers_are_placed_and_released(self):
|
||||
"""A user reads these to decide between the two; both answers must be there."""
|
||||
help_by_option = self._help_by_option()
|
||||
dit_help = help_by_option["--dit-layerwise-residency-lifetime"]
|
||||
for claim in (
|
||||
"'forward' (default)",
|
||||
"'permanent'",
|
||||
"load time",
|
||||
"never released",
|
||||
):
|
||||
self.assertIn(claim, dit_help)
|
||||
per_component_help = help_by_option["--layerwise-residency-lifetime"]
|
||||
for claim in ("transformer=permanent", "never releases"):
|
||||
self.assertIn(claim, per_component_help)
|
||||
# The resident-layer flags point at the lifetime knob instead of
|
||||
# describing a lifetime of their own.
|
||||
self.assertIn(
|
||||
"--dit-layerwise-residency-lifetime",
|
||||
help_by_option["--dit-layerwise-resident-layers"],
|
||||
)
|
||||
self.assertIn(
|
||||
"--layerwise-residency-lifetime",
|
||||
help_by_option["--layerwise-resident-layers"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user