[diffusion] optimization: populate cpu weight stores before host registration to speedup layerwise-offload initialization (#40439)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-09-28 17:45:58 +08:00
committed by GitHub
co-authored by Mick Qian
parent 6582425854
commit c02eaadeb2
2 changed files with 195 additions and 10 deletions
@@ -1486,7 +1486,7 @@ class LayerwiseOffloadManager:
def _initialize_host_stores(
self, layer_groups: Dict, layer_hosting: Dict[int, str]
) -> Iterator[torch.UntypedStorage]:
"""Yield each pinned allocation before copying weights to transfer its lease."""
"""Yield each pinned allocation to transfer its lease."""
# 2. concat and offload (in pinned memory)
for layer_idx, dtype_to_params in layer_groups.items():
self._consolidated_cpu_weights[layer_idx] = {}
@@ -1580,21 +1580,23 @@ class LayerwiseOffloadManager:
total_numel = current_offset
# create concatenated CPU buffer (in pinned memory)
if pin_this_layer:
cpu_buffer = _pinned_empty(total_numel, dtype=dtype)
else:
cpu_buffer = torch.empty(total_numel, dtype=dtype)
cpu_buffer, populated = self._prepare_host_buffer(
contiguous_weights,
aligned_offsets,
total_numel,
dtype,
pin_this_layer,
)
if pin_this_layer:
yield cpu_buffer.untyped_storage()
# offload weights to the buffer
for name, weight, local_weight in contiguous_weights:
current_offset = aligned_offsets[name]
numel = local_weight.numel()
cpu_buffer[current_offset : current_offset + numel].copy_(
local_weight.flatten()
)
if not populated:
cpu_buffer[current_offset : current_offset + numel].copy_(
local_weight.flatten()
)
self._weight_metadata[layer_idx][name] = {
"dtype": dtype,
"offset": current_offset,
@@ -1612,6 +1614,45 @@ class LayerwiseOffloadManager:
self._consolidated_cpu_weights[layer_idx][dtype] = cpu_buffer
@staticmethod
def _prepare_host_buffer(
weights: List[Tuple[str, torch.Tensor, torch.Tensor]],
offsets: Dict[str, int],
total_numel: int,
dtype: torch.dtype,
pin: bool,
) -> Tuple[torch.Tensor, bool]:
"""Return a host buffer and whether CPU weights already populate it."""
nbytes = total_numel * dtype.itemsize
if (
pin
and nbytes >= _REGISTER_MIN_BYTES
and all(local.device.type == "cpu" for _, _, local in weights)
):
storage = _shared_storage(nbytes)
if storage is not None:
buffer = torch.empty(0, dtype=dtype).set_(
storage, 0, (total_numel,), (1,)
)
# CPU copies populate pages in parallel before registration;
# GPU copies must keep their pinned destination to avoid staging
for name, _, local in weights:
offset = offsets[name]
buffer[offset : offset + local.numel()].copy_(local.flatten())
if _pin_in_place(buffer):
return buffer, True
# retain the originals until registration succeeds so fallback
# does not hold an extra full-size populated allocation
del buffer, storage
buffer = (
_pinned_empty(total_numel, dtype=dtype)
if pin
else torch.empty(total_numel, dtype=dtype)
)
# the caller must attach the pin lease before a fallible pinned copy
return buffer, False
def _finalize_initialization(self) -> None:
# prefetch the head of the stream for warm-up. Under `forward` residency
# is not armed yet, so this is layer 0 regardless of policy; permanent
@@ -620,6 +620,150 @@ def test_failed_layerwise_allocation_refunds_only_unallocated_allowance(monkeypa
assert budget.committed_bytes == 64
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("source_device", ["cpu", "cuda"])
@pytest.mark.parametrize("fallback", [None, "register", "shared_storage"])
def test_consolidated_host_copy_order_and_fallback(
monkeypatch, source_device, fallback
):
numel = layerwise_offload_mod._REGISTER_MIN_BYTES // 4
expected = torch.arange(numel, dtype=torch.float32)
model = torch.nn.Module()
block = torch.nn.Module()
block.weight = torch.nn.Parameter(expected.to(source_device).clone())
model.blocks = torch.nn.ModuleList([block])
budget = host_memory_budget.HostPinBudget(
available_bytes=host_memory_budget.MIN_HOST_RESERVE_BYTES + numel * 4
)
register = layerwise_offload_mod._pin_in_place
registrations = []
def observe_registration(tensor):
if (
source_device == "cpu"
and fallback != "shared_storage"
and not registrations
):
torch.testing.assert_close(tensor, expected, rtol=0, atol=0)
registrations.append(tensor.data_ptr())
return False if fallback == "register" else register(tensor)
monkeypatch.setattr(layerwise_offload_mod, "_pin_in_place", observe_registration)
if fallback == "shared_storage":
monkeypatch.setattr(layerwise_offload_mod, "_shared_storage", lambda _: None)
manager = LayerwiseOffloadManager(
model=model,
layers_attr_str="blocks",
num_layers=1,
enabled=True,
pin_cpu_memory=True,
pin_budget=budget,
)
assert registrations
host_alias = manager._consolidated_cpu_weights[0][torch.float32].detach()
assert host_alias.is_pinned()
assert host_alias.untyped_storage().nbytes() == numel * 4
torch.testing.assert_close(host_alias, expected, rtol=0, atol=0)
assert budget.committed_bytes == numel * 4
manager.load_all_layers()
torch.cuda.synchronize()
torch.testing.assert_close(block.weight.cpu(), expected, rtol=0, atol=0)
manager.remove_forward_hooks()
manager.enabled = False
manager.release_host_stores()
assert budget.committed_bytes == numel * 4
del host_alias, manager, model, block
gc.collect()
assert budget.committed_bytes == 0
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_cpu_copy_failure_returns_unregistered_pin_allowance(monkeypatch):
numel = layerwise_offload_mod._REGISTER_MIN_BYTES // 4
model = torch.nn.Module()
block = torch.nn.Module()
block.weight = torch.nn.Parameter(torch.ones(numel))
model.blocks = torch.nn.ModuleList([block])
budget = host_memory_budget.HostPinBudget(
available_bytes=host_memory_budget.MIN_HOST_RESERVE_BYTES + numel * 4 + 64
)
assert budget.request(component_name="other", weight_bytes=64)
manager = LayerwiseOffloadManager(
model=model,
layers_attr_str="blocks",
num_layers=1,
enabled=True,
initialize=False,
pin_cpu_memory=True,
pin_budget=budget,
)
original_copy = torch.Tensor.copy_
def fail_copy(target, source, *args, **kwargs):
if target.device.type == "cpu" and source.numel() == numel:
raise RuntimeError("CPU weight copy failed")
return original_copy(target, source, *args, **kwargs)
monkeypatch.setattr(torch.Tensor, "copy_", fail_copy)
with pytest.raises(RuntimeError, match="CPU weight copy failed"):
manager.initialize()
gc.collect()
assert budget.committed_bytes == 64
assert block.weight.numel() == numel
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize(
"source_device,numel,registration_failure",
[
("cpu", 4, False),
("cuda", layerwise_offload_mod._REGISTER_MIN_BYTES // 4, False),
("cpu", layerwise_offload_mod._REGISTER_MIN_BYTES // 4, True),
],
)
def test_pinned_copy_failure_retains_lease_for_live_alias(
monkeypatch, source_device, numel, registration_failure
):
model = torch.nn.Module()
block = torch.nn.Module()
block.weight = torch.nn.Parameter(torch.ones(numel, device=source_device))
model.blocks = torch.nn.ModuleList([block])
budget = host_memory_budget.HostPinBudget(
available_bytes=host_memory_budget.MIN_HOST_RESERVE_BYTES + numel * 4 + 64
)
assert budget.request(component_name="other", weight_bytes=64)
manager = LayerwiseOffloadManager(
model=model,
layers_attr_str="blocks",
num_layers=1,
enabled=True,
initialize=False,
pin_cpu_memory=True,
pin_budget=budget,
)
if registration_failure:
monkeypatch.setattr(layerwise_offload_mod, "_pin_in_place", lambda _: False)
original_copy = torch.Tensor.copy_
aliases = []
def fail_pinned_copy(target, source, *args, **kwargs):
if target.device.type == "cpu" and target.is_pinned():
aliases.append(target.detach())
raise RuntimeError("pinned weight copy failed")
return original_copy(target, source, *args, **kwargs)
monkeypatch.setattr(torch.Tensor, "copy_", fail_pinned_copy)
with pytest.raises(RuntimeError, match="pinned weight copy failed"):
manager.initialize()
assert len(aliases) == 1
assert aliases[0].is_pinned()
assert budget.committed_bytes == 64 + numel * 4
assert block.weight.numel() == numel
aliases.clear()
gc.collect()
assert budget.committed_bytes == 64
def test_layerwise_configuration_filters_by_component_name(monkeypatch):
monkeypatch.setattr(
layerwise_offload_mod.torch, "get_device_module", lambda: _FakeDeviceModule