From c02eaadeb28c1e0b41e9730d18fb78dbcd4d7f01 Mon Sep 17 00:00:00 2001 From: Mick Date: Mon, 28 Sep 2026 17:45:58 +0800 Subject: [PATCH] [diffusion] optimization: populate cpu weight stores before host registration to speedup layerwise-offload initialization (#40439) Co-authored-by: Mick Qian --- .../memory_managers/layerwise_offload.py | 61 ++++++-- .../test/unit/test_layerwise_offload.py | 144 ++++++++++++++++++ 2 files changed, 195 insertions(+), 10 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py index 9d024cf216..370b7082d4 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py @@ -1486,7 +1486,7 @@ class LayerwiseOffloadManager: def _initialize_host_stores( self, layer_groups: Dict, layer_hosting: Dict[int, str] ) -> Iterator[torch.UntypedStorage]: - """Yield each pinned allocation before copying weights to transfer its lease.""" + """Yield each pinned allocation to transfer its lease.""" # 2. concat and offload (in pinned memory) for layer_idx, dtype_to_params in layer_groups.items(): self._consolidated_cpu_weights[layer_idx] = {} @@ -1580,21 +1580,23 @@ class LayerwiseOffloadManager: total_numel = current_offset - # create concatenated CPU buffer (in pinned memory) - if pin_this_layer: - cpu_buffer = _pinned_empty(total_numel, dtype=dtype) - else: - cpu_buffer = torch.empty(total_numel, dtype=dtype) + cpu_buffer, populated = self._prepare_host_buffer( + contiguous_weights, + aligned_offsets, + total_numel, + dtype, + pin_this_layer, + ) if pin_this_layer: yield cpu_buffer.untyped_storage() - # offload weights to the buffer for name, weight, local_weight in contiguous_weights: current_offset = aligned_offsets[name] numel = local_weight.numel() - cpu_buffer[current_offset : current_offset + numel].copy_( - local_weight.flatten() - ) + if not populated: + cpu_buffer[current_offset : current_offset + numel].copy_( + local_weight.flatten() + ) self._weight_metadata[layer_idx][name] = { "dtype": dtype, "offset": current_offset, @@ -1612,6 +1614,45 @@ class LayerwiseOffloadManager: self._consolidated_cpu_weights[layer_idx][dtype] = cpu_buffer + @staticmethod + def _prepare_host_buffer( + weights: List[Tuple[str, torch.Tensor, torch.Tensor]], + offsets: Dict[str, int], + total_numel: int, + dtype: torch.dtype, + pin: bool, + ) -> Tuple[torch.Tensor, bool]: + """Return a host buffer and whether CPU weights already populate it.""" + nbytes = total_numel * dtype.itemsize + if ( + pin + and nbytes >= _REGISTER_MIN_BYTES + and all(local.device.type == "cpu" for _, _, local in weights) + ): + storage = _shared_storage(nbytes) + if storage is not None: + buffer = torch.empty(0, dtype=dtype).set_( + storage, 0, (total_numel,), (1,) + ) + # CPU copies populate pages in parallel before registration; + # GPU copies must keep their pinned destination to avoid staging + for name, _, local in weights: + offset = offsets[name] + buffer[offset : offset + local.numel()].copy_(local.flatten()) + if _pin_in_place(buffer): + return buffer, True + # retain the originals until registration succeeds so fallback + # does not hold an extra full-size populated allocation + del buffer, storage + + buffer = ( + _pinned_empty(total_numel, dtype=dtype) + if pin + else torch.empty(total_numel, dtype=dtype) + ) + # the caller must attach the pin lease before a fallible pinned copy + return buffer, False + def _finalize_initialization(self) -> None: # prefetch the head of the stream for warm-up. Under `forward` residency # is not armed yet, so this is layer 0 regardless of policy; permanent diff --git a/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py b/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py index 58e52e0350..27cfdaa64c 100644 --- a/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py +++ b/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py @@ -620,6 +620,150 @@ def test_failed_layerwise_allocation_refunds_only_unallocated_allowance(monkeypa assert budget.committed_bytes == 64 +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("source_device", ["cpu", "cuda"]) +@pytest.mark.parametrize("fallback", [None, "register", "shared_storage"]) +def test_consolidated_host_copy_order_and_fallback( + monkeypatch, source_device, fallback +): + numel = layerwise_offload_mod._REGISTER_MIN_BYTES // 4 + expected = torch.arange(numel, dtype=torch.float32) + model = torch.nn.Module() + block = torch.nn.Module() + block.weight = torch.nn.Parameter(expected.to(source_device).clone()) + model.blocks = torch.nn.ModuleList([block]) + budget = host_memory_budget.HostPinBudget( + available_bytes=host_memory_budget.MIN_HOST_RESERVE_BYTES + numel * 4 + ) + register = layerwise_offload_mod._pin_in_place + registrations = [] + + def observe_registration(tensor): + if ( + source_device == "cpu" + and fallback != "shared_storage" + and not registrations + ): + torch.testing.assert_close(tensor, expected, rtol=0, atol=0) + registrations.append(tensor.data_ptr()) + return False if fallback == "register" else register(tensor) + + monkeypatch.setattr(layerwise_offload_mod, "_pin_in_place", observe_registration) + if fallback == "shared_storage": + monkeypatch.setattr(layerwise_offload_mod, "_shared_storage", lambda _: None) + manager = LayerwiseOffloadManager( + model=model, + layers_attr_str="blocks", + num_layers=1, + enabled=True, + pin_cpu_memory=True, + pin_budget=budget, + ) + assert registrations + host_alias = manager._consolidated_cpu_weights[0][torch.float32].detach() + assert host_alias.is_pinned() + assert host_alias.untyped_storage().nbytes() == numel * 4 + torch.testing.assert_close(host_alias, expected, rtol=0, atol=0) + assert budget.committed_bytes == numel * 4 + manager.load_all_layers() + torch.cuda.synchronize() + torch.testing.assert_close(block.weight.cpu(), expected, rtol=0, atol=0) + manager.remove_forward_hooks() + manager.enabled = False + manager.release_host_stores() + assert budget.committed_bytes == numel * 4 + del host_alias, manager, model, block + gc.collect() + assert budget.committed_bytes == 0 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_cpu_copy_failure_returns_unregistered_pin_allowance(monkeypatch): + numel = layerwise_offload_mod._REGISTER_MIN_BYTES // 4 + model = torch.nn.Module() + block = torch.nn.Module() + block.weight = torch.nn.Parameter(torch.ones(numel)) + model.blocks = torch.nn.ModuleList([block]) + budget = host_memory_budget.HostPinBudget( + available_bytes=host_memory_budget.MIN_HOST_RESERVE_BYTES + numel * 4 + 64 + ) + assert budget.request(component_name="other", weight_bytes=64) + manager = LayerwiseOffloadManager( + model=model, + layers_attr_str="blocks", + num_layers=1, + enabled=True, + initialize=False, + pin_cpu_memory=True, + pin_budget=budget, + ) + original_copy = torch.Tensor.copy_ + + def fail_copy(target, source, *args, **kwargs): + if target.device.type == "cpu" and source.numel() == numel: + raise RuntimeError("CPU weight copy failed") + return original_copy(target, source, *args, **kwargs) + + monkeypatch.setattr(torch.Tensor, "copy_", fail_copy) + with pytest.raises(RuntimeError, match="CPU weight copy failed"): + manager.initialize() + gc.collect() + assert budget.committed_bytes == 64 + assert block.weight.numel() == numel + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize( + "source_device,numel,registration_failure", + [ + ("cpu", 4, False), + ("cuda", layerwise_offload_mod._REGISTER_MIN_BYTES // 4, False), + ("cpu", layerwise_offload_mod._REGISTER_MIN_BYTES // 4, True), + ], +) +def test_pinned_copy_failure_retains_lease_for_live_alias( + monkeypatch, source_device, numel, registration_failure +): + model = torch.nn.Module() + block = torch.nn.Module() + block.weight = torch.nn.Parameter(torch.ones(numel, device=source_device)) + model.blocks = torch.nn.ModuleList([block]) + budget = host_memory_budget.HostPinBudget( + available_bytes=host_memory_budget.MIN_HOST_RESERVE_BYTES + numel * 4 + 64 + ) + assert budget.request(component_name="other", weight_bytes=64) + manager = LayerwiseOffloadManager( + model=model, + layers_attr_str="blocks", + num_layers=1, + enabled=True, + initialize=False, + pin_cpu_memory=True, + pin_budget=budget, + ) + if registration_failure: + monkeypatch.setattr(layerwise_offload_mod, "_pin_in_place", lambda _: False) + original_copy = torch.Tensor.copy_ + aliases = [] + + def fail_pinned_copy(target, source, *args, **kwargs): + if target.device.type == "cpu" and target.is_pinned(): + aliases.append(target.detach()) + raise RuntimeError("pinned weight copy failed") + return original_copy(target, source, *args, **kwargs) + + monkeypatch.setattr(torch.Tensor, "copy_", fail_pinned_copy) + with pytest.raises(RuntimeError, match="pinned weight copy failed"): + manager.initialize() + assert len(aliases) == 1 + assert aliases[0].is_pinned() + assert budget.committed_bytes == 64 + numel * 4 + assert block.weight.numel() == numel + aliases.clear() + gc.collect() + assert budget.committed_bytes == 64 + + def test_layerwise_configuration_filters_by_component_name(monkeypatch): monkeypatch.setattr( layerwise_offload_mod.torch, "get_device_module", lambda: _FakeDeviceModule