[diffusion] feat: support in-place lora merge/unmerge under layerwise offload (#36192)
This commit is contained in:
@@ -32,6 +32,9 @@ from sglang.multimodal_gen.runtime.layers.linear import (
|
||||
from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
write_dense_weight,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.precision import get_mixed_precision_state
|
||||
|
||||
torch._dynamo.config.recompile_limit = 64
|
||||
@@ -127,8 +130,17 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
return isinstance(self.base_layer.quant_method, UnquantizedLinearMethod)
|
||||
return True
|
||||
|
||||
@torch.compile()
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# Layerwise rebinds Parameter.data; do not compile the merged path.
|
||||
if self.disable_lora or (self.merged and not self.has_lora_output_offset):
|
||||
return self.base_layer(x)
|
||||
if self.merged:
|
||||
out, output_bias = self.base_layer(x)
|
||||
return self._add_lora_output_offset(out), output_bias
|
||||
return self._forward_with_delta(x)
|
||||
|
||||
@torch.compile()
|
||||
def _forward_with_delta(self, x: torch.Tensor) -> torch.Tensor:
|
||||
lora_A = self.lora_A
|
||||
lora_B = self.lora_B
|
||||
if isinstance(self.lora_B, DTensor):
|
||||
@@ -136,25 +148,22 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
lora_A = self.lora_A.to_local()
|
||||
|
||||
# TODO: Support multiple LoRA adapters when use not merged mode
|
||||
if not self.merged and not self.disable_lora:
|
||||
lora_dtype = lora_A.dtype
|
||||
x_lora = x.to(dtype=lora_dtype)
|
||||
lora_A_sliced = self.slice_lora_a_weights(
|
||||
lora_A.to(device=x.device, non_blocking=True)
|
||||
)
|
||||
lora_B_sliced = self.slice_lora_b_weights(
|
||||
lora_B.to(device=x.device, non_blocking=True)
|
||||
)
|
||||
delta = _compute_lora_delta(x_lora, lora_A_sliced, lora_B_sliced)
|
||||
if self.lora_alpha != self.lora_rank:
|
||||
delta = delta * (
|
||||
self.lora_alpha / self.lora_rank # type: ignore
|
||||
) # type: ignore
|
||||
delta = delta * self.strength
|
||||
out, output_bias = self.base_layer(x)
|
||||
out = out + delta.to(dtype=out.dtype)
|
||||
else:
|
||||
out, output_bias = self.base_layer(x)
|
||||
lora_dtype = lora_A.dtype
|
||||
x_lora = x.to(dtype=lora_dtype)
|
||||
lora_A_sliced = self.slice_lora_a_weights(
|
||||
lora_A.to(device=x.device, non_blocking=True)
|
||||
)
|
||||
lora_B_sliced = self.slice_lora_b_weights(
|
||||
lora_B.to(device=x.device, non_blocking=True)
|
||||
)
|
||||
delta = _compute_lora_delta(x_lora, lora_A_sliced, lora_B_sliced)
|
||||
if self.lora_alpha != self.lora_rank:
|
||||
delta = delta * (
|
||||
self.lora_alpha / self.lora_rank # type: ignore
|
||||
) # type: ignore
|
||||
delta = delta * self.strength
|
||||
out, output_bias = self.base_layer(x)
|
||||
out = out + delta.to(dtype=out.dtype)
|
||||
return self._add_lora_output_offset(out), output_bias
|
||||
|
||||
def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor:
|
||||
@@ -279,6 +288,43 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
elif self.merged:
|
||||
self.unmerge_lora_weights()
|
||||
|
||||
def _active_lora_list(self) -> list[LoRAWeightEntry]:
|
||||
if self.lora_weights_list:
|
||||
return list(self.lora_weights_list)
|
||||
if self.lora_A is not None and self.lora_B is not None:
|
||||
return [
|
||||
(
|
||||
self.lora_A,
|
||||
self.lora_B,
|
||||
self.lora_path,
|
||||
self.strength,
|
||||
self.lora_rank,
|
||||
self.lora_alpha,
|
||||
self.lora_output_offset,
|
||||
)
|
||||
]
|
||||
return []
|
||||
|
||||
def _materialized_weight_src(self) -> tuple[torch.Tensor, torch.device]:
|
||||
"""Layerwise CPU view, or the live GPU weight if it is not a (1,) placeholder."""
|
||||
packed = getattr(self.base_layer, "_packed_weight_cpu", None)
|
||||
if packed is None or packed.numel() <= 1:
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
_refresh_packed_from_manager,
|
||||
)
|
||||
|
||||
_refresh_packed_from_manager(self.base_layer)
|
||||
packed = getattr(self.base_layer, "_packed_weight_cpu", None)
|
||||
weight = self.base_layer.weight.data
|
||||
if weight.numel() <= 1:
|
||||
if packed is None or packed.numel() <= 1:
|
||||
raise RuntimeError(
|
||||
"layerwise LoRA saw a (1,) placeholder with no CPU view; "
|
||||
"convert_to_lora_layers must bind _packed_weight_cpu first."
|
||||
)
|
||||
return packed, torch.device("cpu")
|
||||
return weight, weight.device
|
||||
|
||||
def _ensure_base_snapshot_owned(self) -> None:
|
||||
"""An in-place merge is about to write the base storage; if the
|
||||
snapshot is a zero-copy view into it, materialize the clone now."""
|
||||
@@ -427,25 +473,13 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
if self.disable_lora:
|
||||
return
|
||||
|
||||
self._ensure_base_snapshot_owned()
|
||||
# Zero-copy views unmerge by subtracting B@A; do not clone the base.
|
||||
if not self._base_is_view:
|
||||
self._ensure_base_snapshot_owned()
|
||||
if self.merged:
|
||||
self.unmerge_lora_weights()
|
||||
|
||||
# Use lora_weights_list if available, otherwise fall back to single LoRA for backward compatibility
|
||||
lora_list = self.lora_weights_list if self.lora_weights_list else []
|
||||
if not lora_list and self.lora_A is not None and self.lora_B is not None:
|
||||
lora_list = [
|
||||
(
|
||||
self.lora_A,
|
||||
self.lora_B,
|
||||
self.lora_path,
|
||||
self.strength,
|
||||
self.lora_rank,
|
||||
self.lora_alpha,
|
||||
self.lora_output_offset,
|
||||
)
|
||||
]
|
||||
|
||||
lora_list = self._active_lora_list()
|
||||
if not lora_list:
|
||||
raise ValueError("LoRA weights not set. Please set them first.")
|
||||
if isinstance(self.base_layer.weight, DTensor) and any(
|
||||
@@ -509,27 +543,35 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
offload_policy=offload_policy,
|
||||
)
|
||||
else:
|
||||
current_device = self.base_layer.weight.data.device
|
||||
data = self.base_layer.weight.data.to(get_local_torch_device())
|
||||
data = self._as_mutable_tensor(data)
|
||||
target_dtype = data.dtype
|
||||
if (
|
||||
merge_in_fp32
|
||||
and data.is_floating_point()
|
||||
and data.dtype != torch.float32
|
||||
):
|
||||
data = data.to(torch.float32)
|
||||
|
||||
self._merge_lora_into_data(data, lora_list)
|
||||
|
||||
self.base_layer.weight.data = self._as_mutable_tensor(
|
||||
data.to(current_device, dtype=target_dtype, non_blocking=True)
|
||||
)
|
||||
src, _ = self._materialized_weight_src()
|
||||
data = src.to(get_local_torch_device())
|
||||
self._merge_from_device_data(data, lora_list, merge_in_fp32)
|
||||
return
|
||||
|
||||
self.merged = True
|
||||
|
||||
def _merge_from_device_data(
|
||||
self,
|
||||
data: torch.Tensor,
|
||||
lora_list: list[LoRAWeightEntry] | None = None,
|
||||
merge_in_fp32: bool | None = None,
|
||||
) -> None:
|
||||
"""Merge on GPU and write back to the layer store."""
|
||||
lora_list = lora_list if lora_list is not None else self._active_lora_list()
|
||||
if not lora_list:
|
||||
raise ValueError("LoRA weights not set. Please set them first.")
|
||||
if merge_in_fp32 is None:
|
||||
merge_in_fp32 = self._should_merge_in_fp32(lora_list)
|
||||
data = self._as_mutable_tensor(data)
|
||||
target_dtype = data.dtype
|
||||
if merge_in_fp32 and data.is_floating_point() and data.dtype != torch.float32:
|
||||
data = data.to(torch.float32)
|
||||
self._merge_lora_into_data(data, lora_list)
|
||||
merged = self._as_mutable_tensor(data.to(dtype=target_dtype))
|
||||
write_dense_weight(self.base_layer, merged)
|
||||
self.merged = True
|
||||
|
||||
@torch.no_grad()
|
||||
# @torch.compile(dynamic=True)
|
||||
def unmerge_lora_weights(self) -> None:
|
||||
if self.disable_lora:
|
||||
return
|
||||
@@ -548,6 +590,8 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
)
|
||||
self.base_layer.weight = nn.Parameter(new_weight_data)
|
||||
del old_weight
|
||||
elif self._base_is_view:
|
||||
self._unmerge_by_inverse()
|
||||
else:
|
||||
current_device = self.base_layer.weight.data.device
|
||||
cpu_weight_on_device = self.cpu_weight.to(current_device, non_blocking=True)
|
||||
@@ -565,6 +609,48 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
|
||||
self.merged = False
|
||||
|
||||
@torch.no_grad()
|
||||
def _unmerge_by_inverse(self) -> None:
|
||||
lora_list = self._active_lora_list()
|
||||
if not lora_list:
|
||||
raise ValueError(
|
||||
"LoRA weights not set; cannot unmerge a zero-copy view by inverse."
|
||||
)
|
||||
src, current_device = self._materialized_weight_src()
|
||||
data = self._as_mutable_tensor(src.to(get_local_torch_device()))
|
||||
target_dtype = data.dtype
|
||||
if (
|
||||
self._should_merge_in_fp32(lora_list)
|
||||
and data.is_floating_point()
|
||||
and data.dtype != torch.float32
|
||||
):
|
||||
data = data.to(torch.float32)
|
||||
inverted = [
|
||||
(
|
||||
lora_A,
|
||||
lora_B,
|
||||
lora_path,
|
||||
-lora_strength,
|
||||
lora_rank,
|
||||
lora_alpha,
|
||||
output_offset,
|
||||
)
|
||||
for (
|
||||
lora_A,
|
||||
lora_B,
|
||||
lora_path,
|
||||
lora_strength,
|
||||
lora_rank,
|
||||
lora_alpha,
|
||||
output_offset,
|
||||
) in reversed(lora_list)
|
||||
]
|
||||
self._merge_lora_into_data(data, inverted)
|
||||
write_dense_weight(
|
||||
self.base_layer,
|
||||
self._as_mutable_tensor(data.to(dtype=target_dtype)),
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def commit_merged_as_base(self) -> None:
|
||||
"""Promote the currently merged weights to the permanent base.
|
||||
@@ -882,7 +968,11 @@ class LinearWithLoRA(BaseLayerWithLoRA):
|
||||
return self._add_lora_output_offset(out)
|
||||
|
||||
|
||||
def _use_owned_base_snapshot(snapshot_base: bool, device_type: str) -> bool:
|
||||
def _use_owned_base_snapshot(
|
||||
snapshot_base: bool, device_type: str, numel: int | None = None
|
||||
) -> bool:
|
||||
if numel is not None and numel <= 1:
|
||||
return snapshot_base
|
||||
return snapshot_base or device_type not in ("cpu", "meta")
|
||||
|
||||
|
||||
@@ -910,7 +1000,7 @@ def wrap_with_lora_layer(
|
||||
for src_layer_type, lora_layer_type in supported_layer_types.items():
|
||||
if isinstance(layer, src_layer_type): # type: ignore[arg-type]
|
||||
effective_snapshot_base = _use_owned_base_snapshot(
|
||||
snapshot_base, layer.weight.device.type
|
||||
snapshot_base, layer.weight.device.type, layer.weight.numel()
|
||||
)
|
||||
ret = lora_layer_type(
|
||||
layer,
|
||||
|
||||
+279
-30
@@ -1134,6 +1134,7 @@ class LayerwiseOffloadManager:
|
||||
self._gpu_layers: Set[int] = set()
|
||||
# mapped layers handed to the courier and not yet collected
|
||||
self._mapped_courier: Optional[MappedLayerCourier] = None
|
||||
self._mapped_store_dirty = False
|
||||
self._courier_inflight: Set[int] = set()
|
||||
# layer_idx -> torch.get_device_module().Event for fine-grained sync, to make sure the weight is resident in pre-hook
|
||||
self._prefetch_events: Dict[int, torch.get_device_module().Event] = {}
|
||||
@@ -1923,10 +1924,52 @@ class LayerwiseOffloadManager:
|
||||
def get_target_with_name(self, name: str) -> torch.Tensor:
|
||||
"""get the target model weight/buffer to be replaced"""
|
||||
if name in self._named_parameters:
|
||||
target = self._named_parameters[name]
|
||||
else:
|
||||
target = self._named_buffers[name]
|
||||
return target
|
||||
return self._named_parameters[name]
|
||||
if name in self._named_buffers:
|
||||
return self._named_buffers[name]
|
||||
for alias in offload_param_name_aliases(name):
|
||||
if alias in self._named_parameters:
|
||||
return self._named_parameters[alias]
|
||||
if alias in self._named_buffers:
|
||||
return self._named_buffers[alias]
|
||||
raise KeyError(name)
|
||||
|
||||
def refresh_named_targets(self) -> None:
|
||||
"""Repoint manager targets after LoRA wrap (``*.base_layer.weight``)."""
|
||||
live = dict(self.model.named_parameters())
|
||||
live_buffers = dict(self.model.named_buffers())
|
||||
for name in list(self._named_parameters):
|
||||
for alias in offload_param_name_aliases(name):
|
||||
if alias in live:
|
||||
self._named_parameters[name] = live[alias]
|
||||
break
|
||||
for name in list(self._named_buffers):
|
||||
if name in live_buffers:
|
||||
self._named_buffers[name] = live_buffers[name]
|
||||
|
||||
def get_cpu_weight(self, name: str) -> torch.Tensor | None:
|
||||
"""Current manager CPU tensor for ``name``, or None."""
|
||||
if not self.has_cpu_weight(name):
|
||||
return None
|
||||
layer_idx = self._match_layer_idx(name)
|
||||
if layer_idx is None:
|
||||
return None
|
||||
if self._synchronous_mps:
|
||||
return self._mps_cpu_weights.get(layer_idx, {}).get(name)
|
||||
meta = self._weight_metadata.get(layer_idx, {}).get(name)
|
||||
if meta is None:
|
||||
return None
|
||||
if meta.get("mapped", False):
|
||||
return self._mapped_cpu_weights[layer_idx][name]
|
||||
if meta.get("preserve_strides", False):
|
||||
return self._strided_cpu_weights[layer_idx][name]
|
||||
dtype = meta["dtype"]
|
||||
offset = meta["offset"]
|
||||
numel = meta["numel"]
|
||||
shape = meta["shape"]
|
||||
return self._consolidated_cpu_weights[layer_idx][dtype][
|
||||
offset : offset + numel
|
||||
].reshape(shape)
|
||||
|
||||
@torch.compiler.disable
|
||||
def prefetch_layer(self, layer_idx: int, non_blocking: bool = True) -> None:
|
||||
@@ -1937,6 +1980,9 @@ class LayerwiseOffloadManager:
|
||||
return
|
||||
if layer_idx < 0 or layer_idx >= self.num_layers:
|
||||
return
|
||||
finish_offload_writeback()
|
||||
if self._mapped_store_dirty:
|
||||
self._reset_mapped_courier()
|
||||
if layer_idx in self._gpu_layers:
|
||||
return
|
||||
if layer_idx in self._courier_inflight:
|
||||
@@ -2087,6 +2133,15 @@ class LayerwiseOffloadManager:
|
||||
courier = self._ensure_mapped_courier()
|
||||
return courier is not None and courier.direct_read
|
||||
|
||||
def _reset_mapped_courier(self) -> None:
|
||||
"""Drop courier state after LoRA rewrites a mapped CPU store."""
|
||||
courier = self._mapped_courier
|
||||
if courier is not None:
|
||||
courier.close()
|
||||
self._mapped_courier = None
|
||||
self._courier_inflight.clear()
|
||||
self._mapped_store_dirty = False
|
||||
|
||||
def _ensure_mapped_courier(self) -> Optional[MappedLayerCourier]:
|
||||
"""The courier, built on first use; None where it cannot help."""
|
||||
if self._mapped_courier is not None:
|
||||
@@ -2378,6 +2433,13 @@ class LayerwiseOffloadManager:
|
||||
for layer_idx in list(self._gpu_layers):
|
||||
self.sync_layer_to_cpu(layer_idx)
|
||||
|
||||
def _issue_host_copy(self, dest: torch.Tensor, src: torch.Tensor) -> None:
|
||||
"""Blocking copy into the manager CPU dest."""
|
||||
src = src.detach()
|
||||
if src.device.type == "cuda" and not src.is_contiguous():
|
||||
src = src.contiguous()
|
||||
dest.copy_(src)
|
||||
|
||||
@torch.compiler.disable
|
||||
def update_cpu_weights(
|
||||
self, weight_dict: Dict[str, torch.Tensor]
|
||||
@@ -2450,40 +2512,83 @@ class LayerwiseOffloadManager:
|
||||
f"expected={tuple(meta['shape'])}, "
|
||||
f"loaded={tuple(local_loaded_weight.shape)}"
|
||||
)
|
||||
if local_loaded_weight.dtype != meta["dtype"]:
|
||||
raise ValueError(
|
||||
f"Dtype mismatch for {name}: "
|
||||
f"buffer={meta['dtype']}, incoming={local_loaded_weight.dtype}. "
|
||||
"Casting would silently corrupt offloaded weights."
|
||||
)
|
||||
|
||||
dtype = meta["dtype"]
|
||||
src = local_loaded_weight.detach()
|
||||
if src.dtype != dtype:
|
||||
src = src.to(dtype=dtype)
|
||||
if not src.is_contiguous():
|
||||
src = src.contiguous()
|
||||
if meta.get("mapped", False):
|
||||
# The mapping is a read-only view of the checkpoint, so the new
|
||||
# values cannot be written into it. Own the storage from here
|
||||
# on; every reader of this store copies out of whatever tensor
|
||||
# it holds. This trades mapped bytes for anonymous ones on the
|
||||
# configuration that chose mapping because host memory was
|
||||
# short, so it costs the updated weight's bytes.
|
||||
self._mapped_cpu_weights[layer_idx][name] = (
|
||||
local_loaded_weight.detach().to(dtype=dtype).contiguous()
|
||||
# mmap is read-only; own a new CPU buffer and reuse it later.
|
||||
existing = self._mapped_cpu_weights[layer_idx].get(name)
|
||||
owned = (
|
||||
existing is not None
|
||||
and existing.device.type == "cpu"
|
||||
and tuple(existing.shape) == tuple(meta["shape"])
|
||||
and existing.dtype == dtype
|
||||
and not self._mapped_regions.holds(existing)
|
||||
)
|
||||
if src.device.type == "cpu":
|
||||
host_weight = existing if owned else src
|
||||
if owned:
|
||||
existing.copy_(src)
|
||||
elif owned:
|
||||
host_weight = existing
|
||||
self._issue_host_copy(host_weight, src)
|
||||
else:
|
||||
host_weight = torch.empty(meta["shape"], dtype=dtype)
|
||||
self._issue_host_copy(host_weight, src)
|
||||
self._mapped_cpu_weights[layer_idx][name] = host_weight
|
||||
self._mapped_store_dirty = True
|
||||
self._courier_inflight.discard(layer_idx)
|
||||
courier = self._mapped_courier
|
||||
if courier is not None:
|
||||
with courier._ready:
|
||||
courier._results.pop(layer_idx, None)
|
||||
elif meta.get("preserve_strides", False):
|
||||
self._strided_cpu_weights[layer_idx][name].copy_(
|
||||
local_loaded_weight.to(dtype=dtype)
|
||||
)
|
||||
self._issue_host_copy(self._strided_cpu_weights[layer_idx][name], src)
|
||||
else:
|
||||
offset = meta["offset"]
|
||||
numel = meta["numel"]
|
||||
cpu_buffer = self._consolidated_cpu_weights[layer_idx][dtype]
|
||||
cpu_buffer[offset : offset + numel].copy_(
|
||||
local_loaded_weight.to(dtype=dtype).flatten()
|
||||
self._issue_host_copy(
|
||||
cpu_buffer[offset : offset + numel], src.flatten()
|
||||
)
|
||||
|
||||
# If this layer is currently on GPU, update the live parameter.
|
||||
if layer_idx in self._gpu_layers:
|
||||
target = self.get_target_with_name(name)
|
||||
target_local = self._to_local_tensor(target)
|
||||
target_local.copy_(local_loaded_weight.to(dtype=target_local.dtype))
|
||||
with torch.no_grad():
|
||||
target_local.copy_(
|
||||
src.to(device=target_local.device, dtype=target_local.dtype)
|
||||
)
|
||||
|
||||
updated_names.add(name)
|
||||
|
||||
return updated_names
|
||||
|
||||
def has_cpu_weight(self, name: str) -> bool:
|
||||
"""True if ``name`` is stored in this manager's CPU buffers."""
|
||||
if not self.enabled:
|
||||
return False
|
||||
if self._synchronous_mps:
|
||||
layer_idx = self._match_layer_idx(name)
|
||||
if layer_idx is None:
|
||||
return False
|
||||
return name in self._mps_cpu_weights.get(layer_idx, {})
|
||||
layer_idx = self._match_layer_idx(name)
|
||||
if layer_idx is None:
|
||||
return False
|
||||
meta_layer = self._weight_metadata.get(layer_idx)
|
||||
return bool(meta_layer and name in meta_layer)
|
||||
|
||||
def iter_cpu_weights(self):
|
||||
"""Yield (name, tensor) pairs from consolidated CPU buffers.
|
||||
|
||||
@@ -2613,13 +2718,13 @@ class LayerwiseOffloadableModuleMixin:
|
||||
park_non_layer_weights_between_uses: bool = False
|
||||
|
||||
def _managed_layer_parameter_names(self) -> set:
|
||||
"""Parameter names some layerwise manager already streams."""
|
||||
return {
|
||||
name
|
||||
for manager in self.layerwise_offload_managers
|
||||
for names in manager._weight_metadata.values()
|
||||
for name in names
|
||||
}
|
||||
"""Managed parameter names, including ``*.base_layer.weight`` aliases."""
|
||||
names: set[str] = set()
|
||||
for manager in self.layerwise_offload_managers:
|
||||
for layer_names in manager._weight_metadata.values():
|
||||
for name in layer_names:
|
||||
names.update(offload_param_name_aliases(name))
|
||||
return names
|
||||
|
||||
def park_non_layer_weights(self) -> None:
|
||||
"""Move the parameters no manager streams back to the host.
|
||||
@@ -3013,6 +3118,147 @@ class LayerwiseOffloadableModuleMixin:
|
||||
manager.register_forward_hooks()
|
||||
|
||||
|
||||
def is_offload_placeholder(tensor: torch.Tensor) -> bool:
|
||||
return getattr(tensor, "numel", lambda: 0)() <= 1
|
||||
|
||||
|
||||
def offload_param_name_aliases(name: str) -> tuple[str, ...]:
|
||||
"""``name`` and its ``*.base_layer.*`` counterpart."""
|
||||
names = [name]
|
||||
if ".base_layer." in name:
|
||||
names.append(name.replace(".base_layer.", "."))
|
||||
else:
|
||||
head, sep, leaf = name.rpartition(".")
|
||||
if sep:
|
||||
names.append(f"{head}.base_layer.{leaf}")
|
||||
return tuple(dict.fromkeys(names))
|
||||
|
||||
|
||||
def refresh_layerwise_targets(module: torch.nn.Module) -> None:
|
||||
"""Repoint every manager at live Parameters after LoRA wrap."""
|
||||
if not is_layerwise_offloaded_module(module):
|
||||
return
|
||||
for manager in module.layerwise_offload_managers:
|
||||
manager.refresh_named_targets()
|
||||
|
||||
|
||||
def _refresh_packed_from_manager(layer: torch.nn.Module) -> None:
|
||||
"""Re-bind ``_packed_weight_cpu`` after the manager store is replaced."""
|
||||
root = getattr(layer, "_offload_root", None)
|
||||
prefix = getattr(layer, "_offload_param_prefix", None)
|
||||
if root is None or prefix is None:
|
||||
return
|
||||
name = f"{prefix}.weight"
|
||||
for manager in getattr(root, "layerwise_offload_managers", None) or []:
|
||||
getter = getattr(manager, "get_cpu_weight", None)
|
||||
if not callable(getter):
|
||||
continue
|
||||
tensor = getter(name)
|
||||
if tensor is not None:
|
||||
layer._packed_weight_cpu = tensor.detach()
|
||||
return
|
||||
|
||||
|
||||
def _release_stale_gpu_layer(layer: torch.nn.Module) -> None:
|
||||
"""Release a stale GPU copy so the next prefetch reloads from CPU."""
|
||||
root = getattr(layer, "_offload_root", None)
|
||||
prefix = getattr(layer, "_offload_param_prefix", None)
|
||||
if root is None or prefix is None:
|
||||
return
|
||||
name = f"{prefix}.weight"
|
||||
for manager in getattr(root, "layerwise_offload_managers", None) or []:
|
||||
match = getattr(manager, "_match_layer_idx", None)
|
||||
if not callable(match):
|
||||
continue
|
||||
layer_idx = match(name)
|
||||
if layer_idx is None:
|
||||
continue
|
||||
gpu_layers = getattr(manager, "_gpu_layers", None)
|
||||
if gpu_layers and layer_idx in gpu_layers:
|
||||
manager.release_layer(layer_idx, force=True)
|
||||
|
||||
|
||||
def copy_into_packed_view(dest: torch.Tensor | None, src: torch.Tensor) -> bool:
|
||||
"""Copy ``src`` into an offload CPU view. Refuse dtype/shape casts."""
|
||||
if dest is None or is_offload_placeholder(dest):
|
||||
return False
|
||||
src = src.detach()
|
||||
if dest.shape != src.shape or dest.dtype != src.dtype:
|
||||
return False
|
||||
dest.copy_(src)
|
||||
return True
|
||||
|
||||
|
||||
def write_offload_params(
|
||||
layer: torch.nn.Module, tensors: dict[str, torch.Tensor]
|
||||
) -> bool:
|
||||
"""Write weights into layerwise CPU buffers, not GPU placeholders."""
|
||||
root = getattr(layer, "_offload_root", None)
|
||||
prefix = getattr(layer, "_offload_param_prefix", None)
|
||||
if root is None or prefix is None:
|
||||
return False
|
||||
managers = getattr(root, "layerwise_offload_managers", None)
|
||||
if not managers:
|
||||
return False
|
||||
named = {f"{prefix}.{key}": value for key, value in tensors.items()}
|
||||
owned: dict[str, object] = {}
|
||||
for manager in managers:
|
||||
if not getattr(manager, "enabled", False):
|
||||
continue
|
||||
for name in named:
|
||||
if name in owned:
|
||||
continue
|
||||
has = getattr(manager, "has_cpu_weight", None)
|
||||
if callable(has) and has(name):
|
||||
owned[name] = manager
|
||||
if not owned:
|
||||
return False
|
||||
missing: list[str] = []
|
||||
for name, value in named.items():
|
||||
manager = owned.get(name)
|
||||
if manager is None:
|
||||
continue
|
||||
updated = manager.update_cpu_weights({name: value}) or set()
|
||||
if name not in updated:
|
||||
missing.append(name)
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
f"layerwise LoRA writeback missed {missing} (prefix={prefix})"
|
||||
)
|
||||
_refresh_packed_from_manager(layer)
|
||||
return True
|
||||
|
||||
|
||||
def finish_offload_writeback() -> None:
|
||||
"""No-op; writeback is already synchronous."""
|
||||
return
|
||||
|
||||
|
||||
def write_dense_weight(layer: torch.nn.Module, weight: torch.Tensor) -> None:
|
||||
"""Write into the manager/view store. Do not replace the Parameter object."""
|
||||
dest = layer.weight
|
||||
if hasattr(dest, "to_local"):
|
||||
dest = dest.to_local()
|
||||
if write_offload_params(layer, {"weight": weight}):
|
||||
_release_stale_gpu_layer(layer)
|
||||
return
|
||||
packed_view = getattr(layer, "_packed_weight_cpu", None)
|
||||
if copy_into_packed_view(packed_view, weight):
|
||||
_release_stale_gpu_layer(layer)
|
||||
return
|
||||
if is_offload_placeholder(dest):
|
||||
raise RuntimeError(
|
||||
"layerwise LoRA writeback has no manager CPU view for a (1,) "
|
||||
"placeholder; convert_to_lora_layers must bind the offload root "
|
||||
"and _packed_weight_cpu first."
|
||||
)
|
||||
packed = weight.detach().to(device=dest.device, dtype=dest.dtype)
|
||||
if dest.data.is_inference() or dest.dtype != packed.dtype:
|
||||
dest.data = packed
|
||||
else:
|
||||
dest.data.copy_(packed)
|
||||
|
||||
|
||||
def iter_materialized_weights(module: torch.nn.Module):
|
||||
"""Yield (name, tensor) pairs with materialized weights, even under offload.
|
||||
|
||||
@@ -3033,13 +3279,16 @@ def iter_materialized_weights(module: torch.nn.Module):
|
||||
offloaded_names: set[str] = set()
|
||||
for manager in offload_managers:
|
||||
for name, tensor in manager.iter_cpu_weights():
|
||||
offloaded_names.add(name)
|
||||
offloaded_names.update(offload_param_name_aliases(name))
|
||||
yield name, tensor
|
||||
|
||||
# Yield non-offloaded parameters (e.g. final norms, embeddings).
|
||||
# Skip LoRA-wrapped aliases; those live tensors are often (1,) placeholders.
|
||||
for name, param in module.named_parameters():
|
||||
if name not in offloaded_names:
|
||||
yield name, param
|
||||
if name in offloaded_names:
|
||||
continue
|
||||
if is_offload_placeholder(param):
|
||||
continue
|
||||
yield name, param
|
||||
|
||||
|
||||
def is_layerwise_offloaded_module(module: torch.nn.Module) -> bool:
|
||||
|
||||
@@ -22,6 +22,8 @@ from sglang.multimodal_gen.runtime.layers.lora.linear import (
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
is_layerwise_offloaded_module,
|
||||
iter_materialized_weights,
|
||||
refresh_layerwise_targets,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.weight_snapshot import (
|
||||
restore_weight_snapshot,
|
||||
@@ -252,6 +254,28 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
else:
|
||||
return [], f"Invalid target: {target}. Valid targets: {self.VALID_TARGETS}"
|
||||
|
||||
def _has_layerwise_offload(
|
||||
self,
|
||||
target_modules: list[tuple[str, dict[str, BaseLayerWithLoRA]]] | None = None,
|
||||
) -> bool:
|
||||
names: list[str]
|
||||
if target_modules is not None:
|
||||
names = [module_name for module_name, _ in target_modules]
|
||||
else:
|
||||
names = ["transformer", "transformer_2", "critic"]
|
||||
return any(
|
||||
is_layerwise_offloaded_module(self.modules.get(name)) for name in names
|
||||
)
|
||||
|
||||
def _weight_update_context(
|
||||
self,
|
||||
target_modules: list[tuple[str, dict[str, BaseLayerWithLoRA]]] | None = None,
|
||||
):
|
||||
"""Do not load_all when layerwise is on."""
|
||||
if self._has_layerwise_offload(target_modules):
|
||||
return nullcontext()
|
||||
return self._temporarily_disable_offload(target_modules=target_modules)
|
||||
|
||||
@contextmanager
|
||||
def _temporarily_disable_offload(
|
||||
self,
|
||||
@@ -356,6 +380,11 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
The number of layers converted.
|
||||
"""
|
||||
converted_count = 0
|
||||
materialized = (
|
||||
dict(iter_materialized_weights(module))
|
||||
if is_layerwise_offloaded_module(module)
|
||||
else {}
|
||||
)
|
||||
for name, layer in module.named_modules():
|
||||
if not self.is_target_layer(name):
|
||||
continue
|
||||
@@ -374,12 +403,62 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
snapshot_base=snapshot_base,
|
||||
)
|
||||
if lora_layer is not None:
|
||||
self._bind_lora_offload_view(
|
||||
module,
|
||||
name,
|
||||
lora_layer,
|
||||
materialized,
|
||||
snapshot_base=snapshot_base,
|
||||
)
|
||||
target_lora_layers[name] = lora_layer
|
||||
replace_submodule(self.modules[module_name], name, lora_layer)
|
||||
converted_count += 1
|
||||
|
||||
if converted_count and is_layerwise_offloaded_module(module):
|
||||
refresh_layerwise_targets(module)
|
||||
materialized = dict(iter_materialized_weights(module))
|
||||
bound = 0
|
||||
for name, lora_layer in target_lora_layers.items():
|
||||
self._bind_lora_offload_view(
|
||||
module,
|
||||
name,
|
||||
lora_layer,
|
||||
materialized,
|
||||
snapshot_base=snapshot_base,
|
||||
)
|
||||
packed = getattr(lora_layer.base_layer, "_packed_weight_cpu", None)
|
||||
if packed is not None and packed.numel() > 1:
|
||||
bound += 1
|
||||
logger.info(
|
||||
"Layerwise LoRA bind: %d/%d layers have a CPU view after wrap",
|
||||
bound,
|
||||
converted_count,
|
||||
)
|
||||
|
||||
return converted_count
|
||||
|
||||
def _bind_lora_offload_view(
|
||||
self,
|
||||
module: torch.nn.Module,
|
||||
name: str,
|
||||
lora_layer: BaseLayerWithLoRA,
|
||||
materialized: dict[str, torch.Tensor],
|
||||
snapshot_base: bool = True,
|
||||
) -> None:
|
||||
"""Bind LoRA to the manager CPU view, not the (1,) GPU placeholder."""
|
||||
if not materialized:
|
||||
return
|
||||
weight = materialized.get(f"{name}.weight")
|
||||
if weight is None:
|
||||
return
|
||||
data = weight.detach()
|
||||
lora_layer.cpu_weight = data.clone() if snapshot_base else data
|
||||
lora_layer._base_is_view = not snapshot_base
|
||||
base = lora_layer.base_layer
|
||||
base._offload_root = module
|
||||
base._offload_param_prefix = name
|
||||
base._packed_weight_cpu = data
|
||||
|
||||
def _reject_lora_on_packed_weights(self) -> None:
|
||||
"""Fail before any layer is replaced if a target has no plain weight.
|
||||
|
||||
@@ -997,14 +1076,9 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
# unsupported-LoRA error. Offloaded placeholders still carry the name.
|
||||
self._reject_lora_on_packed_weights()
|
||||
|
||||
# Disable layerwise offload before convert_to_lora_layers to ensure weights are accessible
|
||||
# This is critical because convert_to_lora_layers needs to save cpu_weight from actual weights,
|
||||
# not from offloaded placeholder tensors
|
||||
# Layerwise keeps manager views; do not load_all.
|
||||
if not self.lora_initialized:
|
||||
with self._temporarily_disable_offload(
|
||||
target="all", use_module_names_only=True
|
||||
):
|
||||
self.convert_to_lora_layers()
|
||||
self.convert_to_lora_layers(snapshot_base=not self._has_layerwise_offload())
|
||||
|
||||
# Check adapter presence and load missing adapters
|
||||
adapter_updated = False
|
||||
@@ -1095,9 +1169,7 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
if self._needs_lora_weight_update_context(
|
||||
target_modules, merge_weights_by_module
|
||||
):
|
||||
weight_update_context = self._temporarily_disable_offload(
|
||||
target_modules=target_modules
|
||||
)
|
||||
weight_update_context = self._weight_update_context(target_modules)
|
||||
else:
|
||||
weight_update_context = nullcontext()
|
||||
|
||||
@@ -1149,6 +1221,12 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
tgt_strengths.copy(),
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
finish_offload_writeback,
|
||||
)
|
||||
|
||||
finish_offload_writeback()
|
||||
|
||||
logger.info(
|
||||
"Rank %d: LoRA adapter(s) %s applied to %d layers (targets: %s, strengths: %s, merge_mode=%s)",
|
||||
rank,
|
||||
@@ -1238,9 +1316,7 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
):
|
||||
modules_requiring_unmerge.append((module_name, lora_layers_dict))
|
||||
|
||||
offload_context = self._temporarily_disable_offload(
|
||||
target_modules=modules_requiring_unmerge
|
||||
)
|
||||
offload_context = self._weight_update_context(modules_requiring_unmerge)
|
||||
with offload_context:
|
||||
for module_name, lora_layers_dict in target_modules:
|
||||
for layer in lora_layers_dict.values():
|
||||
@@ -1270,8 +1346,7 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
if not target_modules:
|
||||
return
|
||||
|
||||
# Disable layerwise offload if enabled: load all layers to GPU
|
||||
with self._temporarily_disable_offload(target_modules=target_modules):
|
||||
with self._weight_update_context(target_modules):
|
||||
for module_name, lora_layers_dict in target_modules:
|
||||
if not self._should_merge_lora_for_layers(
|
||||
module_name, lora_layers_dict, self.server_args.lora_merge_mode
|
||||
@@ -1320,6 +1395,11 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
logger.info(
|
||||
"LoRA weights merged for %s (strength: %s)", module_name, strength
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
finish_offload_writeback,
|
||||
)
|
||||
|
||||
finish_offload_writeback()
|
||||
|
||||
def unmerge_lora_weights(self, target: str = "all") -> None:
|
||||
"""
|
||||
@@ -1354,7 +1434,7 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
"LoRA weights are not merged for %s, skipping", module_name
|
||||
)
|
||||
continue
|
||||
with self._temporarily_disable_offload(target_modules=target_modules):
|
||||
with self._weight_update_context(target_modules):
|
||||
for name, layer in lora_layers_dict.items():
|
||||
# Check layer-level state to avoid raising exception
|
||||
if hasattr(layer, "merged") and not layer.merged:
|
||||
@@ -1378,6 +1458,11 @@ class LoRAPipeline(ComposedPipelineBase):
|
||||
self.cur_adapter_strength.pop(module_name, None)
|
||||
self.cur_adapter_config.pop(module_name, None)
|
||||
logger.info("LoRA weights unmerged for %s", module_name)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
finish_offload_writeback,
|
||||
)
|
||||
|
||||
finish_offload_writeback()
|
||||
|
||||
def get_lora_status(self) -> dict[str, Any]:
|
||||
"""
|
||||
|
||||
@@ -2820,3 +2820,103 @@ def test_configure_logs_a_permanent_lifetime(monkeypatch):
|
||||
)
|
||||
assert any("placed 3 permanent resident layers" in line for line in logs)
|
||||
assert logs[-1].endswith("resident=3/8 (permanent), policy=leading")
|
||||
|
||||
|
||||
class _LoRAWrap(torch.nn.Module):
|
||||
"""Mimic BaseLayerWithLoRA: live Parameter name becomes *.base_layer.weight."""
|
||||
|
||||
def __init__(self, base: torch.nn.Module) -> None:
|
||||
super().__init__()
|
||||
self.base_layer = base
|
||||
|
||||
@property
|
||||
def weight(self):
|
||||
return self.base_layer.weight
|
||||
|
||||
|
||||
def test_offload_param_name_aliases_cover_lora_wrap():
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
offload_param_name_aliases,
|
||||
)
|
||||
|
||||
aliases = offload_param_name_aliases("blocks.0.weight")
|
||||
assert "blocks.0.weight" in aliases
|
||||
assert "blocks.0.base_layer.weight" in aliases
|
||||
assert "blocks.0.weight" in offload_param_name_aliases("blocks.0.base_layer.weight")
|
||||
|
||||
|
||||
def test_parking_does_not_steal_lora_wrapped_layer_weights(monkeypatch):
|
||||
"""After wrap, live names are *.base_layer.weight; park must still skip them."""
|
||||
_patch_fake_device(monkeypatch)
|
||||
model = _MixinModel()
|
||||
model.configure_layerwise_offload(_server_args(performance_mode="memory"))
|
||||
_headroom(monkeypatch, 0)
|
||||
|
||||
for i, block in enumerate(list(model.blocks)):
|
||||
model.blocks[i] = _LoRAWrap(block)
|
||||
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
refresh_layerwise_targets,
|
||||
)
|
||||
|
||||
refresh_layerwise_targets(model)
|
||||
live_names = {n for n, _ in model.named_parameters() if n.endswith(".weight")}
|
||||
assert any("base_layer.weight" in n for n in live_names)
|
||||
|
||||
managed = model._managed_layer_parameter_names()
|
||||
assert live_names <= managed
|
||||
|
||||
model.park_non_layer_weights()
|
||||
parked = set(model._parked_non_layer_weights)
|
||||
assert not (parked & live_names), parked
|
||||
|
||||
|
||||
def test_iter_materialized_weights_skips_wrapped_placeholders(monkeypatch):
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
iter_materialized_weights,
|
||||
refresh_layerwise_targets,
|
||||
)
|
||||
|
||||
model = _configure_mixin_model(monkeypatch)
|
||||
before = dict(iter_materialized_weights(model))
|
||||
assert "blocks.0.weight" in before
|
||||
assert before["blocks.0.weight"].numel() > 1
|
||||
|
||||
for i, block in enumerate(list(model.blocks)):
|
||||
model.blocks[i] = _LoRAWrap(block)
|
||||
refresh_layerwise_targets(model)
|
||||
|
||||
after = dict(iter_materialized_weights(model))
|
||||
assert "blocks.0.weight" in after
|
||||
assert after["blocks.0.weight"].numel() > 1
|
||||
assert "blocks.0.base_layer.weight" not in after
|
||||
|
||||
|
||||
def test_write_dense_weight_keeps_manager_parameter(monkeypatch):
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
write_dense_weight,
|
||||
)
|
||||
|
||||
model = _configure_mixin_model(monkeypatch)
|
||||
layer = model.blocks[0]
|
||||
manager = model.layerwise_offload_managers[0]
|
||||
original = manager.get_target_with_name("blocks.0.weight")
|
||||
layer._offload_root = model
|
||||
layer._offload_param_prefix = "blocks.0"
|
||||
|
||||
merged = torch.arange(9, dtype=torch.float32).reshape(3, 3) + 7
|
||||
with torch.no_grad():
|
||||
write_dense_weight(layer, merged)
|
||||
|
||||
assert manager.get_target_with_name("blocks.0.weight") is original
|
||||
got = manager.get_cpu_weight("blocks.0.weight")
|
||||
assert got is not None
|
||||
torch.testing.assert_close(got, merged)
|
||||
|
||||
|
||||
def test_finish_offload_writeback_is_noop_when_idle():
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
finish_offload_writeback,
|
||||
)
|
||||
|
||||
finish_offload_writeback()
|
||||
|
||||
@@ -116,6 +116,7 @@ def test_zero_copy_snapshot_is_limited_to_cpu_backed_layers():
|
||||
assert not _use_owned_base_snapshot(False, "cpu")
|
||||
assert not _use_owned_base_snapshot(False, "meta")
|
||||
assert _use_owned_base_snapshot(False, "cuda")
|
||||
assert not _use_owned_base_snapshot(False, "cuda", numel=1)
|
||||
assert _use_owned_base_snapshot(True, "cpu")
|
||||
|
||||
cpu_layer = wrap_with_lora_layer(
|
||||
@@ -323,3 +324,21 @@ def test_lora_exact_file_url_needs_no_weight_name(tmp_path):
|
||||
"*.json",
|
||||
"adapter.safetensors",
|
||||
]
|
||||
|
||||
|
||||
def test_view_merge_unmerges_by_inverse_without_owned_clone():
|
||||
torch.manual_seed(0)
|
||||
base = torch.nn.Linear(4, 4, bias=False)
|
||||
original = base.weight.detach().clone()
|
||||
layer = wrap_with_lora_layer(base, lora_rank=2, lora_alpha=2, snapshot_base=False)
|
||||
assert layer is not None
|
||||
assert layer._base_is_view
|
||||
A = torch.randn(2, 4)
|
||||
B = torch.randn(4, 2)
|
||||
layer.set_lora_weights(A, B, strength=0.5, clear_existing=True, merge_weights=True)
|
||||
assert layer.merged
|
||||
assert layer._base_is_view
|
||||
torch.testing.assert_close(layer.base_layer.weight, original + 0.5 * (B @ A))
|
||||
layer.unmerge_lora_weights()
|
||||
torch.testing.assert_close(layer.base_layer.weight, original)
|
||||
assert not layer.merged
|
||||
|
||||
Reference in New Issue
Block a user