diff --git a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py index 1e135b3e82..13346c80e5 100644 --- a/python/sglang/multimodal_gen/runtime/layers/lora/linear.py +++ b/python/sglang/multimodal_gen/runtime/layers/lora/linear.py @@ -32,6 +32,9 @@ from sglang.multimodal_gen.runtime.layers.linear import ( from sglang.multimodal_gen.runtime.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) +from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + write_dense_weight, +) from sglang.multimodal_gen.runtime.utils.precision import get_mixed_precision_state torch._dynamo.config.recompile_limit = 64 @@ -127,8 +130,17 @@ class BaseLayerWithLoRA(nn.Module): return isinstance(self.base_layer.quant_method, UnquantizedLinearMethod) return True - @torch.compile() def forward(self, x: torch.Tensor) -> torch.Tensor: + # Layerwise rebinds Parameter.data; do not compile the merged path. + if self.disable_lora or (self.merged and not self.has_lora_output_offset): + return self.base_layer(x) + if self.merged: + out, output_bias = self.base_layer(x) + return self._add_lora_output_offset(out), output_bias + return self._forward_with_delta(x) + + @torch.compile() + def _forward_with_delta(self, x: torch.Tensor) -> torch.Tensor: lora_A = self.lora_A lora_B = self.lora_B if isinstance(self.lora_B, DTensor): @@ -136,25 +148,22 @@ class BaseLayerWithLoRA(nn.Module): lora_A = self.lora_A.to_local() # TODO: Support multiple LoRA adapters when use not merged mode - if not self.merged and not self.disable_lora: - lora_dtype = lora_A.dtype - x_lora = x.to(dtype=lora_dtype) - lora_A_sliced = self.slice_lora_a_weights( - lora_A.to(device=x.device, non_blocking=True) - ) - lora_B_sliced = self.slice_lora_b_weights( - lora_B.to(device=x.device, non_blocking=True) - ) - delta = _compute_lora_delta(x_lora, lora_A_sliced, lora_B_sliced) - if self.lora_alpha != self.lora_rank: - delta = delta * ( - self.lora_alpha / self.lora_rank # type: ignore - ) # type: ignore - delta = delta * self.strength - out, output_bias = self.base_layer(x) - out = out + delta.to(dtype=out.dtype) - else: - out, output_bias = self.base_layer(x) + lora_dtype = lora_A.dtype + x_lora = x.to(dtype=lora_dtype) + lora_A_sliced = self.slice_lora_a_weights( + lora_A.to(device=x.device, non_blocking=True) + ) + lora_B_sliced = self.slice_lora_b_weights( + lora_B.to(device=x.device, non_blocking=True) + ) + delta = _compute_lora_delta(x_lora, lora_A_sliced, lora_B_sliced) + if self.lora_alpha != self.lora_rank: + delta = delta * ( + self.lora_alpha / self.lora_rank # type: ignore + ) # type: ignore + delta = delta * self.strength + out, output_bias = self.base_layer(x) + out = out + delta.to(dtype=out.dtype) return self._add_lora_output_offset(out), output_bias def slice_lora_a_weights(self, A: torch.Tensor) -> torch.Tensor: @@ -279,6 +288,43 @@ class BaseLayerWithLoRA(nn.Module): elif self.merged: self.unmerge_lora_weights() + def _active_lora_list(self) -> list[LoRAWeightEntry]: + if self.lora_weights_list: + return list(self.lora_weights_list) + if self.lora_A is not None and self.lora_B is not None: + return [ + ( + self.lora_A, + self.lora_B, + self.lora_path, + self.strength, + self.lora_rank, + self.lora_alpha, + self.lora_output_offset, + ) + ] + return [] + + def _materialized_weight_src(self) -> tuple[torch.Tensor, torch.device]: + """Layerwise CPU view, or the live GPU weight if it is not a (1,) placeholder.""" + packed = getattr(self.base_layer, "_packed_weight_cpu", None) + if packed is None or packed.numel() <= 1: + from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + _refresh_packed_from_manager, + ) + + _refresh_packed_from_manager(self.base_layer) + packed = getattr(self.base_layer, "_packed_weight_cpu", None) + weight = self.base_layer.weight.data + if weight.numel() <= 1: + if packed is None or packed.numel() <= 1: + raise RuntimeError( + "layerwise LoRA saw a (1,) placeholder with no CPU view; " + "convert_to_lora_layers must bind _packed_weight_cpu first." + ) + return packed, torch.device("cpu") + return weight, weight.device + def _ensure_base_snapshot_owned(self) -> None: """An in-place merge is about to write the base storage; if the snapshot is a zero-copy view into it, materialize the clone now.""" @@ -427,25 +473,13 @@ class BaseLayerWithLoRA(nn.Module): if self.disable_lora: return - self._ensure_base_snapshot_owned() + # Zero-copy views unmerge by subtracting B@A; do not clone the base. + if not self._base_is_view: + self._ensure_base_snapshot_owned() if self.merged: self.unmerge_lora_weights() - # Use lora_weights_list if available, otherwise fall back to single LoRA for backward compatibility - lora_list = self.lora_weights_list if self.lora_weights_list else [] - if not lora_list and self.lora_A is not None and self.lora_B is not None: - lora_list = [ - ( - self.lora_A, - self.lora_B, - self.lora_path, - self.strength, - self.lora_rank, - self.lora_alpha, - self.lora_output_offset, - ) - ] - + lora_list = self._active_lora_list() if not lora_list: raise ValueError("LoRA weights not set. Please set them first.") if isinstance(self.base_layer.weight, DTensor) and any( @@ -509,27 +543,35 @@ class BaseLayerWithLoRA(nn.Module): offload_policy=offload_policy, ) else: - current_device = self.base_layer.weight.data.device - data = self.base_layer.weight.data.to(get_local_torch_device()) - data = self._as_mutable_tensor(data) - target_dtype = data.dtype - if ( - merge_in_fp32 - and data.is_floating_point() - and data.dtype != torch.float32 - ): - data = data.to(torch.float32) - - self._merge_lora_into_data(data, lora_list) - - self.base_layer.weight.data = self._as_mutable_tensor( - data.to(current_device, dtype=target_dtype, non_blocking=True) - ) + src, _ = self._materialized_weight_src() + data = src.to(get_local_torch_device()) + self._merge_from_device_data(data, lora_list, merge_in_fp32) + return self.merged = True + def _merge_from_device_data( + self, + data: torch.Tensor, + lora_list: list[LoRAWeightEntry] | None = None, + merge_in_fp32: bool | None = None, + ) -> None: + """Merge on GPU and write back to the layer store.""" + lora_list = lora_list if lora_list is not None else self._active_lora_list() + if not lora_list: + raise ValueError("LoRA weights not set. Please set them first.") + if merge_in_fp32 is None: + merge_in_fp32 = self._should_merge_in_fp32(lora_list) + data = self._as_mutable_tensor(data) + target_dtype = data.dtype + if merge_in_fp32 and data.is_floating_point() and data.dtype != torch.float32: + data = data.to(torch.float32) + self._merge_lora_into_data(data, lora_list) + merged = self._as_mutable_tensor(data.to(dtype=target_dtype)) + write_dense_weight(self.base_layer, merged) + self.merged = True + @torch.no_grad() - # @torch.compile(dynamic=True) def unmerge_lora_weights(self) -> None: if self.disable_lora: return @@ -548,6 +590,8 @@ class BaseLayerWithLoRA(nn.Module): ) self.base_layer.weight = nn.Parameter(new_weight_data) del old_weight + elif self._base_is_view: + self._unmerge_by_inverse() else: current_device = self.base_layer.weight.data.device cpu_weight_on_device = self.cpu_weight.to(current_device, non_blocking=True) @@ -565,6 +609,48 @@ class BaseLayerWithLoRA(nn.Module): self.merged = False + @torch.no_grad() + def _unmerge_by_inverse(self) -> None: + lora_list = self._active_lora_list() + if not lora_list: + raise ValueError( + "LoRA weights not set; cannot unmerge a zero-copy view by inverse." + ) + src, current_device = self._materialized_weight_src() + data = self._as_mutable_tensor(src.to(get_local_torch_device())) + target_dtype = data.dtype + if ( + self._should_merge_in_fp32(lora_list) + and data.is_floating_point() + and data.dtype != torch.float32 + ): + data = data.to(torch.float32) + inverted = [ + ( + lora_A, + lora_B, + lora_path, + -lora_strength, + lora_rank, + lora_alpha, + output_offset, + ) + for ( + lora_A, + lora_B, + lora_path, + lora_strength, + lora_rank, + lora_alpha, + output_offset, + ) in reversed(lora_list) + ] + self._merge_lora_into_data(data, inverted) + write_dense_weight( + self.base_layer, + self._as_mutable_tensor(data.to(dtype=target_dtype)), + ) + @torch.no_grad() def commit_merged_as_base(self) -> None: """Promote the currently merged weights to the permanent base. @@ -882,7 +968,11 @@ class LinearWithLoRA(BaseLayerWithLoRA): return self._add_lora_output_offset(out) -def _use_owned_base_snapshot(snapshot_base: bool, device_type: str) -> bool: +def _use_owned_base_snapshot( + snapshot_base: bool, device_type: str, numel: int | None = None +) -> bool: + if numel is not None and numel <= 1: + return snapshot_base return snapshot_base or device_type not in ("cpu", "meta") @@ -910,7 +1000,7 @@ def wrap_with_lora_layer( for src_layer_type, lora_layer_type in supported_layer_types.items(): if isinstance(layer, src_layer_type): # type: ignore[arg-type] effective_snapshot_base = _use_owned_base_snapshot( - snapshot_base, layer.weight.device.type + snapshot_base, layer.weight.device.type, layer.weight.numel() ) ret = lora_layer_type( layer, diff --git a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py index 755fe28a50..9d024cf216 100644 --- a/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py +++ b/python/sglang/multimodal_gen/runtime/managers/memory_managers/layerwise_offload.py @@ -1134,6 +1134,7 @@ class LayerwiseOffloadManager: self._gpu_layers: Set[int] = set() # mapped layers handed to the courier and not yet collected self._mapped_courier: Optional[MappedLayerCourier] = None + self._mapped_store_dirty = False self._courier_inflight: Set[int] = set() # layer_idx -> torch.get_device_module().Event for fine-grained sync, to make sure the weight is resident in pre-hook self._prefetch_events: Dict[int, torch.get_device_module().Event] = {} @@ -1923,10 +1924,52 @@ class LayerwiseOffloadManager: def get_target_with_name(self, name: str) -> torch.Tensor: """get the target model weight/buffer to be replaced""" if name in self._named_parameters: - target = self._named_parameters[name] - else: - target = self._named_buffers[name] - return target + return self._named_parameters[name] + if name in self._named_buffers: + return self._named_buffers[name] + for alias in offload_param_name_aliases(name): + if alias in self._named_parameters: + return self._named_parameters[alias] + if alias in self._named_buffers: + return self._named_buffers[alias] + raise KeyError(name) + + def refresh_named_targets(self) -> None: + """Repoint manager targets after LoRA wrap (``*.base_layer.weight``).""" + live = dict(self.model.named_parameters()) + live_buffers = dict(self.model.named_buffers()) + for name in list(self._named_parameters): + for alias in offload_param_name_aliases(name): + if alias in live: + self._named_parameters[name] = live[alias] + break + for name in list(self._named_buffers): + if name in live_buffers: + self._named_buffers[name] = live_buffers[name] + + def get_cpu_weight(self, name: str) -> torch.Tensor | None: + """Current manager CPU tensor for ``name``, or None.""" + if not self.has_cpu_weight(name): + return None + layer_idx = self._match_layer_idx(name) + if layer_idx is None: + return None + if self._synchronous_mps: + return self._mps_cpu_weights.get(layer_idx, {}).get(name) + meta = self._weight_metadata.get(layer_idx, {}).get(name) + if meta is None: + return None + if meta.get("mapped", False): + return self._mapped_cpu_weights[layer_idx][name] + if meta.get("preserve_strides", False): + return self._strided_cpu_weights[layer_idx][name] + dtype = meta["dtype"] + offset = meta["offset"] + numel = meta["numel"] + shape = meta["shape"] + return self._consolidated_cpu_weights[layer_idx][dtype][ + offset : offset + numel + ].reshape(shape) @torch.compiler.disable def prefetch_layer(self, layer_idx: int, non_blocking: bool = True) -> None: @@ -1937,6 +1980,9 @@ class LayerwiseOffloadManager: return if layer_idx < 0 or layer_idx >= self.num_layers: return + finish_offload_writeback() + if self._mapped_store_dirty: + self._reset_mapped_courier() if layer_idx in self._gpu_layers: return if layer_idx in self._courier_inflight: @@ -2087,6 +2133,15 @@ class LayerwiseOffloadManager: courier = self._ensure_mapped_courier() return courier is not None and courier.direct_read + def _reset_mapped_courier(self) -> None: + """Drop courier state after LoRA rewrites a mapped CPU store.""" + courier = self._mapped_courier + if courier is not None: + courier.close() + self._mapped_courier = None + self._courier_inflight.clear() + self._mapped_store_dirty = False + def _ensure_mapped_courier(self) -> Optional[MappedLayerCourier]: """The courier, built on first use; None where it cannot help.""" if self._mapped_courier is not None: @@ -2378,6 +2433,13 @@ class LayerwiseOffloadManager: for layer_idx in list(self._gpu_layers): self.sync_layer_to_cpu(layer_idx) + def _issue_host_copy(self, dest: torch.Tensor, src: torch.Tensor) -> None: + """Blocking copy into the manager CPU dest.""" + src = src.detach() + if src.device.type == "cuda" and not src.is_contiguous(): + src = src.contiguous() + dest.copy_(src) + @torch.compiler.disable def update_cpu_weights( self, weight_dict: Dict[str, torch.Tensor] @@ -2450,40 +2512,83 @@ class LayerwiseOffloadManager: f"expected={tuple(meta['shape'])}, " f"loaded={tuple(local_loaded_weight.shape)}" ) + if local_loaded_weight.dtype != meta["dtype"]: + raise ValueError( + f"Dtype mismatch for {name}: " + f"buffer={meta['dtype']}, incoming={local_loaded_weight.dtype}. " + "Casting would silently corrupt offloaded weights." + ) dtype = meta["dtype"] + src = local_loaded_weight.detach() + if src.dtype != dtype: + src = src.to(dtype=dtype) + if not src.is_contiguous(): + src = src.contiguous() if meta.get("mapped", False): - # The mapping is a read-only view of the checkpoint, so the new - # values cannot be written into it. Own the storage from here - # on; every reader of this store copies out of whatever tensor - # it holds. This trades mapped bytes for anonymous ones on the - # configuration that chose mapping because host memory was - # short, so it costs the updated weight's bytes. - self._mapped_cpu_weights[layer_idx][name] = ( - local_loaded_weight.detach().to(dtype=dtype).contiguous() + # mmap is read-only; own a new CPU buffer and reuse it later. + existing = self._mapped_cpu_weights[layer_idx].get(name) + owned = ( + existing is not None + and existing.device.type == "cpu" + and tuple(existing.shape) == tuple(meta["shape"]) + and existing.dtype == dtype + and not self._mapped_regions.holds(existing) ) + if src.device.type == "cpu": + host_weight = existing if owned else src + if owned: + existing.copy_(src) + elif owned: + host_weight = existing + self._issue_host_copy(host_weight, src) + else: + host_weight = torch.empty(meta["shape"], dtype=dtype) + self._issue_host_copy(host_weight, src) + self._mapped_cpu_weights[layer_idx][name] = host_weight + self._mapped_store_dirty = True + self._courier_inflight.discard(layer_idx) + courier = self._mapped_courier + if courier is not None: + with courier._ready: + courier._results.pop(layer_idx, None) elif meta.get("preserve_strides", False): - self._strided_cpu_weights[layer_idx][name].copy_( - local_loaded_weight.to(dtype=dtype) - ) + self._issue_host_copy(self._strided_cpu_weights[layer_idx][name], src) else: offset = meta["offset"] numel = meta["numel"] cpu_buffer = self._consolidated_cpu_weights[layer_idx][dtype] - cpu_buffer[offset : offset + numel].copy_( - local_loaded_weight.to(dtype=dtype).flatten() + self._issue_host_copy( + cpu_buffer[offset : offset + numel], src.flatten() ) - # If this layer is currently on GPU, update the live parameter. if layer_idx in self._gpu_layers: target = self.get_target_with_name(name) target_local = self._to_local_tensor(target) - target_local.copy_(local_loaded_weight.to(dtype=target_local.dtype)) + with torch.no_grad(): + target_local.copy_( + src.to(device=target_local.device, dtype=target_local.dtype) + ) updated_names.add(name) return updated_names + def has_cpu_weight(self, name: str) -> bool: + """True if ``name`` is stored in this manager's CPU buffers.""" + if not self.enabled: + return False + if self._synchronous_mps: + layer_idx = self._match_layer_idx(name) + if layer_idx is None: + return False + return name in self._mps_cpu_weights.get(layer_idx, {}) + layer_idx = self._match_layer_idx(name) + if layer_idx is None: + return False + meta_layer = self._weight_metadata.get(layer_idx) + return bool(meta_layer and name in meta_layer) + def iter_cpu_weights(self): """Yield (name, tensor) pairs from consolidated CPU buffers. @@ -2613,13 +2718,13 @@ class LayerwiseOffloadableModuleMixin: park_non_layer_weights_between_uses: bool = False def _managed_layer_parameter_names(self) -> set: - """Parameter names some layerwise manager already streams.""" - return { - name - for manager in self.layerwise_offload_managers - for names in manager._weight_metadata.values() - for name in names - } + """Managed parameter names, including ``*.base_layer.weight`` aliases.""" + names: set[str] = set() + for manager in self.layerwise_offload_managers: + for layer_names in manager._weight_metadata.values(): + for name in layer_names: + names.update(offload_param_name_aliases(name)) + return names def park_non_layer_weights(self) -> None: """Move the parameters no manager streams back to the host. @@ -3013,6 +3118,147 @@ class LayerwiseOffloadableModuleMixin: manager.register_forward_hooks() +def is_offload_placeholder(tensor: torch.Tensor) -> bool: + return getattr(tensor, "numel", lambda: 0)() <= 1 + + +def offload_param_name_aliases(name: str) -> tuple[str, ...]: + """``name`` and its ``*.base_layer.*`` counterpart.""" + names = [name] + if ".base_layer." in name: + names.append(name.replace(".base_layer.", ".")) + else: + head, sep, leaf = name.rpartition(".") + if sep: + names.append(f"{head}.base_layer.{leaf}") + return tuple(dict.fromkeys(names)) + + +def refresh_layerwise_targets(module: torch.nn.Module) -> None: + """Repoint every manager at live Parameters after LoRA wrap.""" + if not is_layerwise_offloaded_module(module): + return + for manager in module.layerwise_offload_managers: + manager.refresh_named_targets() + + +def _refresh_packed_from_manager(layer: torch.nn.Module) -> None: + """Re-bind ``_packed_weight_cpu`` after the manager store is replaced.""" + root = getattr(layer, "_offload_root", None) + prefix = getattr(layer, "_offload_param_prefix", None) + if root is None or prefix is None: + return + name = f"{prefix}.weight" + for manager in getattr(root, "layerwise_offload_managers", None) or []: + getter = getattr(manager, "get_cpu_weight", None) + if not callable(getter): + continue + tensor = getter(name) + if tensor is not None: + layer._packed_weight_cpu = tensor.detach() + return + + +def _release_stale_gpu_layer(layer: torch.nn.Module) -> None: + """Release a stale GPU copy so the next prefetch reloads from CPU.""" + root = getattr(layer, "_offload_root", None) + prefix = getattr(layer, "_offload_param_prefix", None) + if root is None or prefix is None: + return + name = f"{prefix}.weight" + for manager in getattr(root, "layerwise_offload_managers", None) or []: + match = getattr(manager, "_match_layer_idx", None) + if not callable(match): + continue + layer_idx = match(name) + if layer_idx is None: + continue + gpu_layers = getattr(manager, "_gpu_layers", None) + if gpu_layers and layer_idx in gpu_layers: + manager.release_layer(layer_idx, force=True) + + +def copy_into_packed_view(dest: torch.Tensor | None, src: torch.Tensor) -> bool: + """Copy ``src`` into an offload CPU view. Refuse dtype/shape casts.""" + if dest is None or is_offload_placeholder(dest): + return False + src = src.detach() + if dest.shape != src.shape or dest.dtype != src.dtype: + return False + dest.copy_(src) + return True + + +def write_offload_params( + layer: torch.nn.Module, tensors: dict[str, torch.Tensor] +) -> bool: + """Write weights into layerwise CPU buffers, not GPU placeholders.""" + root = getattr(layer, "_offload_root", None) + prefix = getattr(layer, "_offload_param_prefix", None) + if root is None or prefix is None: + return False + managers = getattr(root, "layerwise_offload_managers", None) + if not managers: + return False + named = {f"{prefix}.{key}": value for key, value in tensors.items()} + owned: dict[str, object] = {} + for manager in managers: + if not getattr(manager, "enabled", False): + continue + for name in named: + if name in owned: + continue + has = getattr(manager, "has_cpu_weight", None) + if callable(has) and has(name): + owned[name] = manager + if not owned: + return False + missing: list[str] = [] + for name, value in named.items(): + manager = owned.get(name) + if manager is None: + continue + updated = manager.update_cpu_weights({name: value}) or set() + if name not in updated: + missing.append(name) + if missing: + raise RuntimeError( + f"layerwise LoRA writeback missed {missing} (prefix={prefix})" + ) + _refresh_packed_from_manager(layer) + return True + + +def finish_offload_writeback() -> None: + """No-op; writeback is already synchronous.""" + return + + +def write_dense_weight(layer: torch.nn.Module, weight: torch.Tensor) -> None: + """Write into the manager/view store. Do not replace the Parameter object.""" + dest = layer.weight + if hasattr(dest, "to_local"): + dest = dest.to_local() + if write_offload_params(layer, {"weight": weight}): + _release_stale_gpu_layer(layer) + return + packed_view = getattr(layer, "_packed_weight_cpu", None) + if copy_into_packed_view(packed_view, weight): + _release_stale_gpu_layer(layer) + return + if is_offload_placeholder(dest): + raise RuntimeError( + "layerwise LoRA writeback has no manager CPU view for a (1,) " + "placeholder; convert_to_lora_layers must bind the offload root " + "and _packed_weight_cpu first." + ) + packed = weight.detach().to(device=dest.device, dtype=dest.dtype) + if dest.data.is_inference() or dest.dtype != packed.dtype: + dest.data = packed + else: + dest.data.copy_(packed) + + def iter_materialized_weights(module: torch.nn.Module): """Yield (name, tensor) pairs with materialized weights, even under offload. @@ -3033,13 +3279,16 @@ def iter_materialized_weights(module: torch.nn.Module): offloaded_names: set[str] = set() for manager in offload_managers: for name, tensor in manager.iter_cpu_weights(): - offloaded_names.add(name) + offloaded_names.update(offload_param_name_aliases(name)) yield name, tensor - # Yield non-offloaded parameters (e.g. final norms, embeddings). + # Skip LoRA-wrapped aliases; those live tensors are often (1,) placeholders. for name, param in module.named_parameters(): - if name not in offloaded_names: - yield name, param + if name in offloaded_names: + continue + if is_offload_placeholder(param): + continue + yield name, param def is_layerwise_offloaded_module(module: torch.nn.Module) -> bool: diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py b/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py index fe9248a76d..0afa9dc2b0 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/lora/pipeline.py @@ -22,6 +22,8 @@ from sglang.multimodal_gen.runtime.layers.lora.linear import ( from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( is_layerwise_offloaded_module, + iter_materialized_weights, + refresh_layerwise_targets, ) from sglang.multimodal_gen.runtime.managers.memory_managers.weight_snapshot import ( restore_weight_snapshot, @@ -252,6 +254,28 @@ class LoRAPipeline(ComposedPipelineBase): else: return [], f"Invalid target: {target}. Valid targets: {self.VALID_TARGETS}" + def _has_layerwise_offload( + self, + target_modules: list[tuple[str, dict[str, BaseLayerWithLoRA]]] | None = None, + ) -> bool: + names: list[str] + if target_modules is not None: + names = [module_name for module_name, _ in target_modules] + else: + names = ["transformer", "transformer_2", "critic"] + return any( + is_layerwise_offloaded_module(self.modules.get(name)) for name in names + ) + + def _weight_update_context( + self, + target_modules: list[tuple[str, dict[str, BaseLayerWithLoRA]]] | None = None, + ): + """Do not load_all when layerwise is on.""" + if self._has_layerwise_offload(target_modules): + return nullcontext() + return self._temporarily_disable_offload(target_modules=target_modules) + @contextmanager def _temporarily_disable_offload( self, @@ -356,6 +380,11 @@ class LoRAPipeline(ComposedPipelineBase): The number of layers converted. """ converted_count = 0 + materialized = ( + dict(iter_materialized_weights(module)) + if is_layerwise_offloaded_module(module) + else {} + ) for name, layer in module.named_modules(): if not self.is_target_layer(name): continue @@ -374,12 +403,62 @@ class LoRAPipeline(ComposedPipelineBase): snapshot_base=snapshot_base, ) if lora_layer is not None: + self._bind_lora_offload_view( + module, + name, + lora_layer, + materialized, + snapshot_base=snapshot_base, + ) target_lora_layers[name] = lora_layer replace_submodule(self.modules[module_name], name, lora_layer) converted_count += 1 + if converted_count and is_layerwise_offloaded_module(module): + refresh_layerwise_targets(module) + materialized = dict(iter_materialized_weights(module)) + bound = 0 + for name, lora_layer in target_lora_layers.items(): + self._bind_lora_offload_view( + module, + name, + lora_layer, + materialized, + snapshot_base=snapshot_base, + ) + packed = getattr(lora_layer.base_layer, "_packed_weight_cpu", None) + if packed is not None and packed.numel() > 1: + bound += 1 + logger.info( + "Layerwise LoRA bind: %d/%d layers have a CPU view after wrap", + bound, + converted_count, + ) + return converted_count + def _bind_lora_offload_view( + self, + module: torch.nn.Module, + name: str, + lora_layer: BaseLayerWithLoRA, + materialized: dict[str, torch.Tensor], + snapshot_base: bool = True, + ) -> None: + """Bind LoRA to the manager CPU view, not the (1,) GPU placeholder.""" + if not materialized: + return + weight = materialized.get(f"{name}.weight") + if weight is None: + return + data = weight.detach() + lora_layer.cpu_weight = data.clone() if snapshot_base else data + lora_layer._base_is_view = not snapshot_base + base = lora_layer.base_layer + base._offload_root = module + base._offload_param_prefix = name + base._packed_weight_cpu = data + def _reject_lora_on_packed_weights(self) -> None: """Fail before any layer is replaced if a target has no plain weight. @@ -997,14 +1076,9 @@ class LoRAPipeline(ComposedPipelineBase): # unsupported-LoRA error. Offloaded placeholders still carry the name. self._reject_lora_on_packed_weights() - # Disable layerwise offload before convert_to_lora_layers to ensure weights are accessible - # This is critical because convert_to_lora_layers needs to save cpu_weight from actual weights, - # not from offloaded placeholder tensors + # Layerwise keeps manager views; do not load_all. if not self.lora_initialized: - with self._temporarily_disable_offload( - target="all", use_module_names_only=True - ): - self.convert_to_lora_layers() + self.convert_to_lora_layers(snapshot_base=not self._has_layerwise_offload()) # Check adapter presence and load missing adapters adapter_updated = False @@ -1095,9 +1169,7 @@ class LoRAPipeline(ComposedPipelineBase): if self._needs_lora_weight_update_context( target_modules, merge_weights_by_module ): - weight_update_context = self._temporarily_disable_offload( - target_modules=target_modules - ) + weight_update_context = self._weight_update_context(target_modules) else: weight_update_context = nullcontext() @@ -1149,6 +1221,12 @@ class LoRAPipeline(ComposedPipelineBase): tgt_strengths.copy(), ) + from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + finish_offload_writeback, + ) + + finish_offload_writeback() + logger.info( "Rank %d: LoRA adapter(s) %s applied to %d layers (targets: %s, strengths: %s, merge_mode=%s)", rank, @@ -1238,9 +1316,7 @@ class LoRAPipeline(ComposedPipelineBase): ): modules_requiring_unmerge.append((module_name, lora_layers_dict)) - offload_context = self._temporarily_disable_offload( - target_modules=modules_requiring_unmerge - ) + offload_context = self._weight_update_context(modules_requiring_unmerge) with offload_context: for module_name, lora_layers_dict in target_modules: for layer in lora_layers_dict.values(): @@ -1270,8 +1346,7 @@ class LoRAPipeline(ComposedPipelineBase): if not target_modules: return - # Disable layerwise offload if enabled: load all layers to GPU - with self._temporarily_disable_offload(target_modules=target_modules): + with self._weight_update_context(target_modules): for module_name, lora_layers_dict in target_modules: if not self._should_merge_lora_for_layers( module_name, lora_layers_dict, self.server_args.lora_merge_mode @@ -1320,6 +1395,11 @@ class LoRAPipeline(ComposedPipelineBase): logger.info( "LoRA weights merged for %s (strength: %s)", module_name, strength ) + from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + finish_offload_writeback, + ) + + finish_offload_writeback() def unmerge_lora_weights(self, target: str = "all") -> None: """ @@ -1354,7 +1434,7 @@ class LoRAPipeline(ComposedPipelineBase): "LoRA weights are not merged for %s, skipping", module_name ) continue - with self._temporarily_disable_offload(target_modules=target_modules): + with self._weight_update_context(target_modules): for name, layer in lora_layers_dict.items(): # Check layer-level state to avoid raising exception if hasattr(layer, "merged") and not layer.merged: @@ -1378,6 +1458,11 @@ class LoRAPipeline(ComposedPipelineBase): self.cur_adapter_strength.pop(module_name, None) self.cur_adapter_config.pop(module_name, None) logger.info("LoRA weights unmerged for %s", module_name) + from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + finish_offload_writeback, + ) + + finish_offload_writeback() def get_lora_status(self) -> dict[str, Any]: """ diff --git a/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py b/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py index 795806dc89..58e52e0350 100644 --- a/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py +++ b/python/sglang/multimodal_gen/test/unit/test_layerwise_offload.py @@ -2820,3 +2820,103 @@ def test_configure_logs_a_permanent_lifetime(monkeypatch): ) assert any("placed 3 permanent resident layers" in line for line in logs) assert logs[-1].endswith("resident=3/8 (permanent), policy=leading") + + +class _LoRAWrap(torch.nn.Module): + """Mimic BaseLayerWithLoRA: live Parameter name becomes *.base_layer.weight.""" + + def __init__(self, base: torch.nn.Module) -> None: + super().__init__() + self.base_layer = base + + @property + def weight(self): + return self.base_layer.weight + + +def test_offload_param_name_aliases_cover_lora_wrap(): + from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + offload_param_name_aliases, + ) + + aliases = offload_param_name_aliases("blocks.0.weight") + assert "blocks.0.weight" in aliases + assert "blocks.0.base_layer.weight" in aliases + assert "blocks.0.weight" in offload_param_name_aliases("blocks.0.base_layer.weight") + + +def test_parking_does_not_steal_lora_wrapped_layer_weights(monkeypatch): + """After wrap, live names are *.base_layer.weight; park must still skip them.""" + _patch_fake_device(monkeypatch) + model = _MixinModel() + model.configure_layerwise_offload(_server_args(performance_mode="memory")) + _headroom(monkeypatch, 0) + + for i, block in enumerate(list(model.blocks)): + model.blocks[i] = _LoRAWrap(block) + + from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + refresh_layerwise_targets, + ) + + refresh_layerwise_targets(model) + live_names = {n for n, _ in model.named_parameters() if n.endswith(".weight")} + assert any("base_layer.weight" in n for n in live_names) + + managed = model._managed_layer_parameter_names() + assert live_names <= managed + + model.park_non_layer_weights() + parked = set(model._parked_non_layer_weights) + assert not (parked & live_names), parked + + +def test_iter_materialized_weights_skips_wrapped_placeholders(monkeypatch): + from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + iter_materialized_weights, + refresh_layerwise_targets, + ) + + model = _configure_mixin_model(monkeypatch) + before = dict(iter_materialized_weights(model)) + assert "blocks.0.weight" in before + assert before["blocks.0.weight"].numel() > 1 + + for i, block in enumerate(list(model.blocks)): + model.blocks[i] = _LoRAWrap(block) + refresh_layerwise_targets(model) + + after = dict(iter_materialized_weights(model)) + assert "blocks.0.weight" in after + assert after["blocks.0.weight"].numel() > 1 + assert "blocks.0.base_layer.weight" not in after + + +def test_write_dense_weight_keeps_manager_parameter(monkeypatch): + from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + write_dense_weight, + ) + + model = _configure_mixin_model(monkeypatch) + layer = model.blocks[0] + manager = model.layerwise_offload_managers[0] + original = manager.get_target_with_name("blocks.0.weight") + layer._offload_root = model + layer._offload_param_prefix = "blocks.0" + + merged = torch.arange(9, dtype=torch.float32).reshape(3, 3) + 7 + with torch.no_grad(): + write_dense_weight(layer, merged) + + assert manager.get_target_with_name("blocks.0.weight") is original + got = manager.get_cpu_weight("blocks.0.weight") + assert got is not None + torch.testing.assert_close(got, merged) + + +def test_finish_offload_writeback_is_noop_when_idle(): + from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import ( + finish_offload_writeback, + ) + + finish_offload_writeback() diff --git a/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py b/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py index 00e7ef24c2..813c1b1cd9 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py +++ b/python/sglang/multimodal_gen/test/unit/test_lora_pipeline.py @@ -116,6 +116,7 @@ def test_zero_copy_snapshot_is_limited_to_cpu_backed_layers(): assert not _use_owned_base_snapshot(False, "cpu") assert not _use_owned_base_snapshot(False, "meta") assert _use_owned_base_snapshot(False, "cuda") + assert not _use_owned_base_snapshot(False, "cuda", numel=1) assert _use_owned_base_snapshot(True, "cpu") cpu_layer = wrap_with_lora_layer( @@ -323,3 +324,21 @@ def test_lora_exact_file_url_needs_no_weight_name(tmp_path): "*.json", "adapter.safetensors", ] + + +def test_view_merge_unmerges_by_inverse_without_owned_clone(): + torch.manual_seed(0) + base = torch.nn.Linear(4, 4, bias=False) + original = base.weight.detach().clone() + layer = wrap_with_lora_layer(base, lora_rank=2, lora_alpha=2, snapshot_base=False) + assert layer is not None + assert layer._base_is_view + A = torch.randn(2, 4) + B = torch.randn(4, 2) + layer.set_lora_weights(A, B, strength=0.5, clear_existing=True, merge_weights=True) + assert layer.merged + assert layer._base_is_view + torch.testing.assert_close(layer.base_layer.weight, original + 0.5 * (B @ A)) + layer.unmerge_lora_weights() + torch.testing.assert_close(layer.base_layer.weight, original) + assert not layer.merged