[Fix] offloader: Preserve cached weights during CPU offload (#44139)
Co-authored-by: Alex Nails <alex.nails@radixark.ai>
This commit is contained in:
@@ -1120,8 +1120,12 @@ class GemmaRMSNorm(BaseFusedOp):
|
||||
def _weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
assert param.size() == loaded_weight.size()
|
||||
param.data.copy_(loaded_weight)
|
||||
self.refresh_gemma_weight()
|
||||
|
||||
def refresh_gemma_weight(self) -> None:
|
||||
# Keep storage stable for CUDA graphs or fused paths that capture this buffer.
|
||||
torch.add(param.data, 1.0, out=self.gemma_weight)
|
||||
# CPU offload may move the parameter to host while the buffer stays on device.
|
||||
self.gemma_weight.copy_(self.weight.data).add_(1.0)
|
||||
|
||||
def _forward_impl(
|
||||
self,
|
||||
|
||||
@@ -79,6 +79,13 @@ class RadixLinearAttention(nn.Module):
|
||||
self.dt_bias = dt_bias
|
||||
self.lower_bound = lower_bound
|
||||
|
||||
def refresh_conv_weights(self, weight: torch.Tensor) -> None:
|
||||
"""Rebuild the cached convolution view after replacing its parameter."""
|
||||
if weight.dim() == 3 and weight.size(1) == 1:
|
||||
self.conv_weights = weight.view(weight.size(0), weight.size(2))
|
||||
else:
|
||||
self.conv_weights = weight.squeeze() if weight.dim() > 2 else weight
|
||||
|
||||
def forward(
|
||||
self,
|
||||
forward_batch: ForwardBatch,
|
||||
|
||||
@@ -84,6 +84,7 @@ from sglang.srt.distributed import (
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.stage import check_stage_producers
|
||||
from sglang.srt.layers.layernorm import GemmaRMSNorm
|
||||
from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES
|
||||
from sglang.srt.layers.moe.utils import (
|
||||
resolve_shared_experts_config,
|
||||
@@ -92,6 +93,7 @@ from sglang.srt.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
|
||||
from sglang.srt.model_loader.remote_instance_weight_loader_utils import (
|
||||
trigger_transferring_weights_request,
|
||||
)
|
||||
@@ -2047,14 +2049,8 @@ class PreshardedModelLoader(DefaultModelLoader):
|
||||
@staticmethod
|
||||
def _rebind_parameter_aliases(model: nn.Module) -> None:
|
||||
for _, module in model.named_modules():
|
||||
gemma_w = getattr(module, "gemma_weight", None)
|
||||
weight = getattr(module, "weight", None)
|
||||
if (
|
||||
isinstance(gemma_w, torch.Tensor)
|
||||
and isinstance(weight, torch.nn.Parameter)
|
||||
and gemma_w.shape == weight.shape
|
||||
):
|
||||
torch.add(weight.data, 1.0, out=gemma_w)
|
||||
if isinstance(module, GemmaRMSNorm):
|
||||
module.refresh_gemma_weight()
|
||||
|
||||
attn = getattr(module, "attn", None)
|
||||
conv1d = getattr(module, "conv1d", None)
|
||||
@@ -2067,13 +2063,8 @@ class PreshardedModelLoader(DefaultModelLoader):
|
||||
if conv1d is None:
|
||||
continue
|
||||
cweight = getattr(conv1d, "weight", None)
|
||||
if cweight is not None and hasattr(attn, "conv_weights"):
|
||||
if cweight.dim() == 3 and cweight.size(1) == 1:
|
||||
attn.conv_weights = cweight.view(cweight.size(0), cweight.size(2))
|
||||
else:
|
||||
attn.conv_weights = (
|
||||
cweight.squeeze() if cweight.dim() > 2 else cweight
|
||||
)
|
||||
if cweight is not None and isinstance(attn, RadixLinearAttention):
|
||||
attn.refresh_conv_weights(cweight)
|
||||
if hasattr(conv1d, "bias") and hasattr(attn, "bias"):
|
||||
attn.bias = conv1d.bias
|
||||
|
||||
|
||||
@@ -98,7 +98,16 @@ def _get_resident_parameter_ids(module: torch.nn.Module):
|
||||
# functional_call only replaces registered parameters and buffers, so cached
|
||||
# tensors held as ordinary attributes need their backing weights to stay put.
|
||||
resident = set()
|
||||
cached_storages = set()
|
||||
for owner in module.modules():
|
||||
for value in vars(owner).values():
|
||||
if (
|
||||
isinstance(value, torch.Tensor)
|
||||
and not isinstance(value, torch.nn.Parameter)
|
||||
and not value.is_meta
|
||||
and value.numel() > 0
|
||||
):
|
||||
cached_storages.add((value.device, value.untyped_storage().data_ptr()))
|
||||
# MLA post_load_weights derives w_kc/w_vc from kv_b_proj weights on
|
||||
# their current device. These attributes already exist before loading.
|
||||
projection = getattr(owner, "kv_b_proj", None)
|
||||
@@ -108,29 +117,14 @@ def _get_resident_parameter_ids(module: torch.nn.Module):
|
||||
and hasattr(owner, "w_vc")
|
||||
):
|
||||
resident.update(id(parameter) for parameter in projection.parameters())
|
||||
# KDA caches a storage-sharing view of qkv_conv1d.weight in conv_weights
|
||||
# during construction. Offloading the weight would leave that view stale.
|
||||
projection = getattr(owner, "qkv_conv1d", None)
|
||||
attention = getattr(owner, "attn", None)
|
||||
if isinstance(projection, torch.nn.Module) and isinstance(
|
||||
attention, torch.nn.Module
|
||||
for parameter in module.parameters():
|
||||
if (
|
||||
not parameter.is_meta
|
||||
and parameter.numel() > 0
|
||||
and (parameter.device, parameter.untyped_storage().data_ptr())
|
||||
in cached_storages
|
||||
):
|
||||
weight = getattr(projection, "weight", None)
|
||||
cached = getattr(attention, "conv_weights", None)
|
||||
if (
|
||||
isinstance(weight, torch.nn.Parameter)
|
||||
and isinstance(cached, torch.Tensor)
|
||||
and cached.device == weight.device
|
||||
):
|
||||
weight_storage = weight.untyped_storage()
|
||||
cached_storage = cached.untyped_storage()
|
||||
if (
|
||||
weight_storage.nbytes() > 0
|
||||
and weight_storage.data_ptr() != 0
|
||||
and weight_storage.nbytes() == cached_storage.nbytes()
|
||||
and weight_storage.data_ptr() == cached_storage.data_ptr()
|
||||
):
|
||||
resident.add(id(weight))
|
||||
resident.add(id(parameter))
|
||||
return resident
|
||||
|
||||
|
||||
@@ -267,6 +261,7 @@ class OffloaderV2(BaseOffloader):
|
||||
module=submodule,
|
||||
alt_stream=alt_stream,
|
||||
whitelist_param_names=whitelist_param_names,
|
||||
resident_parameter_ids=_get_resident_parameter_ids(module),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -328,6 +323,7 @@ class _ModuleOffloader(ABC):
|
||||
module: torch.nn.Module,
|
||||
alt_stream: torch.cuda.Stream,
|
||||
whitelist_param_names: List[str],
|
||||
resident_parameter_ids: set[int],
|
||||
):
|
||||
self.mode = mode
|
||||
self.module = module
|
||||
@@ -349,6 +345,7 @@ class _ModuleOffloader(ABC):
|
||||
self._param_offloaders = {
|
||||
name: _BaseParamOffloader.create(mode, module=module, param_name=name)
|
||||
for name in whitelist_param_names
|
||||
if id(param_dict[name]) not in resident_parameter_ids
|
||||
}
|
||||
|
||||
def post_init(self):
|
||||
|
||||
@@ -261,9 +261,7 @@ class IpcModelLoader(BaseModelLoader):
|
||||
attn = getattr(module, "attn", None)
|
||||
if conv1d is not None and isinstance(attn, RadixLinearAttention):
|
||||
if hasattr(conv1d, "weight") and conv1d.weight is not None:
|
||||
attn.conv_weights = conv1d.weight.view(
|
||||
conv1d.weight.size(0), conv1d.weight.size(2)
|
||||
)
|
||||
attn.refresh_conv_weights(conv1d.weight)
|
||||
if hasattr(conv1d, "bias") and conv1d.bias is not None:
|
||||
attn.bias = conv1d.bias
|
||||
count += 1
|
||||
|
||||
Reference in New Issue
Block a user