[Fix] offloader: Preserve cached weights during CPU offload (#44139)

Co-authored-by: Alex Nails <alex.nails@radixark.ai>
This commit is contained in:
Yao Matrix
2026-10-23 00:50:24 -07:00
committed by GitHub
co-authored by Alex Nails
parent 6f466f655a
commit 6d6c9cd4ae
6 changed files with 230 additions and 41 deletions
+5 -1
View File
@@ -1120,8 +1120,12 @@ class GemmaRMSNorm(BaseFusedOp):
def _weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
assert param.size() == loaded_weight.size()
param.data.copy_(loaded_weight)
self.refresh_gemma_weight()
def refresh_gemma_weight(self) -> None:
# Keep storage stable for CUDA graphs or fused paths that capture this buffer.
torch.add(param.data, 1.0, out=self.gemma_weight)
# CPU offload may move the parameter to host while the buffer stays on device.
self.gemma_weight.copy_(self.weight.data).add_(1.0)
def _forward_impl(
self,
@@ -79,6 +79,13 @@ class RadixLinearAttention(nn.Module):
self.dt_bias = dt_bias
self.lower_bound = lower_bound
def refresh_conv_weights(self, weight: torch.Tensor) -> None:
"""Rebuild the cached convolution view after replacing its parameter."""
if weight.dim() == 3 and weight.size(1) == 1:
self.conv_weights = weight.view(weight.size(0), weight.size(2))
else:
self.conv_weights = weight.squeeze() if weight.dim() > 2 else weight
def forward(
self,
forward_batch: ForwardBatch,
+6 -15
View File
@@ -84,6 +84,7 @@ from sglang.srt.distributed import (
model_parallel_is_initialized,
)
from sglang.srt.layers.layer_boundary.stage import check_stage_producers
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES
from sglang.srt.layers.moe.utils import (
resolve_shared_experts_config,
@@ -92,6 +93,7 @@ from sglang.srt.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
from sglang.srt.model_loader.remote_instance_weight_loader_utils import (
trigger_transferring_weights_request,
)
@@ -2047,14 +2049,8 @@ class PreshardedModelLoader(DefaultModelLoader):
@staticmethod
def _rebind_parameter_aliases(model: nn.Module) -> None:
for _, module in model.named_modules():
gemma_w = getattr(module, "gemma_weight", None)
weight = getattr(module, "weight", None)
if (
isinstance(gemma_w, torch.Tensor)
and isinstance(weight, torch.nn.Parameter)
and gemma_w.shape == weight.shape
):
torch.add(weight.data, 1.0, out=gemma_w)
if isinstance(module, GemmaRMSNorm):
module.refresh_gemma_weight()
attn = getattr(module, "attn", None)
conv1d = getattr(module, "conv1d", None)
@@ -2067,13 +2063,8 @@ class PreshardedModelLoader(DefaultModelLoader):
if conv1d is None:
continue
cweight = getattr(conv1d, "weight", None)
if cweight is not None and hasattr(attn, "conv_weights"):
if cweight.dim() == 3 and cweight.size(1) == 1:
attn.conv_weights = cweight.view(cweight.size(0), cweight.size(2))
else:
attn.conv_weights = (
cweight.squeeze() if cweight.dim() > 2 else cweight
)
if cweight is not None and isinstance(attn, RadixLinearAttention):
attn.refresh_conv_weights(cweight)
if hasattr(conv1d, "bias") and hasattr(attn, "bias"):
attn.bias = conv1d.bias
+19 -22
View File
@@ -98,7 +98,16 @@ def _get_resident_parameter_ids(module: torch.nn.Module):
# functional_call only replaces registered parameters and buffers, so cached
# tensors held as ordinary attributes need their backing weights to stay put.
resident = set()
cached_storages = set()
for owner in module.modules():
for value in vars(owner).values():
if (
isinstance(value, torch.Tensor)
and not isinstance(value, torch.nn.Parameter)
and not value.is_meta
and value.numel() > 0
):
cached_storages.add((value.device, value.untyped_storage().data_ptr()))
# MLA post_load_weights derives w_kc/w_vc from kv_b_proj weights on
# their current device. These attributes already exist before loading.
projection = getattr(owner, "kv_b_proj", None)
@@ -108,29 +117,14 @@ def _get_resident_parameter_ids(module: torch.nn.Module):
and hasattr(owner, "w_vc")
):
resident.update(id(parameter) for parameter in projection.parameters())
# KDA caches a storage-sharing view of qkv_conv1d.weight in conv_weights
# during construction. Offloading the weight would leave that view stale.
projection = getattr(owner, "qkv_conv1d", None)
attention = getattr(owner, "attn", None)
if isinstance(projection, torch.nn.Module) and isinstance(
attention, torch.nn.Module
for parameter in module.parameters():
if (
not parameter.is_meta
and parameter.numel() > 0
and (parameter.device, parameter.untyped_storage().data_ptr())
in cached_storages
):
weight = getattr(projection, "weight", None)
cached = getattr(attention, "conv_weights", None)
if (
isinstance(weight, torch.nn.Parameter)
and isinstance(cached, torch.Tensor)
and cached.device == weight.device
):
weight_storage = weight.untyped_storage()
cached_storage = cached.untyped_storage()
if (
weight_storage.nbytes() > 0
and weight_storage.data_ptr() != 0
and weight_storage.nbytes() == cached_storage.nbytes()
and weight_storage.data_ptr() == cached_storage.data_ptr()
):
resident.add(id(weight))
resident.add(id(parameter))
return resident
@@ -267,6 +261,7 @@ class OffloaderV2(BaseOffloader):
module=submodule,
alt_stream=alt_stream,
whitelist_param_names=whitelist_param_names,
resident_parameter_ids=_get_resident_parameter_ids(module),
)
)
@@ -328,6 +323,7 @@ class _ModuleOffloader(ABC):
module: torch.nn.Module,
alt_stream: torch.cuda.Stream,
whitelist_param_names: List[str],
resident_parameter_ids: set[int],
):
self.mode = mode
self.module = module
@@ -349,6 +345,7 @@ class _ModuleOffloader(ABC):
self._param_offloaders = {
name: _BaseParamOffloader.create(mode, module=module, param_name=name)
for name in whitelist_param_names
if id(param_dict[name]) not in resident_parameter_ids
}
def post_init(self):
+1 -3
View File
@@ -261,9 +261,7 @@ class IpcModelLoader(BaseModelLoader):
attn = getattr(module, "attn", None)
if conv1d is not None and isinstance(attn, RadixLinearAttention):
if hasattr(conv1d, "weight") and conv1d.weight is not None:
attn.conv_weights = conv1d.weight.view(
conv1d.weight.size(0), conv1d.weight.size(2)
)
attn.refresh_conv_weights(conv1d.weight)
if hasattr(conv1d, "bias") and conv1d.bias is not None:
attn.bias = conv1d.bias
count += 1