From 6d6c9cd4ae4491e5293e9a2b06e8f71dfde6dd18 Mon Sep 17 00:00:00 2001 From: Yao Matrix Date: Fri, 23 Oct 2026 00:50:24 -0700 Subject: [PATCH] [Fix] offloader: Preserve cached weights during CPU offload (#44139) Co-authored-by: Alex Nails --- python/sglang/srt/layers/layernorm.py | 6 +- .../srt/layers/radix_linear_attention.py | 7 + python/sglang/srt/model_loader/loader.py | 21 +- python/sglang/srt/utils/offloader.py | 41 ++-- python/sglang/srt/weight_cache/ipc_loader.py | 4 +- .../model_loading/test_cpu_offloader.py | 192 ++++++++++++++++++ 6 files changed, 230 insertions(+), 41 deletions(-) create mode 100644 test/registered/model_loading/test_cpu_offloader.py diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index ec64ba55fb..5786487764 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -1120,8 +1120,12 @@ class GemmaRMSNorm(BaseFusedOp): def _weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None: assert param.size() == loaded_weight.size() param.data.copy_(loaded_weight) + self.refresh_gemma_weight() + + def refresh_gemma_weight(self) -> None: # Keep storage stable for CUDA graphs or fused paths that capture this buffer. - torch.add(param.data, 1.0, out=self.gemma_weight) + # CPU offload may move the parameter to host while the buffer stays on device. + self.gemma_weight.copy_(self.weight.data).add_(1.0) def _forward_impl( self, diff --git a/python/sglang/srt/layers/radix_linear_attention.py b/python/sglang/srt/layers/radix_linear_attention.py index 00d44400f8..91c7c0ff04 100644 --- a/python/sglang/srt/layers/radix_linear_attention.py +++ b/python/sglang/srt/layers/radix_linear_attention.py @@ -79,6 +79,13 @@ class RadixLinearAttention(nn.Module): self.dt_bias = dt_bias self.lower_bound = lower_bound + def refresh_conv_weights(self, weight: torch.Tensor) -> None: + """Rebuild the cached convolution view after replacing its parameter.""" + if weight.dim() == 3 and weight.size(1) == 1: + self.conv_weights = weight.view(weight.size(0), weight.size(2)) + else: + self.conv_weights = weight.squeeze() if weight.dim() > 2 else weight + def forward( self, forward_batch: ForwardBatch, diff --git a/python/sglang/srt/model_loader/loader.py b/python/sglang/srt/model_loader/loader.py index fdb3187567..711af8a86b 100644 --- a/python/sglang/srt/model_loader/loader.py +++ b/python/sglang/srt/model_loader/loader.py @@ -84,6 +84,7 @@ from sglang.srt.distributed import ( model_parallel_is_initialized, ) from sglang.srt.layers.layer_boundary.stage import check_stage_producers +from sglang.srt.layers.layernorm import GemmaRMSNorm from sglang.srt.layers.modelopt_utils import QUANT_CFG_CHOICES from sglang.srt.layers.moe.utils import ( resolve_shared_experts_config, @@ -92,6 +93,7 @@ from sglang.srt.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, ) +from sglang.srt.layers.radix_linear_attention import RadixLinearAttention from sglang.srt.model_loader.remote_instance_weight_loader_utils import ( trigger_transferring_weights_request, ) @@ -2047,14 +2049,8 @@ class PreshardedModelLoader(DefaultModelLoader): @staticmethod def _rebind_parameter_aliases(model: nn.Module) -> None: for _, module in model.named_modules(): - gemma_w = getattr(module, "gemma_weight", None) - weight = getattr(module, "weight", None) - if ( - isinstance(gemma_w, torch.Tensor) - and isinstance(weight, torch.nn.Parameter) - and gemma_w.shape == weight.shape - ): - torch.add(weight.data, 1.0, out=gemma_w) + if isinstance(module, GemmaRMSNorm): + module.refresh_gemma_weight() attn = getattr(module, "attn", None) conv1d = getattr(module, "conv1d", None) @@ -2067,13 +2063,8 @@ class PreshardedModelLoader(DefaultModelLoader): if conv1d is None: continue cweight = getattr(conv1d, "weight", None) - if cweight is not None and hasattr(attn, "conv_weights"): - if cweight.dim() == 3 and cweight.size(1) == 1: - attn.conv_weights = cweight.view(cweight.size(0), cweight.size(2)) - else: - attn.conv_weights = ( - cweight.squeeze() if cweight.dim() > 2 else cweight - ) + if cweight is not None and isinstance(attn, RadixLinearAttention): + attn.refresh_conv_weights(cweight) if hasattr(conv1d, "bias") and hasattr(attn, "bias"): attn.bias = conv1d.bias diff --git a/python/sglang/srt/utils/offloader.py b/python/sglang/srt/utils/offloader.py index 17bf994d92..0d348795c7 100644 --- a/python/sglang/srt/utils/offloader.py +++ b/python/sglang/srt/utils/offloader.py @@ -98,7 +98,16 @@ def _get_resident_parameter_ids(module: torch.nn.Module): # functional_call only replaces registered parameters and buffers, so cached # tensors held as ordinary attributes need their backing weights to stay put. resident = set() + cached_storages = set() for owner in module.modules(): + for value in vars(owner).values(): + if ( + isinstance(value, torch.Tensor) + and not isinstance(value, torch.nn.Parameter) + and not value.is_meta + and value.numel() > 0 + ): + cached_storages.add((value.device, value.untyped_storage().data_ptr())) # MLA post_load_weights derives w_kc/w_vc from kv_b_proj weights on # their current device. These attributes already exist before loading. projection = getattr(owner, "kv_b_proj", None) @@ -108,29 +117,14 @@ def _get_resident_parameter_ids(module: torch.nn.Module): and hasattr(owner, "w_vc") ): resident.update(id(parameter) for parameter in projection.parameters()) - # KDA caches a storage-sharing view of qkv_conv1d.weight in conv_weights - # during construction. Offloading the weight would leave that view stale. - projection = getattr(owner, "qkv_conv1d", None) - attention = getattr(owner, "attn", None) - if isinstance(projection, torch.nn.Module) and isinstance( - attention, torch.nn.Module + for parameter in module.parameters(): + if ( + not parameter.is_meta + and parameter.numel() > 0 + and (parameter.device, parameter.untyped_storage().data_ptr()) + in cached_storages ): - weight = getattr(projection, "weight", None) - cached = getattr(attention, "conv_weights", None) - if ( - isinstance(weight, torch.nn.Parameter) - and isinstance(cached, torch.Tensor) - and cached.device == weight.device - ): - weight_storage = weight.untyped_storage() - cached_storage = cached.untyped_storage() - if ( - weight_storage.nbytes() > 0 - and weight_storage.data_ptr() != 0 - and weight_storage.nbytes() == cached_storage.nbytes() - and weight_storage.data_ptr() == cached_storage.data_ptr() - ): - resident.add(id(weight)) + resident.add(id(parameter)) return resident @@ -267,6 +261,7 @@ class OffloaderV2(BaseOffloader): module=submodule, alt_stream=alt_stream, whitelist_param_names=whitelist_param_names, + resident_parameter_ids=_get_resident_parameter_ids(module), ) ) @@ -328,6 +323,7 @@ class _ModuleOffloader(ABC): module: torch.nn.Module, alt_stream: torch.cuda.Stream, whitelist_param_names: List[str], + resident_parameter_ids: set[int], ): self.mode = mode self.module = module @@ -349,6 +345,7 @@ class _ModuleOffloader(ABC): self._param_offloaders = { name: _BaseParamOffloader.create(mode, module=module, param_name=name) for name in whitelist_param_names + if id(param_dict[name]) not in resident_parameter_ids } def post_init(self): diff --git a/python/sglang/srt/weight_cache/ipc_loader.py b/python/sglang/srt/weight_cache/ipc_loader.py index 6bf6a9305b..0fe4abb0a1 100644 --- a/python/sglang/srt/weight_cache/ipc_loader.py +++ b/python/sglang/srt/weight_cache/ipc_loader.py @@ -261,9 +261,7 @@ class IpcModelLoader(BaseModelLoader): attn = getattr(module, "attn", None) if conv1d is not None and isinstance(attn, RadixLinearAttention): if hasattr(conv1d, "weight") and conv1d.weight is not None: - attn.conv_weights = conv1d.weight.view( - conv1d.weight.size(0), conv1d.weight.size(2) - ) + attn.refresh_conv_weights(conv1d.weight) if hasattr(conv1d, "bias") and conv1d.bias is not None: attn.bias = conv1d.bias count += 1 diff --git a/test/registered/model_loading/test_cpu_offloader.py b/test/registered/model_loading/test_cpu_offloader.py new file mode 100644 index 0000000000..21eac36061 --- /dev/null +++ b/test/registered/model_loading/test_cpu_offloader.py @@ -0,0 +1,192 @@ +"""CPU offload must preserve device buffers and cached convolution views.""" + +import os +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch +from torch import nn +from torch.nn import functional as F + +from sglang.srt.layers.layernorm import GemmaRMSNorm +from sglang.srt.layers.linear import ColumnParallelLinear, MergedColumnParallelLinear +from sglang.srt.layers.radix_linear_attention import RadixLinearAttention +from sglang.srt.model_loader.loader import PreshardedModelLoader +from sglang.srt.utils.offloader import OffloaderV1, OffloaderV2 +from sglang.srt.weight_cache.ipc_loader import IpcModelLoader +from sglang.test.ci.ci_register import register_cuda_ci, register_xpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small") +# backend-specific: XPU CPU-offload loads host Gemma weights into device buffers +# and must retain cached GDN/KDA views after checkpoint loading. +register_xpu_ci(est_time=10, suite="stage-b-test-1-gpu-xpu") + + +class CachedConvolution(nn.Module): + def __init__( + self, + projection_name: str, + device: torch.device, + cache_name: str = "conv_weights", + ) -> None: + super().__init__() + if projection_name == "qkv_conv1d": + convolution = MergedColumnParallelLinear( + 3, [1, 1, 1], bias=False, parallel_group="replicated" + ) + else: + convolution = ColumnParallelLinear( + 3, 3, bias=False, parallel_group="replicated" + ) + convolution.to(device) + convolution.weight.data = convolution.weight.data.unsqueeze(1) + self.add_module(projection_name, convolution) + self.projection = nn.Linear(3, 2, bias=False, device=device) + with torch.no_grad(): + convolution.weight.zero_() + # Match GDN's view and KDA's squeeze before handing weights to attention. + conv_weights = ( + convolution.weight.squeeze(1) + if projection_name == "qkv_conv1d" + else convolution.weight.view(3, 3) + ) + self.attn = RadixLinearAttention( + layer_id=0, + num_q_heads=1, + num_k_heads=1, + num_v_heads=1, + head_q_dim=1, + head_k_dim=1, + head_v_dim=1, + conv_weights=conv_weights, + ) + self.cache_name = cache_name + if cache_name != "conv_weights": + setattr(self.attn, cache_name, self.attn.conv_weights) + self.attn.conv_weights = None + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + convolution = F.conv1d( + inputs, getattr(self.attn, self.cache_name).unsqueeze(1), groups=3 + ).squeeze(-1) + return self.projection(convolution) + + +class TestCpuOffloader(CustomTestCase): + def setUp(self) -> None: + super().setUp() + self.device = torch.accelerator.current_accelerator(check_available=True) + if self.device is None: + self.skipTest("An accelerator is required to exercise CPU offload") + + def test_cpu_offloaded_gemma_weight_fills_device_buffer(self) -> None: + norm = GemmaRMSNorm(16).to(self.device) + norm = OffloaderV1(cpu_offload_max_bytes=1024**2).maybe_offload_to_cpu(norm) + buffer_address = norm.gemma_weight.data_ptr() + loaded_weights = torch.randn(16, device="cpu") + + norm.weight.weight_loader(norm.weight, loaded_weights) + + self.assertEqual(norm.weight.device.type, "cpu") + self.assertEqual(norm.gemma_weight.device.type, self.device.type) + self.assertEqual(norm.gemma_weight.data_ptr(), buffer_address) + torch.testing.assert_close(norm.gemma_weight.cpu(), loaded_weights + 1.0) + + def test_presharded_rebind_refreshes_cpu_offloaded_gemma_buffer(self) -> None: + norm = GemmaRMSNorm(16).to(self.device) + norm = OffloaderV1(cpu_offload_max_bytes=1024**2).maybe_offload_to_cpu(norm) + buffer_address = norm.gemma_weight.data_ptr() + loaded_weights = torch.randn(16, device="cpu") + norm.weight.data.copy_(loaded_weights) + norm.gemma_weight.zero_() + + PreshardedModelLoader._rebind_parameter_aliases(norm) + + self.assertEqual(norm.weight.device.type, "cpu") + self.assertEqual(norm.gemma_weight.device.type, self.device.type) + self.assertEqual(norm.gemma_weight.data_ptr(), buffer_address) + torch.testing.assert_close(norm.gemma_weight.cpu(), loaded_weights + 1.0) + + def test_checkpoint_load_updates_cached_convolution(self) -> None: + for projection_name, cache_name in ( + ("qkv_conv1d", "conv_weights"), + ("conv1d", "conv_weights"), + ("temporal_projection", "cached_kernel"), + ): + with self.subTest(projection_name=projection_name), torch.no_grad(): + model = CachedConvolution(projection_name, self.device, cache_name) + model = OffloaderV1(cpu_offload_max_bytes=1024**2).maybe_offload_to_cpu( + model + ) + weight = getattr(model, projection_name).weight + weight.weight_loader(weight, torch.ones_like(weight, device="cpu")) + model.projection.weight.fill_(1) + + output = model(torch.ones((1, 3, 3), device=self.device)) + + torch.testing.assert_close( + output, torch.full((1, 2), 9.0, device=self.device) + ) + self.assertEqual(model.projection.weight.device.type, "cpu") + + def test_v2_whitelist_preserves_cached_convolution_storage(self) -> None: + with torch.no_grad(): + model = CachedConvolution("conv1d", self.device) + convolution = model.conv1d + convolution.bias = nn.Parameter(torch.zeros(3, device=self.device)) + # Check V2's parameter selection before its CUDA-only stream transfers. + # The cached view is held outside the selected projection submodule. + with ( + patch( + "sglang.srt.utils.offloader.get_parallel", + return_value=SimpleNamespace(dp_rank=0, num_dp_ranks=1), + ), + patch("sglang.srt.utils.offloader.get_stream", return_value=None), + patch("torch.cuda.memory_allocated", return_value=0), + patch.dict(os.environ, {"SGLANG_RUN_ID": "test-cpu-offloader"}), + ): + offloader = OffloaderV2( + group_size=1, num_in_group=1, prefetch_step=1, mode="cpu" + ) + offloader.wrap_modules( + iter([model]), + submodule_accessor=lambda module: module.conv1d, + whitelist_param_names_creator=lambda module: ["weight", "bias"], + ) + convolution.weight.weight_loader( + convolution.weight, torch.ones((3, 1, 3), device="cpu") + ) + output = F.conv1d( + torch.ones((1, 3, 3), device=self.device), + model.attn.conv_weights.unsqueeze(1), + groups=3, + ) + + torch.testing.assert_close( + output, torch.full((1, 3, 1), 3.0, device=self.device) + ) + self.assertEqual(convolution.weight.device.type, self.device.type) + self.assertEqual(convolution.bias.device.type, "cpu") + + def test_loaders_rebuild_replaced_convolution_view(self) -> None: + for rebind in ( + PreshardedModelLoader._rebind_parameter_aliases, + IpcModelLoader._rebuild_stale_views, + ): + with self.subTest(loader=rebind.__qualname__), torch.no_grad(): + model = CachedConvolution("conv1d", self.device) + model.conv1d.weight = nn.Parameter(torch.ones_like(model.conv1d.weight)) + model.projection.weight.fill_(1) + + rebind(model) + output = model(torch.ones((1, 3, 3), device=self.device)) + + torch.testing.assert_close( + output, torch.full((1, 2), 9.0, device=self.device) + ) + + +if __name__ == "__main__": + unittest.main()