Files
sglang/test/registered/model_loading/test_cpu_offloader.py
T

193 lines
8.1 KiB
Python

"""CPU offload must preserve device buffers and cached convolution views."""
import os
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from torch import nn
from torch.nn import functional as F
from sglang.srt.layers.layernorm import GemmaRMSNorm
from sglang.srt.layers.linear import ColumnParallelLinear, MergedColumnParallelLinear
from sglang.srt.layers.radix_linear_attention import RadixLinearAttention
from sglang.srt.model_loader.loader import PreshardedModelLoader
from sglang.srt.utils.offloader import OffloaderV1, OffloaderV2
from sglang.srt.weight_cache.ipc_loader import IpcModelLoader
from sglang.test.ci.ci_register import register_cuda_ci, register_xpu_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small")
# backend-specific: XPU CPU-offload loads host Gemma weights into device buffers
# and must retain cached GDN/KDA views after checkpoint loading.
register_xpu_ci(est_time=10, suite="stage-b-test-1-gpu-xpu")
class CachedConvolution(nn.Module):
def __init__(
self,
projection_name: str,
device: torch.device,
cache_name: str = "conv_weights",
) -> None:
super().__init__()
if projection_name == "qkv_conv1d":
convolution = MergedColumnParallelLinear(
3, [1, 1, 1], bias=False, parallel_group="replicated"
)
else:
convolution = ColumnParallelLinear(
3, 3, bias=False, parallel_group="replicated"
)
convolution.to(device)
convolution.weight.data = convolution.weight.data.unsqueeze(1)
self.add_module(projection_name, convolution)
self.projection = nn.Linear(3, 2, bias=False, device=device)
with torch.no_grad():
convolution.weight.zero_()
# Match GDN's view and KDA's squeeze before handing weights to attention.
conv_weights = (
convolution.weight.squeeze(1)
if projection_name == "qkv_conv1d"
else convolution.weight.view(3, 3)
)
self.attn = RadixLinearAttention(
layer_id=0,
num_q_heads=1,
num_k_heads=1,
num_v_heads=1,
head_q_dim=1,
head_k_dim=1,
head_v_dim=1,
conv_weights=conv_weights,
)
self.cache_name = cache_name
if cache_name != "conv_weights":
setattr(self.attn, cache_name, self.attn.conv_weights)
self.attn.conv_weights = None
def forward(self, inputs: torch.Tensor) -> torch.Tensor:
convolution = F.conv1d(
inputs, getattr(self.attn, self.cache_name).unsqueeze(1), groups=3
).squeeze(-1)
return self.projection(convolution)
class TestCpuOffloader(CustomTestCase):
def setUp(self) -> None:
super().setUp()
self.device = torch.accelerator.current_accelerator(check_available=True)
if self.device is None:
self.skipTest("An accelerator is required to exercise CPU offload")
def test_cpu_offloaded_gemma_weight_fills_device_buffer(self) -> None:
norm = GemmaRMSNorm(16).to(self.device)
norm = OffloaderV1(cpu_offload_max_bytes=1024**2).maybe_offload_to_cpu(norm)
buffer_address = norm.gemma_weight.data_ptr()
loaded_weights = torch.randn(16, device="cpu")
norm.weight.weight_loader(norm.weight, loaded_weights)
self.assertEqual(norm.weight.device.type, "cpu")
self.assertEqual(norm.gemma_weight.device.type, self.device.type)
self.assertEqual(norm.gemma_weight.data_ptr(), buffer_address)
torch.testing.assert_close(norm.gemma_weight.cpu(), loaded_weights + 1.0)
def test_presharded_rebind_refreshes_cpu_offloaded_gemma_buffer(self) -> None:
norm = GemmaRMSNorm(16).to(self.device)
norm = OffloaderV1(cpu_offload_max_bytes=1024**2).maybe_offload_to_cpu(norm)
buffer_address = norm.gemma_weight.data_ptr()
loaded_weights = torch.randn(16, device="cpu")
norm.weight.data.copy_(loaded_weights)
norm.gemma_weight.zero_()
PreshardedModelLoader._rebind_parameter_aliases(norm)
self.assertEqual(norm.weight.device.type, "cpu")
self.assertEqual(norm.gemma_weight.device.type, self.device.type)
self.assertEqual(norm.gemma_weight.data_ptr(), buffer_address)
torch.testing.assert_close(norm.gemma_weight.cpu(), loaded_weights + 1.0)
def test_checkpoint_load_updates_cached_convolution(self) -> None:
for projection_name, cache_name in (
("qkv_conv1d", "conv_weights"),
("conv1d", "conv_weights"),
("temporal_projection", "cached_kernel"),
):
with self.subTest(projection_name=projection_name), torch.no_grad():
model = CachedConvolution(projection_name, self.device, cache_name)
model = OffloaderV1(cpu_offload_max_bytes=1024**2).maybe_offload_to_cpu(
model
)
weight = getattr(model, projection_name).weight
weight.weight_loader(weight, torch.ones_like(weight, device="cpu"))
model.projection.weight.fill_(1)
output = model(torch.ones((1, 3, 3), device=self.device))
torch.testing.assert_close(
output, torch.full((1, 2), 9.0, device=self.device)
)
self.assertEqual(model.projection.weight.device.type, "cpu")
def test_v2_whitelist_preserves_cached_convolution_storage(self) -> None:
with torch.no_grad():
model = CachedConvolution("conv1d", self.device)
convolution = model.conv1d
convolution.bias = nn.Parameter(torch.zeros(3, device=self.device))
# Check V2's parameter selection before its CUDA-only stream transfers.
# The cached view is held outside the selected projection submodule.
with (
patch(
"sglang.srt.utils.offloader.get_parallel",
return_value=SimpleNamespace(dp_rank=0, num_dp_ranks=1),
),
patch("sglang.srt.utils.offloader.get_stream", return_value=None),
patch("torch.cuda.memory_allocated", return_value=0),
patch.dict(os.environ, {"SGLANG_RUN_ID": "test-cpu-offloader"}),
):
offloader = OffloaderV2(
group_size=1, num_in_group=1, prefetch_step=1, mode="cpu"
)
offloader.wrap_modules(
iter([model]),
submodule_accessor=lambda module: module.conv1d,
whitelist_param_names_creator=lambda module: ["weight", "bias"],
)
convolution.weight.weight_loader(
convolution.weight, torch.ones((3, 1, 3), device="cpu")
)
output = F.conv1d(
torch.ones((1, 3, 3), device=self.device),
model.attn.conv_weights.unsqueeze(1),
groups=3,
)
torch.testing.assert_close(
output, torch.full((1, 3, 1), 3.0, device=self.device)
)
self.assertEqual(convolution.weight.device.type, self.device.type)
self.assertEqual(convolution.bias.device.type, "cpu")
def test_loaders_rebuild_replaced_convolution_view(self) -> None:
for rebind in (
PreshardedModelLoader._rebind_parameter_aliases,
IpcModelLoader._rebuild_stale_views,
):
with self.subTest(loader=rebind.__qualname__), torch.no_grad():
model = CachedConvolution("conv1d", self.device)
model.conv1d.weight = nn.Parameter(torch.ones_like(model.conv1d.weight))
model.projection.weight.fill_(1)
rebind(model)
output = model(torch.ones((1, 3, 3), device=self.device))
torch.testing.assert_close(
output, torch.full((1, 2), 9.0, device=self.device)
)
if __name__ == "__main__":
unittest.main()