From c938b6e454903cd740ba59d6af96527fa9a3e045 Mon Sep 17 00:00:00 2001 From: Ke Bao Date: Sun, 11 Oct 2026 22:35:52 +0800 Subject: [PATCH] Build Mamba host pools from device buffer infos (#43596) --- .../sglang/srt/mem_cache/device_pool_info.py | 73 +++++++++++++++ .../hybrid_cache/hybrid_pool_assembler.py | 28 ++++-- python/sglang/srt/mem_cache/memory_pool.py | 28 ++++++ .../sglang/srt/mem_cache/pool_host/mamba.py | 60 ++++++------ .../kernels/ops/mamba/test_transfer_mamba.py | 91 +++++++++++++++++++ .../mem_cache/test_hicache_host_register.py | 2 +- .../test_hicache_mamba_slot_side_states.py | 5 + ...test_hicache_staged_write_back_dispatch.py | 9 ++ .../mem_cache/test_hybrid_pool_assembler.py | 2 +- .../unit/mem_cache/test_mem_pool_host.py | 33 ++++++- .../mem_cache/test_npu_mamba_async_layout.py | 5 + .../test_unified_hicache_regressions.py | 2 +- 12 files changed, 295 insertions(+), 43 deletions(-) create mode 100644 python/sglang/srt/mem_cache/device_pool_info.py diff --git a/python/sglang/srt/mem_cache/device_pool_info.py b/python/sglang/srt/mem_cache/device_pool_info.py new file mode 100644 index 0000000000..6872c71f19 --- /dev/null +++ b/python/sglang/srt/mem_cache/device_pool_info.py @@ -0,0 +1,73 @@ +"""Device buffer facts consumed by host assembly. + +The device provider enumerates persistent tensors. Host assembly owns capacity, +layer routing and transfers. This first consumer describes Mamba checkpoints. +""" + +from __future__ import annotations + +import msgspec +import torch + +from sglang.srt.mem_cache.hicache_storage import PoolName + + +class MambaStateBufferInfo(msgspec.Struct, frozen=True, kw_only=True): + """Persistent checkpoint tensors. Layer and checkpoint axes come first. + + Extra buffers share checkpoint IDs but have no layer axis. Speculative + intermediate tensors are work space and do not belong in host checkpoints. + """ + + temporal: torch.Tensor + conv: tuple[torch.Tensor, ...] + extra_checkpoint_buffers: tuple[torch.Tensor, ...] = () + + def validate(self, *, layer_ids: tuple[int, ...] | None = None) -> None: + shape = tuple(self.temporal.shape) + if self.temporal.ndim < 2 or not self.conv: + raise ValueError( + f"Mamba temporal requires [layer, checkpoint, ...] and conv buffers, " + f"got temporal.shape={shape}, conv count={len(self.conv)}" + ) + if layer_ids is not None and len(layer_ids) != shape[0]: + raise ValueError( + f"Mamba layer_ids={layer_ids} must match temporal.shape[0]={shape[0]}" + ) + for index, buffer in enumerate(self.conv): + if buffer.shape[:2] != self.temporal.shape[:2]: + raise ValueError( + f"Mamba conv[{index}].shape={tuple(buffer.shape)} must start " + f"with layer/checkpoint axes {shape[:2]}" + ) + if buffer.dtype != self.conv[0].dtype: + raise ValueError( + f"Mamba conv[{index}].dtype={buffer.dtype}, expected {self.conv[0].dtype}" + ) + for index, buffer in enumerate(self.extra_checkpoint_buffers): + if buffer.ndim < 1 or buffer.shape[0] != shape[1]: + raise ValueError( + f"Mamba extra_checkpoint_buffers[{index}].shape={tuple(buffer.shape)} " + f"must start with checkpoint capacity {shape[1]}" + ) + for buffer in (*self.conv, *self.extra_checkpoint_buffers): + if buffer.device != self.temporal.device: + raise ValueError( + f"Mamba buffer.device={buffer.device}, expected {self.temporal.device}" + ) + + +class DevicePoolInfo(msgspec.Struct, frozen=True, kw_only=True): + pool_name: PoolName + indices_from_pool: PoolName + # Model layer IDs, including the PP stage offset, in buffer-axis order. + layer_ids: tuple[int, ...] + buffer_info: MambaStateBufferInfo + + def __post_init__(self) -> None: + # A PP stage may own no checkpoint layers. Buffer validation decides + # whether an empty layer axis is valid for the physical format. + if len(set(self.layer_ids)) != len(self.layer_ids) or any( + layer < 0 for layer in self.layer_ids + ): + raise ValueError(f"{self.pool_name}: invalid model layers {self.layer_ids}") diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index 365fa94bc0..4aadb2a1a0 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -1104,6 +1104,22 @@ def build_deepseek_v4_hicache_stack( return host_pool_group, cache_controller +def build_mamba_host_pool(mamba_pool, *, host_size: float, layout: str): + (pool_info,) = mamba_pool.get_device_pool_infos() + pool_info.buffer_info.validate(layer_ids=pool_info.layer_ids) + capacity = mamba_pool.host_capacity_tokens + if capacity is None: + capacity = mamba_pool.size + return MambaPoolHost( + pool_info.buffer_info, + get_memory().hicache_ratio, + host_size, + device_capacity=capacity, + allocator_type=_get_allocator_type(), + layout=layout, + ) + + def build_hybrid_mamba_stack( *, params: CacheInitParams, @@ -1167,11 +1183,9 @@ def build_hybrid_mamba_stack( if get_memory().hicache_mem_layout == "page_first_kv_split" else get_memory().hicache_mem_layout ) - mamba_host_pool = MambaPoolHost( + mamba_host_pool = build_mamba_host_pool( mamba_pool, - get_memory().hicache_ratio, - mamba_host_size, - allocator_type=_get_allocator_type(), + host_size=mamba_host_size, layout=mamba_layout, ) entries = _build_pool_entries(config=config, root_host_pool=kv_host_pool) + [ @@ -1281,11 +1295,9 @@ def build_hybrid_mamba_swa_stack( host_size=swa_host_size, pool_label="swa", ) - mamba_host_pool = MambaPoolHost( + mamba_host_pool = build_mamba_host_pool( mamba_pool, - get_memory().hicache_ratio, - mamba_host_size, - allocator_type=_get_allocator_type(), + host_size=mamba_host_size, layout=get_memory().hicache_mem_layout, ) entries = [ diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 3aa56b5fad..bbec4bc42a 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -100,6 +100,7 @@ from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter if TYPE_CHECKING: from sglang.srt.managers.cache_controller import LayerDoneCounter from sglang.srt.managers.schedule_batch import Req + from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo logger = logging.getLogger(__name__) @@ -500,6 +501,8 @@ class ReqToTokenPool: class MambaPool: + host_capacity_tokens: Optional[int] = None + # Axis of each two-dimensional conv state that represents the sliding window. # Upstream states use (dim, K-1); subclasses may preserve another layout. conv_window_axis = -1 @@ -1066,6 +1069,31 @@ class MambaPool: self.conv_slice_axis = getattr(cache_params.shape, "conv_slice_axis", 0) self._warmup_fused_copy_slot_kernel() + def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]: + """Checkpoint buffers for host transfer, excluding speculative work space.""" + from sglang.srt.mem_cache.device_pool_info import ( + DevicePoolInfo, + MambaStateBufferInfo, + ) + from sglang.srt.mem_cache.hicache_storage import PoolName + + return ( + DevicePoolInfo( + pool_name=PoolName.MAMBA, + indices_from_pool=PoolName.MAMBA, + layer_ids=tuple(self.mamba_layer_ids), + buffer_info=MambaStateBufferInfo( + temporal=self.mamba_cache.temporal, + conv=tuple(self.mamba_cache.conv), + extra_checkpoint_buffers=tuple( + buffer + for sibling in self._slot_siblings + for _, buffer, _, _ in sibling.iter_transfer_state_entries() + ), + ), + ), + ) + def get_speculative_mamba2_params_all_layers(self) -> SpeculativeState: assert isinstance(self.mamba_cache, self.SpeculativeState) return self.mamba_cache diff --git a/python/sglang/srt/mem_cache/pool_host/mamba.py b/python/sglang/srt/mem_cache/pool_host/mamba.py index 118247fd35..b08bfe8250 100644 --- a/python/sglang/srt/mem_cache/pool_host/mamba.py +++ b/python/sglang/srt/mem_cache/pool_host/mamba.py @@ -8,7 +8,7 @@ from typing import Optional import numpy as np import torch -from sglang.srt.mem_cache.memory_pool import MambaPool +from sglang.srt.mem_cache.device_pool_info import MambaStateBufferInfo from sglang.srt.mem_cache.pool_host.base import ( HostKVCache, host_memory_budget_bytes, @@ -79,15 +79,20 @@ class MambaPoolHost(HostKVCache): def __init__( self, - device_pool: MambaPool, + buffer_info: MambaStateBufferInfo, host_to_device_ratio: float, host_size: int, + *, + device_capacity: int, pin_memory: bool = True, device: str = "cpu", allocator_type: str = "default", layout: str = "layer_first", ): - self.device_pool = device_pool + buffer_info.validate() + self.buffer_info = buffer_info + self.device_pool = None + self._buffer_device = buffer_info.temporal.device self.page_size = 1 assert layout in [ @@ -99,32 +104,25 @@ class MambaPoolHost(HostKVCache): self.pin_memory = pin_memory self.device = device self.allocator = get_allocator_from_storage(allocator_type) - self.num_mamba_layers = device_pool.num_mamba_layers + self.num_mamba_layers = buffer_info.temporal.shape[0] self.conv_state_shapes = [ - conv_state.shape[2:] for conv_state in device_pool.mamba_cache.conv + conv_state.shape[2:] for conv_state in self.buffer_info.conv ] - self.temporal_state_shape = device_pool.mamba_cache.temporal.shape[2:] + self.temporal_state_shape = self.buffer_info.temporal.shape[2:] self.temporal_state_elem_size = int(np.prod(self.temporal_state_shape)) self.conv_state_elem_sizes = [ int(np.prod(conv_shape)) for conv_shape in self.conv_state_shapes ] - self.conv_dtype = device_pool.mamba_cache.conv[0].dtype - self.temporal_dtype = device_pool.mamba_cache.temporal.dtype + self.conv_dtype = self.buffer_info.conv[0].dtype + self.temporal_dtype = self.buffer_info.temporal.dtype self.dtype = self.conv_dtype # Registered side states share the checkpoint slot with conv/temporal, # so a host round trip that skips them leaves the previous occupant's rows. - self.slot_state_device_tensors = [ - state - for sibling in device_pool._slot_siblings - for _, state, _, _ in sibling.iter_transfer_state_entries() - ] + self.slot_state_device_tensors = list(buffer_info.extra_checkpoint_buffers) self.slot_state_buffers = [] self.size_per_token = self.get_size_per_token() - device_capacity = getattr(device_pool, "host_capacity_tokens", None) - if device_capacity is None: - device_capacity = device_pool.size if host_size > 0: self.size = sync_fixed_hicache_size( int(host_size * 1e9 // self.size_per_token), host_size @@ -161,19 +159,19 @@ class MambaPoolHost(HostKVCache): self.temporal_device_ptrs = torch.tensor( [ - device_pool.mamba_cache.temporal[i].data_ptr() + self.buffer_info.temporal[i].data_ptr() for i in range(self.num_mamba_layers) ], dtype=torch.uint64, - device=self.device_pool.device, + device=self._buffer_device, ) self.conv_device_ptrs = [ torch.tensor( [conv_state[i].data_ptr() for i in range(self.num_mamba_layers)], dtype=torch.uint64, - device=self.device_pool.device, + device=self._buffer_device, ) - for conv_state in device_pool.mamba_cache.conv + for conv_state in self.buffer_info.conv ] self.kv_buffer = self.init_kv_buffer() @@ -208,7 +206,7 @@ class MambaPoolHost(HostKVCache): logger.info("NPU HiCache Mamba state transfer mode: native async.") def init_kv_buffer(self): - _host_alloc = ALLOC_MEMORY_FUNCS[self.device_pool.device] + _host_alloc = ALLOC_MEMORY_FUNCS[self._buffer_device.type] def alloc_func(dims, *, dtype, device, pin_memory, allocator): # conv-only linear attention has no ssm state: mmap can't map the @@ -662,7 +660,7 @@ class MambaPoolHost(HostKVCache): # transfer errors, same guard as the per-layer path below if self.temporal_state_elem_size > 0: transfer_mamba_state( - device_buf=device_pool.mamba_cache.temporal, + device_buf=self.buffer_info.temporal, host_buf=self.temporal_buffer, device_indices=device_indices, host_indices=host_indices, @@ -670,7 +668,7 @@ class MambaPoolHost(HostKVCache): ) for conv_idx in range(len(self.conv_state_shapes)): transfer_mamba_state( - device_buf=device_pool.mamba_cache.conv[conv_idx], + device_buf=self.buffer_info.conv[conv_idx], host_buf=self.conv_buffer[conv_idx], device_indices=device_indices, host_indices=host_indices, @@ -681,7 +679,7 @@ class MambaPoolHost(HostKVCache): if self.temporal_state_elem_size > 0: self._copy_tensor_pf_lf( src=self.temporal_buffer, - dst=device_pool.mamba_cache.temporal[layer_id], + dst=self.buffer_info.temporal[layer_id], src_indices=host_indices, dst_indices=device_indices, layer_id=layer_id, @@ -691,7 +689,7 @@ class MambaPoolHost(HostKVCache): for conv_idx in range(len(self.conv_state_shapes)): self._copy_tensor_pf_lf( src=self.conv_buffer[conv_idx], - dst=device_pool.mamba_cache.conv[conv_idx][layer_id], + dst=self.buffer_info.conv[conv_idx][layer_id], src_indices=host_indices, dst_indices=device_indices, layer_id=layer_id, @@ -701,7 +699,7 @@ class MambaPoolHost(HostKVCache): else: self._copy_tensor( self.temporal_buffer[layer_id], - device_pool.mamba_cache.temporal[layer_id], + self.buffer_info.temporal[layer_id], host_indices, device_indices, io_backend, @@ -709,7 +707,7 @@ class MambaPoolHost(HostKVCache): for conv_idx in range(len(self.conv_state_shapes)): self._copy_tensor( self.conv_buffer[conv_idx][layer_id], - device_pool.mamba_cache.conv[conv_idx][layer_id], + self.buffer_info.conv[conv_idx][layer_id], host_indices, device_indices, io_backend, @@ -727,7 +725,7 @@ class MambaPoolHost(HostKVCache): # no ssm state on conv-only models: a 0-size batched memcpy errors if self.temporal_state_elem_size > 0: self._copy_tensor_all_layers_lf_pf( - src_layers=device_pool.mamba_cache.temporal, + src_layers=self.buffer_info.temporal, dst=self.temporal_buffer, src_indices=device_indices, dst_indices=host_indices, @@ -739,7 +737,7 @@ class MambaPoolHost(HostKVCache): ) for conv_idx in range(len(self.conv_state_shapes)): self._copy_tensor_all_layers_lf_pf( - src_layers=device_pool.mamba_cache.conv[conv_idx], + src_layers=self.buffer_info.conv[conv_idx], dst=self.conv_buffer[conv_idx], src_indices=device_indices, dst_indices=host_indices, @@ -752,7 +750,7 @@ class MambaPoolHost(HostKVCache): else: for layer_id in range(self.num_mamba_layers): self._copy_tensor( - device_pool.mamba_cache.temporal[layer_id], + self.buffer_info.temporal[layer_id], self.temporal_buffer[layer_id], device_indices, host_indices, @@ -760,7 +758,7 @@ class MambaPoolHost(HostKVCache): ) for conv_idx in range(len(self.conv_state_shapes)): self._copy_tensor( - device_pool.mamba_cache.conv[conv_idx][layer_id], + self.buffer_info.conv[conv_idx][layer_id], self.conv_buffer[conv_idx][layer_id], device_indices, host_indices, diff --git a/test/registered/kernels/ops/mamba/test_transfer_mamba.py b/test/registered/kernels/ops/mamba/test_transfer_mamba.py index ff552f30fb..384ec8d42b 100644 --- a/test/registered/kernels/ops/mamba/test_transfer_mamba.py +++ b/test/registered/kernels/ops/mamba/test_transfer_mamba.py @@ -12,6 +12,8 @@ from types import SimpleNamespace import pytest import torch +from sglang.srt.mem_cache.device_pool_info import MambaStateBufferInfo +from sglang.srt.mem_cache.memory_pool import MambaPool from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci @@ -88,6 +90,10 @@ def make_host_pool(dtype, layout): # Device pointers (needed for backup kernel path) device_pool = make_device_pool(dtype) host.device_pool = device_pool + host.buffer_info = MambaStateBufferInfo( + temporal=device_pool.mamba_cache.temporal, + conv=tuple(device_pool.mamba_cache.conv), + ) host.temporal_device_ptrs = torch.tensor( [device_pool.mamba_cache.temporal[i].data_ptr() for i in range(NUM_LAYERS)], dtype=torch.uint64, @@ -345,5 +351,90 @@ def test_mamba_kernel_full_indices(dtype, layout): assert_device_matches_host(host, device_pool, host_indices, load_indices) +@pytest.mark.parametrize( + "layout,io_backend", + [ + ("page_first", "kernel"), + ("page_first_direct", "kernel"), + ("page_first_direct", "direct"), + ], +) +@pytest.mark.parametrize("kind", ["dense", "strided", "conv_only"]) +def test_captured_mamba_state_round_trip(layout, io_backend, kind): + """Transfer must preserve checkpoint bytes without reading the device pool. + + Unified state is strided and short-conv-only models have no temporal payload. + Both still carry registered extra checkpoint rows through the same restore. + """ + pool = object.__new__(MambaPool) + pool.mamba_layer_ids = [7, 11] + if kind == "strided": + temporal = torch.empty(6, 2, 64, device=DEVICE).transpose(0, 1) + conv = torch.empty(6, 2, 64, 4, device=DEVICE).transpose(0, 1) + else: + width = 0 if kind == "conv_only" else 64 + temporal = torch.empty(2, 6, width, device=DEVICE) + conv = torch.empty(2, 6, 64, 4, device=DEVICE) + extra = torch.arange(6 * 8, device=DEVICE).reshape(6, 8) + sibling = SimpleNamespace( + iter_transfer_state_entries=lambda: iter((("extra", extra, None, None),)) + ) + pool._slot_siblings = (sibling,) + pool.mamba_cache = MambaPool.State(conv=[conv], temporal=temporal) + (info,) = pool.get_device_pool_infos() + info.buffer_info.validate(layer_ids=info.layer_ids) + host = MambaPoolHost( + info.buffer_info, + host_to_device_ratio=2, + host_size=0, + device_capacity=6, + layout=layout, + ) + try: + assert host.size == 13 + del pool.mamba_cache + del pool._slot_siblings + assert host.device_pool is None + + index_device = "cpu" if io_backend == "direct" else DEVICE + device_indices = torch.tensor([1, 4], device=index_device) + host_indices = torch.tensor( + [0, 2], device="cpu" if io_backend == "direct" else DEVICE + ) + buffers = (temporal, conv) + for buffer in buffers: + buffer.copy_( + torch.arange(buffer.numel(), device=DEVICE).reshape(buffer.shape) + ) + expected = [buffer[:, device_indices].clone() for buffer in buffers] + extra_expected = extra[device_indices].clone() + # Controllers run transfers on a dedicated stream. CUDA batch copies + # reject the legacy default stream used by ordinary tensor operations. + transfer_stream = torch.cuda.Stream() + transfer_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(transfer_stream): + host.backup_from_device_all_layer( + pool, host_indices, device_indices, io_backend + ) + transfer_stream.synchronize() + for buffer in buffers: + buffer.zero_() + extra.zero_() + transfer_stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(transfer_stream): + for layer in range(2): + host.load_to_device_per_layer( + pool, host_indices, device_indices, layer, io_backend + ) + transfer_stream.synchronize() + for buffer, original in zip(buffers, expected): + assert torch.equal(buffer[:, device_indices], original) + assert not torch.count_nonzero(buffer[:, [0, 2, 3, 5]]) + assert torch.equal(extra[device_indices], extra_expected) + assert not torch.count_nonzero(extra[[0, 2, 3, 5]]) + finally: + host.destroy() + + if __name__ == "__main__": sys.exit(pytest.main([__file__, "-v", "-s"])) diff --git a/test/registered/unit/mem_cache/test_hicache_host_register.py b/test/registered/unit/mem_cache/test_hicache_host_register.py index a7f74cbc13..7712664be1 100644 --- a/test/registered/unit/mem_cache/test_hicache_host_register.py +++ b/test/registered/unit/mem_cache/test_hicache_host_register.py @@ -161,7 +161,7 @@ class TestHiCacheHostRegister(unittest.TestCase): pool.conv_state_shapes = [(7,), (2, 2)] pool.temporal_dtype = torch.float16 pool.conv_dtype = torch.float32 - pool.device_pool = SimpleNamespace(device="cuda") + pool._buffer_device = torch.device("cuda") pool.device = "cpu" pool.pin_memory = True pool.allocator = object() diff --git a/test/registered/unit/mem_cache/test_hicache_mamba_slot_side_states.py b/test/registered/unit/mem_cache/test_hicache_mamba_slot_side_states.py index c48cc24c39..587f0f6043 100644 --- a/test/registered/unit/mem_cache/test_hicache_mamba_slot_side_states.py +++ b/test/registered/unit/mem_cache/test_hicache_mamba_slot_side_states.py @@ -19,6 +19,7 @@ from unittest import mock import torch +from sglang.srt.mem_cache.device_pool_info import MambaStateBufferInfo from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -78,6 +79,10 @@ def _make_pools(with_side_state: bool): host = MambaPoolHost.__new__(MambaPoolHost) host.device_pool = device_pool + host.buffer_info = MambaStateBufferInfo( + temporal=device_pool.mamba_cache.temporal, + conv=tuple(device_pool.mamba_cache.conv), + ) host.page_size = 1 host.layout = "page_first" host.device = "cpu" diff --git a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py index 25e4780d28..338bd416fb 100644 --- a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py +++ b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py @@ -12,6 +12,7 @@ from sglang.srt.managers.cache_controller import CacheOperation, HiCacheControll from sglang.srt.mem_cache import l2_transfer as transfer_module from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline +from sglang.srt.mem_cache.device_pool_info import MambaStateBufferInfo from sglang.srt.mem_cache.hicache_storage import ( PoolHitPolicy, PoolName, @@ -742,6 +743,10 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): expected_conv = device_pool.mamba_cache.conv[0][:, device_indices].clone() host = MambaPoolHost.__new__(MambaPoolHost) + host.buffer_info = MambaStateBufferInfo( + temporal=device_pool.mamba_cache.temporal, + conv=tuple(device_pool.mamba_cache.conv), + ) host.layout = "page_first" host.num_mamba_layers = num_layers host.device_pool = SimpleNamespace(device="cpu") @@ -829,6 +834,10 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): expected_conv = device_pool.mamba_cache.conv[0][:, device_indices].clone() host = MambaPoolHost.__new__(MambaPoolHost) + host.buffer_info = MambaStateBufferInfo( + temporal=device_pool.mamba_cache.temporal, + conv=tuple(device_pool.mamba_cache.conv), + ) host.layout = "page_first_direct" host.num_mamba_layers = num_layers host.temporal_state_elem_size = 3 diff --git a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py index 00e8b603a2..264dc62580 100644 --- a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py @@ -798,7 +798,7 @@ class TestHybridMambaDeclaredIndexer(CustomTestCase): patch.object(pool_host_dsa, "DSAIndexerPoolHost", dummy_indexer_host), patch.object( hybrid_pool_assembler, - "MambaPoolHost", + "build_mamba_host_pool", return_value=SimpleNamespace(layer_num=2, can_use_write_back_jit=False), ), patch.object(hybrid_pool_assembler, "HybridCacheController", MagicMock()), diff --git a/test/registered/unit/mem_cache/test_mem_pool_host.py b/test/registered/unit/mem_cache/test_mem_pool_host.py index 005b771232..8dae448426 100644 --- a/test/registered/unit/mem_cache/test_mem_pool_host.py +++ b/test/registered/unit/mem_cache/test_mem_pool_host.py @@ -9,7 +9,11 @@ from types import SimpleNamespace import torch from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer -from sglang.srt.mem_cache.memory_pool import MHATokenToKOnlyPool, MHATokenToKVPool +from sglang.srt.mem_cache.memory_pool import ( + MambaPool, + MHATokenToKOnlyPool, + MHATokenToKVPool, +) from sglang.srt.mem_cache.memory_pool_host import ( DeepSeekV4PagedHostPool, LogicalHostPool, @@ -278,6 +282,33 @@ class TestLazyHostPoolRelease(CustomTestCase): def test_mamba_pool_lazy_release(self): self._assert_lazy_release(self._make_mamba_pool()) + def test_mamba_host_preserves_empty_pp_stage(self): + pool = MambaPool.__new__(MambaPool) + pool.mamba_layer_ids = [] + pool.mamba_cache = SimpleNamespace( + temporal=torch.empty((0, 5, 2, 2)), + conv=[torch.empty((0, 5, 3, 2))], + ) + pool._slot_siblings = [] + (info,) = pool.get_device_pool_infos() + info.buffer_info.validate(layer_ids=info.layer_ids) + host = MambaPoolHost( + info.buffer_info, + host_to_device_ratio=2, + host_size=0, + device_capacity=4, + layout="page_first", + pin_memory=False, + ) + try: + self.assertEqual(info.layer_ids, ()) + self.assertEqual(host.num_mamba_layers, 0) + self.assertEqual(host.size_per_token, 0) + self.assertEqual(host.size, 9) + self.assertEqual(host.kv_buffer, []) + finally: + host.destroy() + def test_deepseek_v4_pool_lazy_release(self): pool = self._make_deepseek_v4_pool() self._assert_lazy_release(pool) diff --git a/test/registered/unit/mem_cache/test_npu_mamba_async_layout.py b/test/registered/unit/mem_cache/test_npu_mamba_async_layout.py index 01b05bd960..0bf91c8afd 100644 --- a/test/registered/unit/mem_cache/test_npu_mamba_async_layout.py +++ b/test/registered/unit/mem_cache/test_npu_mamba_async_layout.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock, patch import torch import sglang.srt.mem_cache.pool_host.mamba as mamba_pool_host +from sglang.srt.mem_cache.device_pool_info import MambaStateBufferInfo from sglang.srt.mem_cache.pool_host.mamba import ( MambaPoolHost, _npu_hicache_mamba_io_mode, @@ -173,6 +174,10 @@ class TestNPUMambaAsyncConfig(unittest.TestCase): ) ) + pool.buffer_info = MambaStateBufferInfo( + temporal=device_pool.mamba_cache.temporal, + conv=tuple(device_pool.mamba_cache.conv), + ) pool.load_to_device_per_layer( device_pool=device_pool, host_indices=torch.tensor([1]), diff --git a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py index 3f6fb68119..245a03a450 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py @@ -186,7 +186,7 @@ class TestTriPoolAssembly(unittest.TestCase): assembler, "_get_allocator_type", return_value="default" ), patch.object(assembler, "build_kv_host_pool", return_value=host), - patch.object(assembler, "MambaPoolHost", return_value=host), + patch.object(assembler, "build_mamba_host_pool", return_value=host), patch.object(assembler, "HybridCacheController"), ): group, _ = assembler.build_hybrid_mamba_swa_stack(