From 5606fb8592d37e96fa4f795437958565d9c332df Mon Sep 17 00:00:00 2001 From: JarJarBeatyourattitude <62723106+JarJarBeatyourattitude@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:15:59 -0700 Subject: [PATCH] [Fix] HiCache: carry registered Mamba slot side states (Qwen4-Exp PLE) through the host tier (#39862) Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> Co-authored-by: alphabetc1 <2508695655@qq.com> Co-authored-by: ispobock --- python/sglang/srt/mem_cache/memory_pool.py | 11 + .../sglang/srt/mem_cache/pool_host/mamba.py | 96 +++++++- .../test_hicache_mamba_slot_side_states.py | 217 ++++++++++++++++++ 3 files changed, 321 insertions(+), 3 deletions(-) create mode 100644 test/registered/unit/mem_cache/test_hicache_mamba_slot_side_states.py diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index fd8d4c8b74..260c4ad9dd 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -1562,6 +1562,17 @@ class HybridReqToTokenPool(ReqToTokenPool): return self.short_conv_pool.layer_intermediate_cache(layer_id) def get_ngram_context(self, ngram_indices: torch.Tensor) -> torch.Tensor: + # Read once per forward, BEFORE the decoder-layer loop, so unlike + # short_conv_layer_cache it has no per-layer barrier of its own. The + # host tier restores side state on the first Mamba layer's transfer, so + # wait for that layer before reading the history. + if ( + self.layer_transfer_counter is not None + and self.layer_transfer_counter.consumer_index >= 0 + and self.mamba_map + ): + first_mamba_layer = min(self.mamba_map) + self.layer_transfer_counter.wait_until(first_mamba_layer - self.start_layer) return self.ngram_pool.get_context(ngram_indices) def set_ngram_context( diff --git a/python/sglang/srt/mem_cache/pool_host/mamba.py b/python/sglang/srt/mem_cache/pool_host/mamba.py index 79accc17e4..118247fd35 100644 --- a/python/sglang/srt/mem_cache/pool_host/mamba.py +++ b/python/sglang/srt/mem_cache/pool_host/mamba.py @@ -72,6 +72,11 @@ def _npu_hicache_mamba_io_mode() -> str: class MambaPoolHost(HostKVCache): + # Class-level defaults so a model without side state, and the test stubs that + # skip __init__, still read an empty pair rather than raising. + slot_state_device_tensors: tuple = () + slot_state_buffers: tuple = () + def __init__( self, device_pool: MambaPool, @@ -107,6 +112,14 @@ class MambaPoolHost(HostKVCache): self.conv_dtype = device_pool.mamba_cache.conv[0].dtype self.temporal_dtype = device_pool.mamba_cache.temporal.dtype self.dtype = self.conv_dtype + # Registered side states share the checkpoint slot with conv/temporal, + # so a host round trip that skips them leaves the previous occupant's rows. + self.slot_state_device_tensors = [ + state + for sibling in device_pool._slot_siblings + for _, state, _, _ in sibling.iter_transfer_state_entries() + ] + self.slot_state_buffers = [] self.size_per_token = self.get_size_per_token() device_capacity = getattr(device_pool, "host_capacity_tokens", None) @@ -264,11 +277,29 @@ class MambaPoolHost(HostKVCache): allocator=self.allocator, ) ) + # Side state keeps its own dtype and its own slot axis (axis 0 of the + # per-layer view the device pool yields). + self.slot_state_buffers = [ + alloc_func( + (self.size,) + tuple(state.shape[1:]), + dtype=state.dtype, + device=self.device, + pin_memory=self.pin_memory, + allocator=self.allocator, + ) + for state in self.slot_state_device_tensors + ] # destroy() unregisters via kv_buffer; without this list the pinned # registrations leak past the buffers' mmap. 0-element buffers # (conv-only models' temporal state) were never registered. return [ - buf for buf in (self.temporal_buffer, *self.conv_buffer) if buf.numel() > 0 + buf + for buf in ( + self.temporal_buffer, + *self.conv_buffer, + *self.slot_state_buffers, + ) + if buf.numel() > 0 ] def _init_write_back_staging_buffers(self): @@ -284,7 +315,7 @@ class MambaPoolHost(HostKVCache): def get_hybrid_pool_buffer(self): # Expose all mamba host tensors that need Mooncake buffer registration. - return [self.temporal_buffer, *self.conv_buffer] + return [self.temporal_buffer, *self.conv_buffer, *self.slot_state_buffers] def _iter_page_tensors(self, index: int): if self.layout in ["page_first", "page_first_direct"]: @@ -295,6 +326,8 @@ class MambaPoolHost(HostKVCache): yield self.temporal_buffer[:, index : index + self.page_size] for conv_buf in self.conv_buffer: yield conv_buf[:, index : index + self.page_size] + for state_buf in self.slot_state_buffers: + yield state_buf[index : index + self.page_size] @staticmethod def _flatten_tensor_bytes(tensor: torch.Tensor) -> torch.Tensor: @@ -343,7 +376,13 @@ class MambaPoolHost(HostKVCache): for conv_elem_size in self.conv_state_elem_sizes ) temporal_size = self.temporal_state_elem_size * self.temporal_dtype.itemsize - return (conv_total_size + temporal_size) * self.num_mamba_layers + # Side-state rows are per-slot, not per-layer: each entry is already a + # single layer's view, so its own row size is added once. + side_size = sum( + state[0].numel() * state.element_size() + for state in self.slot_state_device_tensors + ) + return (conv_total_size + temporal_size) * self.num_mamba_layers + side_size def get_ksize_per_token(self): return self.get_size_per_token() @@ -609,6 +648,11 @@ class MambaPoolHost(HostKVCache): *, is_draft: bool = False, ): + # Side state is not per-layer, so it rides on the first layer's call -- + # before that layer's completion event, which is what every reader of a + # restored slot waits on. + if layer_id == 0: + self._transfer_slot_states(host_indices, device_indices, to_device=True) if self.layout in ["page_first", "page_first_direct"]: if io_backend == "kernel_ascend" and transfer_mamba_state is not None: # NPU: transfer all layers at once via dedicated kernel. @@ -722,6 +766,30 @@ class MambaPoolHost(HostKVCache): host_indices, io_backend, ) + self._transfer_slot_states(host_indices, device_indices, to_device=False) + + def _transfer_slot_states(self, host_indices, device_indices, *, to_device): + """Copy the registered side-state rows for these slots, one leg or the + other. Blocking copies: the rows are a few hundred KB per checkpoint and + correctness of the ordering against the completion event matters more + than the microseconds.""" + if not self.slot_state_device_tensors: + return + for state, host_buf in zip( + self.slot_state_device_tensors, self.slot_state_buffers + ): + dev_idx = device_indices.reshape(-1).to( + device=state.device, dtype=torch.long + ) + host_idx = host_indices.reshape(-1).to( + device=host_buf.device, dtype=torch.long + ) + if to_device: + values = host_buf.index_select(0, host_idx) + state.index_copy_(0, dev_idx, values.to(state.device)) + else: + values = state.index_select(0, dev_idx) + host_buf.index_copy_(0, host_idx, values.to(host_buf.device)) def get_data_page(self, index, flat: bool = True) -> torch.Tensor: data_page = torch.cat( @@ -746,11 +814,24 @@ class MambaPoolHost(HostKVCache): data_page: torch.Tensor, ) -> None: flat_bytes = data_page.contiguous().view(torch.uint8).reshape(-1) + expected = self.page_size * self.size_per_token + if self.slot_state_device_tensors and flat_bytes.numel() != expected: + # A page stored by a build without the side state is shorter; say so + # instead of failing inside a reshape halfway through the restore. + raise ValueError( + f"Mamba state page has {flat_bytes.numel()} bytes, expected " + f"{expected}: the page predates registered slot side states." + ) start = 0 for tensor in self._iter_page_tensors(index): num_bytes = tensor.numel() * tensor.element_size() tensor_bytes = flat_bytes[start : start + num_bytes] start += num_bytes + if tensor_bytes.storage_offset() % tensor.element_size(): + # With side state the page holds components of different element + # sizes, so a later component can begin at a byte offset its own + # dtype cannot be viewed at. A contiguous copy re-bases it at 0. + tensor_bytes = tensor_bytes.clone() restored = tensor_bytes.view(dtype=tensor.dtype).reshape(tensor.shape) tensor.copy_(restored) @@ -760,6 +841,11 @@ class MambaPoolHost(HostKVCache): Only page-first layouts are supported for mamba storage zero-copy because each page slot in temporal/conv buffers is directly addressable. """ + if self.slot_state_device_tensors: + raise NotImplementedError( + "Mamba storage zero-copy does not carry registered slot side " + "states; use a whole-page storage backend." + ) assert len(indices) % self.page_size == 0 if self.layout not in ["page_first", "page_first_direct"]: raise ValueError( @@ -817,6 +903,10 @@ class MambaPoolHost(HostKVCache): return ptr_list, element_size_list def is_stride_page_aligned(self, page_size_bytes: int = 4096) -> bool: + if self.slot_state_device_tensors: + # Side-state rows are a second, differently-strided buffer: a page is + # no longer one contiguous slice, so no zero-copy claim is made. + return False if self.layout not in ["page_first", "page_first_direct"]: return False temporal_stride = ( diff --git a/test/registered/unit/mem_cache/test_hicache_mamba_slot_side_states.py b/test/registered/unit/mem_cache/test_hicache_mamba_slot_side_states.py new file mode 100644 index 0000000000..c48cc24c39 --- /dev/null +++ b/test/registered/unit/mem_cache/test_hicache_mamba_slot_side_states.py @@ -0,0 +1,217 @@ +"""HiCache: a Mamba slot's registered side states must survive the host round trip. + +`MambaPool.register_slot_state` attaches state that lives in the same physical +checkpoint slot as conv/temporal but is neither per-layer nor part of +`mamba_cache` (Qwen4-Exp registers a PLE short-conv window and a PLE N-gram +token history this way). Every other mover of a slot carries it -- `copy_from`, +`clear_slots`, `get_cpu_copy`/`load_cpu_copy`, and the RDMA registration in +`_iter_transfer_state_entries`. Before this test the HiCache host tier did not, +so a slot restored from host kept whatever the previous occupant left in those +rows and the model conditioned on another request's tokens. + +The copy kernels are CUDA-only, so the two conv/temporal helpers are stubbed and +the side-state legs are exercised at their real call sites on CPU tensors. +""" + +import unittest +from types import SimpleNamespace +from unittest import mock + +import torch + +from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +NUM_LAYERS = 2 +NUM_DEVICE_SLOTS = 4 +NUM_HOST_SLOTS = 4 +CONV_SHAPE = (3, 2) +TEMPORAL_SHAPE = (2, 2) +SIDE_CONV_SHAPE = (5, 3) +NGRAM_WIDTH = 2 + + +class _RegisteredSideState: + """Stands in for ple_state_pool.ShortConvPool / NGramPool: the only thing the + host pool may assume is the transfer-entry protocol, whose tensors carry the + slot on axis 0.""" + + def __init__(self, name, tensor): + self.name = name + self.tensor = tensor + + def iter_transfer_state_entries(self): + yield self.name, self.tensor, None, 0 + + +def _make_pools(with_side_state: bool): + conv = [ + torch.zeros((NUM_LAYERS, NUM_DEVICE_SLOTS) + CONV_SHAPE, dtype=torch.bfloat16) + ] + temporal = torch.zeros( + (NUM_LAYERS, NUM_DEVICE_SLOTS) + TEMPORAL_SHAPE, dtype=torch.bfloat16 + ) + siblings = () + if with_side_state: + siblings = ( + _RegisteredSideState( + "ple_short_conv", + torch.zeros( + (NUM_DEVICE_SLOTS,) + SIDE_CONV_SHAPE, dtype=torch.bfloat16 + ), + ), + _RegisteredSideState( + "ple_ngram", + torch.zeros((NUM_DEVICE_SLOTS, NGRAM_WIDTH), dtype=torch.int64), + ), + ) + device_pool = SimpleNamespace( + mamba_cache=SimpleNamespace(conv=conv, temporal=temporal), + num_mamba_layers=NUM_LAYERS, + size=NUM_DEVICE_SLOTS, + device="cpu", + _slot_siblings=siblings, + ) + + host = MambaPoolHost.__new__(MambaPoolHost) + host.device_pool = device_pool + host.page_size = 1 + host.layout = "page_first" + host.device = "cpu" + host.pin_memory = False + host.num_mamba_layers = NUM_LAYERS + host.size = NUM_HOST_SLOTS + host.conv_state_shapes = [c.shape[2:] for c in conv] + host.temporal_state_shape = temporal.shape[2:] + host.temporal_state_elem_size = int(temporal[0, 0].numel()) + host.conv_state_elem_sizes = [int(c[0, 0].numel()) for c in conv] + host.conv_dtype = conv[0].dtype + host.temporal_dtype = temporal.dtype + host.dtype = host.conv_dtype + host.slot_state_device_tensors = [ + state + for sibling in device_pool._slot_siblings + for _, state, _, _ in sibling.iter_transfer_state_entries() + ] + host.slot_state_buffers = [ + torch.zeros((NUM_HOST_SLOTS,) + tuple(state.shape[1:]), dtype=state.dtype) + for state in host.slot_state_device_tensors + ] + host.temporal_buffer = torch.zeros( + (NUM_HOST_SLOTS, NUM_LAYERS, 1) + TEMPORAL_SHAPE, dtype=temporal.dtype + ) + host.conv_buffer = [ + torch.zeros((NUM_HOST_SLOTS, NUM_LAYERS, 1) + CONV_SHAPE, dtype=conv[0].dtype) + ] + host._init_write_back_staging_buffers() + # The kernel path takes raw device pointers; the stubbed copies never + # dereference them, but the real methods read the attributes. + host.temporal_device_ptrs = torch.zeros(NUM_LAYERS, dtype=torch.uint64) + host.conv_device_ptrs = [torch.zeros(NUM_LAYERS, dtype=torch.uint64) for _ in conv] + host.size_per_token = host.get_size_per_token() + return host, device_pool + + +def _stub_conv_and_temporal_copies(): + """The conv/temporal legs are CUDA kernels; this test is about the side state.""" + return ( + mock.patch.object( + MambaPoolHost, "_copy_tensor_pf_lf", staticmethod(lambda **kw: None) + ), + mock.patch.object( + MambaPoolHost, + "_copy_tensor_all_layers_lf_pf", + staticmethod(lambda **kw: None), + ), + ) + + +class TestHiCacheMambaSlotSideStates(CustomTestCase): + def test_size_per_token_accounts_for_side_state(self): + with_side, _ = _make_pools(True) + without_side, _ = _make_pools(False) + side_bytes = ( + SIDE_CONV_SHAPE[0] * SIDE_CONV_SHAPE[1] * 2 # bfloat16 + + NGRAM_WIDTH * 8 # int64 + ) + self.assertEqual( + with_side.get_size_per_token(), + without_side.get_size_per_token() + side_bytes, + ) + + def test_side_state_survives_the_host_round_trip(self): + host, device_pool = _make_pools(True) + short_conv, ngram = (s.tensor for s in device_pool._slot_siblings) + + checkpoint_slot = torch.tensor([1]) + host_slot = torch.tensor([2]) + other_slot = torch.tensor([3]) + + short_conv[checkpoint_slot] = 1.5 + ngram[checkpoint_slot] = torch.tensor([[11, 321]]) + # Another request owned the destination slot before the restore. + short_conv[other_slot] = -7.0 + ngram[other_slot] = torch.tensor([[310, 279]]) + + pf_lf, lf_pf = _stub_conv_and_temporal_copies() + with pf_lf, lf_pf: + host.backup_from_device_all_layer( + device_pool, host_slot, checkpoint_slot, "direct" + ) + host.load_to_device_per_layer( + device_pool, host_slot, other_slot, 0, "direct" + ) + + self.assertTrue( + torch.equal(short_conv[other_slot], short_conv[checkpoint_slot]), + "the restored slot kept the previous occupant's PLE conv window", + ) + self.assertEqual( + ngram[other_slot].tolist(), + [[11, 321]], + "the restored slot kept the previous occupant's N-gram history", + ) + + def test_data_page_carries_side_state(self): + """The L3 page is the same payload: what RAM carries, disk must carry.""" + host, device_pool = _make_pools(True) + short_conv_host, ngram_host = host.slot_state_buffers + short_conv_host[1] = 2.25 + ngram_host[1] = torch.tensor([7, 9]) + + page = host.get_data_page(1) + self.assertEqual(page.numel(), host.page_size * host.size_per_token) + + short_conv_host[1] = 0 + ngram_host[1] = 0 + host.set_from_flat_data_page(1, page) + self.assertTrue(torch.all(short_conv_host[1] == 2.25)) + self.assertEqual(ngram_host[1].tolist(), [7, 9]) + + def test_short_page_is_refused_rather_than_half_applied(self): + host, _ = _make_pools(True) + short = torch.zeros(host.size_per_token - 1, dtype=torch.uint8) + with self.assertRaises(ValueError): + host.set_from_flat_data_page(0, short) + + def test_model_without_side_state_is_unaffected(self): + host, device_pool = _make_pools(False) + self.assertEqual(host.slot_state_device_tensors, []) + self.assertEqual(host.slot_state_buffers, []) + page = host.get_data_page(0) + self.assertEqual(page.numel(), host.page_size * host.size_per_token) + pf_lf, lf_pf = _stub_conv_and_temporal_copies() + with pf_lf, lf_pf: + host.backup_from_device_all_layer( + device_pool, torch.tensor([0]), torch.tensor([0]), "direct" + ) + host.load_to_device_per_layer( + device_pool, torch.tensor([0]), torch.tensor([1]), 0, "direct" + ) + + +if __name__ == "__main__": + unittest.main()