[Fix] HiCache: carry registered Mamba slot side states (Qwen4-Exp PLE) through the host tier (#39862)

Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
Co-authored-by: alphabetc1 <2508695655@qq.com>
Co-authored-by: ispobock <ispobaoke@gmail.com>
This commit is contained in:
JarJarBeatyourattitude
2026-10-04 10:15:59 +08:00
committed by GitHub
co-authored by Shuwen Wang alphabetc1 ispobock
parent f2385314b2
commit 5606fb8592
3 changed files with 321 additions and 3 deletions
@@ -1562,6 +1562,17 @@ class HybridReqToTokenPool(ReqToTokenPool):
return self.short_conv_pool.layer_intermediate_cache(layer_id)
def get_ngram_context(self, ngram_indices: torch.Tensor) -> torch.Tensor:
# Read once per forward, BEFORE the decoder-layer loop, so unlike
# short_conv_layer_cache it has no per-layer barrier of its own. The
# host tier restores side state on the first Mamba layer's transfer, so
# wait for that layer before reading the history.
if (
self.layer_transfer_counter is not None
and self.layer_transfer_counter.consumer_index >= 0
and self.mamba_map
):
first_mamba_layer = min(self.mamba_map)
self.layer_transfer_counter.wait_until(first_mamba_layer - self.start_layer)
return self.ngram_pool.get_context(ngram_indices)
def set_ngram_context(
+93 -3
View File
@@ -72,6 +72,11 @@ def _npu_hicache_mamba_io_mode() -> str:
class MambaPoolHost(HostKVCache):
# Class-level defaults so a model without side state, and the test stubs that
# skip __init__, still read an empty pair rather than raising.
slot_state_device_tensors: tuple = ()
slot_state_buffers: tuple = ()
def __init__(
self,
device_pool: MambaPool,
@@ -107,6 +112,14 @@ class MambaPoolHost(HostKVCache):
self.conv_dtype = device_pool.mamba_cache.conv[0].dtype
self.temporal_dtype = device_pool.mamba_cache.temporal.dtype
self.dtype = self.conv_dtype
# Registered side states share the checkpoint slot with conv/temporal,
# so a host round trip that skips them leaves the previous occupant's rows.
self.slot_state_device_tensors = [
state
for sibling in device_pool._slot_siblings
for _, state, _, _ in sibling.iter_transfer_state_entries()
]
self.slot_state_buffers = []
self.size_per_token = self.get_size_per_token()
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
@@ -264,11 +277,29 @@ class MambaPoolHost(HostKVCache):
allocator=self.allocator,
)
)
# Side state keeps its own dtype and its own slot axis (axis 0 of the
# per-layer view the device pool yields).
self.slot_state_buffers = [
alloc_func(
(self.size,) + tuple(state.shape[1:]),
dtype=state.dtype,
device=self.device,
pin_memory=self.pin_memory,
allocator=self.allocator,
)
for state in self.slot_state_device_tensors
]
# destroy() unregisters via kv_buffer; without this list the pinned
# registrations leak past the buffers' mmap. 0-element buffers
# (conv-only models' temporal state) were never registered.
return [
buf for buf in (self.temporal_buffer, *self.conv_buffer) if buf.numel() > 0
buf
for buf in (
self.temporal_buffer,
*self.conv_buffer,
*self.slot_state_buffers,
)
if buf.numel() > 0
]
def _init_write_back_staging_buffers(self):
@@ -284,7 +315,7 @@ class MambaPoolHost(HostKVCache):
def get_hybrid_pool_buffer(self):
# Expose all mamba host tensors that need Mooncake buffer registration.
return [self.temporal_buffer, *self.conv_buffer]
return [self.temporal_buffer, *self.conv_buffer, *self.slot_state_buffers]
def _iter_page_tensors(self, index: int):
if self.layout in ["page_first", "page_first_direct"]:
@@ -295,6 +326,8 @@ class MambaPoolHost(HostKVCache):
yield self.temporal_buffer[:, index : index + self.page_size]
for conv_buf in self.conv_buffer:
yield conv_buf[:, index : index + self.page_size]
for state_buf in self.slot_state_buffers:
yield state_buf[index : index + self.page_size]
@staticmethod
def _flatten_tensor_bytes(tensor: torch.Tensor) -> torch.Tensor:
@@ -343,7 +376,13 @@ class MambaPoolHost(HostKVCache):
for conv_elem_size in self.conv_state_elem_sizes
)
temporal_size = self.temporal_state_elem_size * self.temporal_dtype.itemsize
return (conv_total_size + temporal_size) * self.num_mamba_layers
# Side-state rows are per-slot, not per-layer: each entry is already a
# single layer's view, so its own row size is added once.
side_size = sum(
state[0].numel() * state.element_size()
for state in self.slot_state_device_tensors
)
return (conv_total_size + temporal_size) * self.num_mamba_layers + side_size
def get_ksize_per_token(self):
return self.get_size_per_token()
@@ -609,6 +648,11 @@ class MambaPoolHost(HostKVCache):
*,
is_draft: bool = False,
):
# Side state is not per-layer, so it rides on the first layer's call --
# before that layer's completion event, which is what every reader of a
# restored slot waits on.
if layer_id == 0:
self._transfer_slot_states(host_indices, device_indices, to_device=True)
if self.layout in ["page_first", "page_first_direct"]:
if io_backend == "kernel_ascend" and transfer_mamba_state is not None:
# NPU: transfer all layers at once via dedicated kernel.
@@ -722,6 +766,30 @@ class MambaPoolHost(HostKVCache):
host_indices,
io_backend,
)
self._transfer_slot_states(host_indices, device_indices, to_device=False)
def _transfer_slot_states(self, host_indices, device_indices, *, to_device):
"""Copy the registered side-state rows for these slots, one leg or the
other. Blocking copies: the rows are a few hundred KB per checkpoint and
correctness of the ordering against the completion event matters more
than the microseconds."""
if not self.slot_state_device_tensors:
return
for state, host_buf in zip(
self.slot_state_device_tensors, self.slot_state_buffers
):
dev_idx = device_indices.reshape(-1).to(
device=state.device, dtype=torch.long
)
host_idx = host_indices.reshape(-1).to(
device=host_buf.device, dtype=torch.long
)
if to_device:
values = host_buf.index_select(0, host_idx)
state.index_copy_(0, dev_idx, values.to(state.device))
else:
values = state.index_select(0, dev_idx)
host_buf.index_copy_(0, host_idx, values.to(host_buf.device))
def get_data_page(self, index, flat: bool = True) -> torch.Tensor:
data_page = torch.cat(
@@ -746,11 +814,24 @@ class MambaPoolHost(HostKVCache):
data_page: torch.Tensor,
) -> None:
flat_bytes = data_page.contiguous().view(torch.uint8).reshape(-1)
expected = self.page_size * self.size_per_token
if self.slot_state_device_tensors and flat_bytes.numel() != expected:
# A page stored by a build without the side state is shorter; say so
# instead of failing inside a reshape halfway through the restore.
raise ValueError(
f"Mamba state page has {flat_bytes.numel()} bytes, expected "
f"{expected}: the page predates registered slot side states."
)
start = 0
for tensor in self._iter_page_tensors(index):
num_bytes = tensor.numel() * tensor.element_size()
tensor_bytes = flat_bytes[start : start + num_bytes]
start += num_bytes
if tensor_bytes.storage_offset() % tensor.element_size():
# With side state the page holds components of different element
# sizes, so a later component can begin at a byte offset its own
# dtype cannot be viewed at. A contiguous copy re-bases it at 0.
tensor_bytes = tensor_bytes.clone()
restored = tensor_bytes.view(dtype=tensor.dtype).reshape(tensor.shape)
tensor.copy_(restored)
@@ -760,6 +841,11 @@ class MambaPoolHost(HostKVCache):
Only page-first layouts are supported for mamba storage zero-copy because
each page slot in temporal/conv buffers is directly addressable.
"""
if self.slot_state_device_tensors:
raise NotImplementedError(
"Mamba storage zero-copy does not carry registered slot side "
"states; use a whole-page storage backend."
)
assert len(indices) % self.page_size == 0
if self.layout not in ["page_first", "page_first_direct"]:
raise ValueError(
@@ -817,6 +903,10 @@ class MambaPoolHost(HostKVCache):
return ptr_list, element_size_list
def is_stride_page_aligned(self, page_size_bytes: int = 4096) -> bool:
if self.slot_state_device_tensors:
# Side-state rows are a second, differently-strided buffer: a page is
# no longer one contiguous slice, so no zero-copy claim is made.
return False
if self.layout not in ["page_first", "page_first_direct"]:
return False
temporal_stride = (