[Fix] HiCache: carry registered Mamba slot side states (Qwen4-Exp PLE) through the host tier (#39862)
Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> Co-authored-by: alphabetc1 <2508695655@qq.com> Co-authored-by: ispobock <ispobaoke@gmail.com>
This commit is contained in:
co-authored by
Shuwen Wang
alphabetc1
ispobock
parent
f2385314b2
commit
5606fb8592
@@ -1562,6 +1562,17 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
return self.short_conv_pool.layer_intermediate_cache(layer_id)
|
||||
|
||||
def get_ngram_context(self, ngram_indices: torch.Tensor) -> torch.Tensor:
|
||||
# Read once per forward, BEFORE the decoder-layer loop, so unlike
|
||||
# short_conv_layer_cache it has no per-layer barrier of its own. The
|
||||
# host tier restores side state on the first Mamba layer's transfer, so
|
||||
# wait for that layer before reading the history.
|
||||
if (
|
||||
self.layer_transfer_counter is not None
|
||||
and self.layer_transfer_counter.consumer_index >= 0
|
||||
and self.mamba_map
|
||||
):
|
||||
first_mamba_layer = min(self.mamba_map)
|
||||
self.layer_transfer_counter.wait_until(first_mamba_layer - self.start_layer)
|
||||
return self.ngram_pool.get_context(ngram_indices)
|
||||
|
||||
def set_ngram_context(
|
||||
|
||||
@@ -72,6 +72,11 @@ def _npu_hicache_mamba_io_mode() -> str:
|
||||
|
||||
|
||||
class MambaPoolHost(HostKVCache):
|
||||
# Class-level defaults so a model without side state, and the test stubs that
|
||||
# skip __init__, still read an empty pair rather than raising.
|
||||
slot_state_device_tensors: tuple = ()
|
||||
slot_state_buffers: tuple = ()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool: MambaPool,
|
||||
@@ -107,6 +112,14 @@ class MambaPoolHost(HostKVCache):
|
||||
self.conv_dtype = device_pool.mamba_cache.conv[0].dtype
|
||||
self.temporal_dtype = device_pool.mamba_cache.temporal.dtype
|
||||
self.dtype = self.conv_dtype
|
||||
# Registered side states share the checkpoint slot with conv/temporal,
|
||||
# so a host round trip that skips them leaves the previous occupant's rows.
|
||||
self.slot_state_device_tensors = [
|
||||
state
|
||||
for sibling in device_pool._slot_siblings
|
||||
for _, state, _, _ in sibling.iter_transfer_state_entries()
|
||||
]
|
||||
self.slot_state_buffers = []
|
||||
self.size_per_token = self.get_size_per_token()
|
||||
|
||||
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
|
||||
@@ -264,11 +277,29 @@ class MambaPoolHost(HostKVCache):
|
||||
allocator=self.allocator,
|
||||
)
|
||||
)
|
||||
# Side state keeps its own dtype and its own slot axis (axis 0 of the
|
||||
# per-layer view the device pool yields).
|
||||
self.slot_state_buffers = [
|
||||
alloc_func(
|
||||
(self.size,) + tuple(state.shape[1:]),
|
||||
dtype=state.dtype,
|
||||
device=self.device,
|
||||
pin_memory=self.pin_memory,
|
||||
allocator=self.allocator,
|
||||
)
|
||||
for state in self.slot_state_device_tensors
|
||||
]
|
||||
# destroy() unregisters via kv_buffer; without this list the pinned
|
||||
# registrations leak past the buffers' mmap. 0-element buffers
|
||||
# (conv-only models' temporal state) were never registered.
|
||||
return [
|
||||
buf for buf in (self.temporal_buffer, *self.conv_buffer) if buf.numel() > 0
|
||||
buf
|
||||
for buf in (
|
||||
self.temporal_buffer,
|
||||
*self.conv_buffer,
|
||||
*self.slot_state_buffers,
|
||||
)
|
||||
if buf.numel() > 0
|
||||
]
|
||||
|
||||
def _init_write_back_staging_buffers(self):
|
||||
@@ -284,7 +315,7 @@ class MambaPoolHost(HostKVCache):
|
||||
|
||||
def get_hybrid_pool_buffer(self):
|
||||
# Expose all mamba host tensors that need Mooncake buffer registration.
|
||||
return [self.temporal_buffer, *self.conv_buffer]
|
||||
return [self.temporal_buffer, *self.conv_buffer, *self.slot_state_buffers]
|
||||
|
||||
def _iter_page_tensors(self, index: int):
|
||||
if self.layout in ["page_first", "page_first_direct"]:
|
||||
@@ -295,6 +326,8 @@ class MambaPoolHost(HostKVCache):
|
||||
yield self.temporal_buffer[:, index : index + self.page_size]
|
||||
for conv_buf in self.conv_buffer:
|
||||
yield conv_buf[:, index : index + self.page_size]
|
||||
for state_buf in self.slot_state_buffers:
|
||||
yield state_buf[index : index + self.page_size]
|
||||
|
||||
@staticmethod
|
||||
def _flatten_tensor_bytes(tensor: torch.Tensor) -> torch.Tensor:
|
||||
@@ -343,7 +376,13 @@ class MambaPoolHost(HostKVCache):
|
||||
for conv_elem_size in self.conv_state_elem_sizes
|
||||
)
|
||||
temporal_size = self.temporal_state_elem_size * self.temporal_dtype.itemsize
|
||||
return (conv_total_size + temporal_size) * self.num_mamba_layers
|
||||
# Side-state rows are per-slot, not per-layer: each entry is already a
|
||||
# single layer's view, so its own row size is added once.
|
||||
side_size = sum(
|
||||
state[0].numel() * state.element_size()
|
||||
for state in self.slot_state_device_tensors
|
||||
)
|
||||
return (conv_total_size + temporal_size) * self.num_mamba_layers + side_size
|
||||
|
||||
def get_ksize_per_token(self):
|
||||
return self.get_size_per_token()
|
||||
@@ -609,6 +648,11 @@ class MambaPoolHost(HostKVCache):
|
||||
*,
|
||||
is_draft: bool = False,
|
||||
):
|
||||
# Side state is not per-layer, so it rides on the first layer's call --
|
||||
# before that layer's completion event, which is what every reader of a
|
||||
# restored slot waits on.
|
||||
if layer_id == 0:
|
||||
self._transfer_slot_states(host_indices, device_indices, to_device=True)
|
||||
if self.layout in ["page_first", "page_first_direct"]:
|
||||
if io_backend == "kernel_ascend" and transfer_mamba_state is not None:
|
||||
# NPU: transfer all layers at once via dedicated kernel.
|
||||
@@ -722,6 +766,30 @@ class MambaPoolHost(HostKVCache):
|
||||
host_indices,
|
||||
io_backend,
|
||||
)
|
||||
self._transfer_slot_states(host_indices, device_indices, to_device=False)
|
||||
|
||||
def _transfer_slot_states(self, host_indices, device_indices, *, to_device):
|
||||
"""Copy the registered side-state rows for these slots, one leg or the
|
||||
other. Blocking copies: the rows are a few hundred KB per checkpoint and
|
||||
correctness of the ordering against the completion event matters more
|
||||
than the microseconds."""
|
||||
if not self.slot_state_device_tensors:
|
||||
return
|
||||
for state, host_buf in zip(
|
||||
self.slot_state_device_tensors, self.slot_state_buffers
|
||||
):
|
||||
dev_idx = device_indices.reshape(-1).to(
|
||||
device=state.device, dtype=torch.long
|
||||
)
|
||||
host_idx = host_indices.reshape(-1).to(
|
||||
device=host_buf.device, dtype=torch.long
|
||||
)
|
||||
if to_device:
|
||||
values = host_buf.index_select(0, host_idx)
|
||||
state.index_copy_(0, dev_idx, values.to(state.device))
|
||||
else:
|
||||
values = state.index_select(0, dev_idx)
|
||||
host_buf.index_copy_(0, host_idx, values.to(host_buf.device))
|
||||
|
||||
def get_data_page(self, index, flat: bool = True) -> torch.Tensor:
|
||||
data_page = torch.cat(
|
||||
@@ -746,11 +814,24 @@ class MambaPoolHost(HostKVCache):
|
||||
data_page: torch.Tensor,
|
||||
) -> None:
|
||||
flat_bytes = data_page.contiguous().view(torch.uint8).reshape(-1)
|
||||
expected = self.page_size * self.size_per_token
|
||||
if self.slot_state_device_tensors and flat_bytes.numel() != expected:
|
||||
# A page stored by a build without the side state is shorter; say so
|
||||
# instead of failing inside a reshape halfway through the restore.
|
||||
raise ValueError(
|
||||
f"Mamba state page has {flat_bytes.numel()} bytes, expected "
|
||||
f"{expected}: the page predates registered slot side states."
|
||||
)
|
||||
start = 0
|
||||
for tensor in self._iter_page_tensors(index):
|
||||
num_bytes = tensor.numel() * tensor.element_size()
|
||||
tensor_bytes = flat_bytes[start : start + num_bytes]
|
||||
start += num_bytes
|
||||
if tensor_bytes.storage_offset() % tensor.element_size():
|
||||
# With side state the page holds components of different element
|
||||
# sizes, so a later component can begin at a byte offset its own
|
||||
# dtype cannot be viewed at. A contiguous copy re-bases it at 0.
|
||||
tensor_bytes = tensor_bytes.clone()
|
||||
restored = tensor_bytes.view(dtype=tensor.dtype).reshape(tensor.shape)
|
||||
tensor.copy_(restored)
|
||||
|
||||
@@ -760,6 +841,11 @@ class MambaPoolHost(HostKVCache):
|
||||
Only page-first layouts are supported for mamba storage zero-copy because
|
||||
each page slot in temporal/conv buffers is directly addressable.
|
||||
"""
|
||||
if self.slot_state_device_tensors:
|
||||
raise NotImplementedError(
|
||||
"Mamba storage zero-copy does not carry registered slot side "
|
||||
"states; use a whole-page storage backend."
|
||||
)
|
||||
assert len(indices) % self.page_size == 0
|
||||
if self.layout not in ["page_first", "page_first_direct"]:
|
||||
raise ValueError(
|
||||
@@ -817,6 +903,10 @@ class MambaPoolHost(HostKVCache):
|
||||
return ptr_list, element_size_list
|
||||
|
||||
def is_stride_page_aligned(self, page_size_bytes: int = 4096) -> bool:
|
||||
if self.slot_state_device_tensors:
|
||||
# Side-state rows are a second, differently-strided buffer: a page is
|
||||
# no longer one contiguous slice, so no zero-copy claim is made.
|
||||
return False
|
||||
if self.layout not in ["page_first", "page_first_direct"]:
|
||||
return False
|
||||
temporal_stride = (
|
||||
|
||||
Reference in New Issue
Block a user