Build Mamba host pools from device buffer infos (#43596)

This commit is contained in:
Ke Bao
2026-10-11 22:35:52 +08:00
committed by GitHub
parent ef3e5c83c5
commit c938b6e454
12 changed files with 295 additions and 43 deletions
@@ -0,0 +1,73 @@
"""Device buffer facts consumed by host assembly.
The device provider enumerates persistent tensors. Host assembly owns capacity,
layer routing and transfers. This first consumer describes Mamba checkpoints.
"""
from __future__ import annotations
import msgspec
import torch
from sglang.srt.mem_cache.hicache_storage import PoolName
class MambaStateBufferInfo(msgspec.Struct, frozen=True, kw_only=True):
"""Persistent checkpoint tensors. Layer and checkpoint axes come first.
Extra buffers share checkpoint IDs but have no layer axis. Speculative
intermediate tensors are work space and do not belong in host checkpoints.
"""
temporal: torch.Tensor
conv: tuple[torch.Tensor, ...]
extra_checkpoint_buffers: tuple[torch.Tensor, ...] = ()
def validate(self, *, layer_ids: tuple[int, ...] | None = None) -> None:
shape = tuple(self.temporal.shape)
if self.temporal.ndim < 2 or not self.conv:
raise ValueError(
f"Mamba temporal requires [layer, checkpoint, ...] and conv buffers, "
f"got temporal.shape={shape}, conv count={len(self.conv)}"
)
if layer_ids is not None and len(layer_ids) != shape[0]:
raise ValueError(
f"Mamba layer_ids={layer_ids} must match temporal.shape[0]={shape[0]}"
)
for index, buffer in enumerate(self.conv):
if buffer.shape[:2] != self.temporal.shape[:2]:
raise ValueError(
f"Mamba conv[{index}].shape={tuple(buffer.shape)} must start "
f"with layer/checkpoint axes {shape[:2]}"
)
if buffer.dtype != self.conv[0].dtype:
raise ValueError(
f"Mamba conv[{index}].dtype={buffer.dtype}, expected {self.conv[0].dtype}"
)
for index, buffer in enumerate(self.extra_checkpoint_buffers):
if buffer.ndim < 1 or buffer.shape[0] != shape[1]:
raise ValueError(
f"Mamba extra_checkpoint_buffers[{index}].shape={tuple(buffer.shape)} "
f"must start with checkpoint capacity {shape[1]}"
)
for buffer in (*self.conv, *self.extra_checkpoint_buffers):
if buffer.device != self.temporal.device:
raise ValueError(
f"Mamba buffer.device={buffer.device}, expected {self.temporal.device}"
)
class DevicePoolInfo(msgspec.Struct, frozen=True, kw_only=True):
pool_name: PoolName
indices_from_pool: PoolName
# Model layer IDs, including the PP stage offset, in buffer-axis order.
layer_ids: tuple[int, ...]
buffer_info: MambaStateBufferInfo
def __post_init__(self) -> None:
# A PP stage may own no checkpoint layers. Buffer validation decides
# whether an empty layer axis is valid for the physical format.
if len(set(self.layer_ids)) != len(self.layer_ids) or any(
layer < 0 for layer in self.layer_ids
):
raise ValueError(f"{self.pool_name}: invalid model layers {self.layer_ids}")
@@ -1104,6 +1104,22 @@ def build_deepseek_v4_hicache_stack(
return host_pool_group, cache_controller
def build_mamba_host_pool(mamba_pool, *, host_size: float, layout: str):
(pool_info,) = mamba_pool.get_device_pool_infos()
pool_info.buffer_info.validate(layer_ids=pool_info.layer_ids)
capacity = mamba_pool.host_capacity_tokens
if capacity is None:
capacity = mamba_pool.size
return MambaPoolHost(
pool_info.buffer_info,
get_memory().hicache_ratio,
host_size,
device_capacity=capacity,
allocator_type=_get_allocator_type(),
layout=layout,
)
def build_hybrid_mamba_stack(
*,
params: CacheInitParams,
@@ -1167,11 +1183,9 @@ def build_hybrid_mamba_stack(
if get_memory().hicache_mem_layout == "page_first_kv_split"
else get_memory().hicache_mem_layout
)
mamba_host_pool = MambaPoolHost(
mamba_host_pool = build_mamba_host_pool(
mamba_pool,
get_memory().hicache_ratio,
mamba_host_size,
allocator_type=_get_allocator_type(),
host_size=mamba_host_size,
layout=mamba_layout,
)
entries = _build_pool_entries(config=config, root_host_pool=kv_host_pool) + [
@@ -1281,11 +1295,9 @@ def build_hybrid_mamba_swa_stack(
host_size=swa_host_size,
pool_label="swa",
)
mamba_host_pool = MambaPoolHost(
mamba_host_pool = build_mamba_host_pool(
mamba_pool,
get_memory().hicache_ratio,
mamba_host_size,
allocator_type=_get_allocator_type(),
host_size=mamba_host_size,
layout=get_memory().hicache_mem_layout,
)
entries = [
@@ -100,6 +100,7 @@ from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
if TYPE_CHECKING:
from sglang.srt.managers.cache_controller import LayerDoneCounter
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo
logger = logging.getLogger(__name__)
@@ -500,6 +501,8 @@ class ReqToTokenPool:
class MambaPool:
host_capacity_tokens: Optional[int] = None
# Axis of each two-dimensional conv state that represents the sliding window.
# Upstream states use (dim, K-1); subclasses may preserve another layout.
conv_window_axis = -1
@@ -1066,6 +1069,31 @@ class MambaPool:
self.conv_slice_axis = getattr(cache_params.shape, "conv_slice_axis", 0)
self._warmup_fused_copy_slot_kernel()
def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]:
"""Checkpoint buffers for host transfer, excluding speculative work space."""
from sglang.srt.mem_cache.device_pool_info import (
DevicePoolInfo,
MambaStateBufferInfo,
)
from sglang.srt.mem_cache.hicache_storage import PoolName
return (
DevicePoolInfo(
pool_name=PoolName.MAMBA,
indices_from_pool=PoolName.MAMBA,
layer_ids=tuple(self.mamba_layer_ids),
buffer_info=MambaStateBufferInfo(
temporal=self.mamba_cache.temporal,
conv=tuple(self.mamba_cache.conv),
extra_checkpoint_buffers=tuple(
buffer
for sibling in self._slot_siblings
for _, buffer, _, _ in sibling.iter_transfer_state_entries()
),
),
),
)
def get_speculative_mamba2_params_all_layers(self) -> SpeculativeState:
assert isinstance(self.mamba_cache, self.SpeculativeState)
return self.mamba_cache
+29 -31
View File
@@ -8,7 +8,7 @@ from typing import Optional
import numpy as np
import torch
from sglang.srt.mem_cache.memory_pool import MambaPool
from sglang.srt.mem_cache.device_pool_info import MambaStateBufferInfo
from sglang.srt.mem_cache.pool_host.base import (
HostKVCache,
host_memory_budget_bytes,
@@ -79,15 +79,20 @@ class MambaPoolHost(HostKVCache):
def __init__(
self,
device_pool: MambaPool,
buffer_info: MambaStateBufferInfo,
host_to_device_ratio: float,
host_size: int,
*,
device_capacity: int,
pin_memory: bool = True,
device: str = "cpu",
allocator_type: str = "default",
layout: str = "layer_first",
):
self.device_pool = device_pool
buffer_info.validate()
self.buffer_info = buffer_info
self.device_pool = None
self._buffer_device = buffer_info.temporal.device
self.page_size = 1
assert layout in [
@@ -99,32 +104,25 @@ class MambaPoolHost(HostKVCache):
self.pin_memory = pin_memory
self.device = device
self.allocator = get_allocator_from_storage(allocator_type)
self.num_mamba_layers = device_pool.num_mamba_layers
self.num_mamba_layers = buffer_info.temporal.shape[0]
self.conv_state_shapes = [
conv_state.shape[2:] for conv_state in device_pool.mamba_cache.conv
conv_state.shape[2:] for conv_state in self.buffer_info.conv
]
self.temporal_state_shape = device_pool.mamba_cache.temporal.shape[2:]
self.temporal_state_shape = self.buffer_info.temporal.shape[2:]
self.temporal_state_elem_size = int(np.prod(self.temporal_state_shape))
self.conv_state_elem_sizes = [
int(np.prod(conv_shape)) for conv_shape in self.conv_state_shapes
]
self.conv_dtype = device_pool.mamba_cache.conv[0].dtype
self.temporal_dtype = device_pool.mamba_cache.temporal.dtype
self.conv_dtype = self.buffer_info.conv[0].dtype
self.temporal_dtype = self.buffer_info.temporal.dtype
self.dtype = self.conv_dtype
# Registered side states share the checkpoint slot with conv/temporal,
# so a host round trip that skips them leaves the previous occupant's rows.
self.slot_state_device_tensors = [
state
for sibling in device_pool._slot_siblings
for _, state, _, _ in sibling.iter_transfer_state_entries()
]
self.slot_state_device_tensors = list(buffer_info.extra_checkpoint_buffers)
self.slot_state_buffers = []
self.size_per_token = self.get_size_per_token()
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
if device_capacity is None:
device_capacity = device_pool.size
if host_size > 0:
self.size = sync_fixed_hicache_size(
int(host_size * 1e9 // self.size_per_token), host_size
@@ -161,19 +159,19 @@ class MambaPoolHost(HostKVCache):
self.temporal_device_ptrs = torch.tensor(
[
device_pool.mamba_cache.temporal[i].data_ptr()
self.buffer_info.temporal[i].data_ptr()
for i in range(self.num_mamba_layers)
],
dtype=torch.uint64,
device=self.device_pool.device,
device=self._buffer_device,
)
self.conv_device_ptrs = [
torch.tensor(
[conv_state[i].data_ptr() for i in range(self.num_mamba_layers)],
dtype=torch.uint64,
device=self.device_pool.device,
device=self._buffer_device,
)
for conv_state in device_pool.mamba_cache.conv
for conv_state in self.buffer_info.conv
]
self.kv_buffer = self.init_kv_buffer()
@@ -208,7 +206,7 @@ class MambaPoolHost(HostKVCache):
logger.info("NPU HiCache Mamba state transfer mode: native async.")
def init_kv_buffer(self):
_host_alloc = ALLOC_MEMORY_FUNCS[self.device_pool.device]
_host_alloc = ALLOC_MEMORY_FUNCS[self._buffer_device.type]
def alloc_func(dims, *, dtype, device, pin_memory, allocator):
# conv-only linear attention has no ssm state: mmap can't map the
@@ -662,7 +660,7 @@ class MambaPoolHost(HostKVCache):
# transfer errors, same guard as the per-layer path below
if self.temporal_state_elem_size > 0:
transfer_mamba_state(
device_buf=device_pool.mamba_cache.temporal,
device_buf=self.buffer_info.temporal,
host_buf=self.temporal_buffer,
device_indices=device_indices,
host_indices=host_indices,
@@ -670,7 +668,7 @@ class MambaPoolHost(HostKVCache):
)
for conv_idx in range(len(self.conv_state_shapes)):
transfer_mamba_state(
device_buf=device_pool.mamba_cache.conv[conv_idx],
device_buf=self.buffer_info.conv[conv_idx],
host_buf=self.conv_buffer[conv_idx],
device_indices=device_indices,
host_indices=host_indices,
@@ -681,7 +679,7 @@ class MambaPoolHost(HostKVCache):
if self.temporal_state_elem_size > 0:
self._copy_tensor_pf_lf(
src=self.temporal_buffer,
dst=device_pool.mamba_cache.temporal[layer_id],
dst=self.buffer_info.temporal[layer_id],
src_indices=host_indices,
dst_indices=device_indices,
layer_id=layer_id,
@@ -691,7 +689,7 @@ class MambaPoolHost(HostKVCache):
for conv_idx in range(len(self.conv_state_shapes)):
self._copy_tensor_pf_lf(
src=self.conv_buffer[conv_idx],
dst=device_pool.mamba_cache.conv[conv_idx][layer_id],
dst=self.buffer_info.conv[conv_idx][layer_id],
src_indices=host_indices,
dst_indices=device_indices,
layer_id=layer_id,
@@ -701,7 +699,7 @@ class MambaPoolHost(HostKVCache):
else:
self._copy_tensor(
self.temporal_buffer[layer_id],
device_pool.mamba_cache.temporal[layer_id],
self.buffer_info.temporal[layer_id],
host_indices,
device_indices,
io_backend,
@@ -709,7 +707,7 @@ class MambaPoolHost(HostKVCache):
for conv_idx in range(len(self.conv_state_shapes)):
self._copy_tensor(
self.conv_buffer[conv_idx][layer_id],
device_pool.mamba_cache.conv[conv_idx][layer_id],
self.buffer_info.conv[conv_idx][layer_id],
host_indices,
device_indices,
io_backend,
@@ -727,7 +725,7 @@ class MambaPoolHost(HostKVCache):
# no ssm state on conv-only models: a 0-size batched memcpy errors
if self.temporal_state_elem_size > 0:
self._copy_tensor_all_layers_lf_pf(
src_layers=device_pool.mamba_cache.temporal,
src_layers=self.buffer_info.temporal,
dst=self.temporal_buffer,
src_indices=device_indices,
dst_indices=host_indices,
@@ -739,7 +737,7 @@ class MambaPoolHost(HostKVCache):
)
for conv_idx in range(len(self.conv_state_shapes)):
self._copy_tensor_all_layers_lf_pf(
src_layers=device_pool.mamba_cache.conv[conv_idx],
src_layers=self.buffer_info.conv[conv_idx],
dst=self.conv_buffer[conv_idx],
src_indices=device_indices,
dst_indices=host_indices,
@@ -752,7 +750,7 @@ class MambaPoolHost(HostKVCache):
else:
for layer_id in range(self.num_mamba_layers):
self._copy_tensor(
device_pool.mamba_cache.temporal[layer_id],
self.buffer_info.temporal[layer_id],
self.temporal_buffer[layer_id],
device_indices,
host_indices,
@@ -760,7 +758,7 @@ class MambaPoolHost(HostKVCache):
)
for conv_idx in range(len(self.conv_state_shapes)):
self._copy_tensor(
device_pool.mamba_cache.conv[conv_idx][layer_id],
self.buffer_info.conv[conv_idx][layer_id],
self.conv_buffer[conv_idx][layer_id],
device_indices,
host_indices,