Build Mamba host pools from device buffer infos (#43596)
This commit is contained in:
@@ -0,0 +1,73 @@
|
||||
"""Device buffer facts consumed by host assembly.
|
||||
|
||||
The device provider enumerates persistent tensors. Host assembly owns capacity,
|
||||
layer routing and transfers. This first consumer describes Mamba checkpoints.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName
|
||||
|
||||
|
||||
class MambaStateBufferInfo(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Persistent checkpoint tensors. Layer and checkpoint axes come first.
|
||||
|
||||
Extra buffers share checkpoint IDs but have no layer axis. Speculative
|
||||
intermediate tensors are work space and do not belong in host checkpoints.
|
||||
"""
|
||||
|
||||
temporal: torch.Tensor
|
||||
conv: tuple[torch.Tensor, ...]
|
||||
extra_checkpoint_buffers: tuple[torch.Tensor, ...] = ()
|
||||
|
||||
def validate(self, *, layer_ids: tuple[int, ...] | None = None) -> None:
|
||||
shape = tuple(self.temporal.shape)
|
||||
if self.temporal.ndim < 2 or not self.conv:
|
||||
raise ValueError(
|
||||
f"Mamba temporal requires [layer, checkpoint, ...] and conv buffers, "
|
||||
f"got temporal.shape={shape}, conv count={len(self.conv)}"
|
||||
)
|
||||
if layer_ids is not None and len(layer_ids) != shape[0]:
|
||||
raise ValueError(
|
||||
f"Mamba layer_ids={layer_ids} must match temporal.shape[0]={shape[0]}"
|
||||
)
|
||||
for index, buffer in enumerate(self.conv):
|
||||
if buffer.shape[:2] != self.temporal.shape[:2]:
|
||||
raise ValueError(
|
||||
f"Mamba conv[{index}].shape={tuple(buffer.shape)} must start "
|
||||
f"with layer/checkpoint axes {shape[:2]}"
|
||||
)
|
||||
if buffer.dtype != self.conv[0].dtype:
|
||||
raise ValueError(
|
||||
f"Mamba conv[{index}].dtype={buffer.dtype}, expected {self.conv[0].dtype}"
|
||||
)
|
||||
for index, buffer in enumerate(self.extra_checkpoint_buffers):
|
||||
if buffer.ndim < 1 or buffer.shape[0] != shape[1]:
|
||||
raise ValueError(
|
||||
f"Mamba extra_checkpoint_buffers[{index}].shape={tuple(buffer.shape)} "
|
||||
f"must start with checkpoint capacity {shape[1]}"
|
||||
)
|
||||
for buffer in (*self.conv, *self.extra_checkpoint_buffers):
|
||||
if buffer.device != self.temporal.device:
|
||||
raise ValueError(
|
||||
f"Mamba buffer.device={buffer.device}, expected {self.temporal.device}"
|
||||
)
|
||||
|
||||
|
||||
class DevicePoolInfo(msgspec.Struct, frozen=True, kw_only=True):
|
||||
pool_name: PoolName
|
||||
indices_from_pool: PoolName
|
||||
# Model layer IDs, including the PP stage offset, in buffer-axis order.
|
||||
layer_ids: tuple[int, ...]
|
||||
buffer_info: MambaStateBufferInfo
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
# A PP stage may own no checkpoint layers. Buffer validation decides
|
||||
# whether an empty layer axis is valid for the physical format.
|
||||
if len(set(self.layer_ids)) != len(self.layer_ids) or any(
|
||||
layer < 0 for layer in self.layer_ids
|
||||
):
|
||||
raise ValueError(f"{self.pool_name}: invalid model layers {self.layer_ids}")
|
||||
@@ -1104,6 +1104,22 @@ def build_deepseek_v4_hicache_stack(
|
||||
return host_pool_group, cache_controller
|
||||
|
||||
|
||||
def build_mamba_host_pool(mamba_pool, *, host_size: float, layout: str):
|
||||
(pool_info,) = mamba_pool.get_device_pool_infos()
|
||||
pool_info.buffer_info.validate(layer_ids=pool_info.layer_ids)
|
||||
capacity = mamba_pool.host_capacity_tokens
|
||||
if capacity is None:
|
||||
capacity = mamba_pool.size
|
||||
return MambaPoolHost(
|
||||
pool_info.buffer_info,
|
||||
get_memory().hicache_ratio,
|
||||
host_size,
|
||||
device_capacity=capacity,
|
||||
allocator_type=_get_allocator_type(),
|
||||
layout=layout,
|
||||
)
|
||||
|
||||
|
||||
def build_hybrid_mamba_stack(
|
||||
*,
|
||||
params: CacheInitParams,
|
||||
@@ -1167,11 +1183,9 @@ def build_hybrid_mamba_stack(
|
||||
if get_memory().hicache_mem_layout == "page_first_kv_split"
|
||||
else get_memory().hicache_mem_layout
|
||||
)
|
||||
mamba_host_pool = MambaPoolHost(
|
||||
mamba_host_pool = build_mamba_host_pool(
|
||||
mamba_pool,
|
||||
get_memory().hicache_ratio,
|
||||
mamba_host_size,
|
||||
allocator_type=_get_allocator_type(),
|
||||
host_size=mamba_host_size,
|
||||
layout=mamba_layout,
|
||||
)
|
||||
entries = _build_pool_entries(config=config, root_host_pool=kv_host_pool) + [
|
||||
@@ -1281,11 +1295,9 @@ def build_hybrid_mamba_swa_stack(
|
||||
host_size=swa_host_size,
|
||||
pool_label="swa",
|
||||
)
|
||||
mamba_host_pool = MambaPoolHost(
|
||||
mamba_host_pool = build_mamba_host_pool(
|
||||
mamba_pool,
|
||||
get_memory().hicache_ratio,
|
||||
mamba_host_size,
|
||||
allocator_type=_get_allocator_type(),
|
||||
host_size=mamba_host_size,
|
||||
layout=get_memory().hicache_mem_layout,
|
||||
)
|
||||
entries = [
|
||||
|
||||
@@ -100,6 +100,7 @@ from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.cache_controller import LayerDoneCounter
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.mem_cache.device_pool_info import DevicePoolInfo
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -500,6 +501,8 @@ class ReqToTokenPool:
|
||||
|
||||
|
||||
class MambaPool:
|
||||
host_capacity_tokens: Optional[int] = None
|
||||
|
||||
# Axis of each two-dimensional conv state that represents the sliding window.
|
||||
# Upstream states use (dim, K-1); subclasses may preserve another layout.
|
||||
conv_window_axis = -1
|
||||
@@ -1066,6 +1069,31 @@ class MambaPool:
|
||||
self.conv_slice_axis = getattr(cache_params.shape, "conv_slice_axis", 0)
|
||||
self._warmup_fused_copy_slot_kernel()
|
||||
|
||||
def get_device_pool_infos(self) -> tuple[DevicePoolInfo, ...]:
|
||||
"""Checkpoint buffers for host transfer, excluding speculative work space."""
|
||||
from sglang.srt.mem_cache.device_pool_info import (
|
||||
DevicePoolInfo,
|
||||
MambaStateBufferInfo,
|
||||
)
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName
|
||||
|
||||
return (
|
||||
DevicePoolInfo(
|
||||
pool_name=PoolName.MAMBA,
|
||||
indices_from_pool=PoolName.MAMBA,
|
||||
layer_ids=tuple(self.mamba_layer_ids),
|
||||
buffer_info=MambaStateBufferInfo(
|
||||
temporal=self.mamba_cache.temporal,
|
||||
conv=tuple(self.mamba_cache.conv),
|
||||
extra_checkpoint_buffers=tuple(
|
||||
buffer
|
||||
for sibling in self._slot_siblings
|
||||
for _, buffer, _, _ in sibling.iter_transfer_state_entries()
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
def get_speculative_mamba2_params_all_layers(self) -> SpeculativeState:
|
||||
assert isinstance(self.mamba_cache, self.SpeculativeState)
|
||||
return self.mamba_cache
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import Optional
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.memory_pool import MambaPool
|
||||
from sglang.srt.mem_cache.device_pool_info import MambaStateBufferInfo
|
||||
from sglang.srt.mem_cache.pool_host.base import (
|
||||
HostKVCache,
|
||||
host_memory_budget_bytes,
|
||||
@@ -79,15 +79,20 @@ class MambaPoolHost(HostKVCache):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool: MambaPool,
|
||||
buffer_info: MambaStateBufferInfo,
|
||||
host_to_device_ratio: float,
|
||||
host_size: int,
|
||||
*,
|
||||
device_capacity: int,
|
||||
pin_memory: bool = True,
|
||||
device: str = "cpu",
|
||||
allocator_type: str = "default",
|
||||
layout: str = "layer_first",
|
||||
):
|
||||
self.device_pool = device_pool
|
||||
buffer_info.validate()
|
||||
self.buffer_info = buffer_info
|
||||
self.device_pool = None
|
||||
self._buffer_device = buffer_info.temporal.device
|
||||
self.page_size = 1
|
||||
|
||||
assert layout in [
|
||||
@@ -99,32 +104,25 @@ class MambaPoolHost(HostKVCache):
|
||||
self.pin_memory = pin_memory
|
||||
self.device = device
|
||||
self.allocator = get_allocator_from_storage(allocator_type)
|
||||
self.num_mamba_layers = device_pool.num_mamba_layers
|
||||
self.num_mamba_layers = buffer_info.temporal.shape[0]
|
||||
|
||||
self.conv_state_shapes = [
|
||||
conv_state.shape[2:] for conv_state in device_pool.mamba_cache.conv
|
||||
conv_state.shape[2:] for conv_state in self.buffer_info.conv
|
||||
]
|
||||
self.temporal_state_shape = device_pool.mamba_cache.temporal.shape[2:]
|
||||
self.temporal_state_shape = self.buffer_info.temporal.shape[2:]
|
||||
self.temporal_state_elem_size = int(np.prod(self.temporal_state_shape))
|
||||
self.conv_state_elem_sizes = [
|
||||
int(np.prod(conv_shape)) for conv_shape in self.conv_state_shapes
|
||||
]
|
||||
self.conv_dtype = device_pool.mamba_cache.conv[0].dtype
|
||||
self.temporal_dtype = device_pool.mamba_cache.temporal.dtype
|
||||
self.conv_dtype = self.buffer_info.conv[0].dtype
|
||||
self.temporal_dtype = self.buffer_info.temporal.dtype
|
||||
self.dtype = self.conv_dtype
|
||||
# Registered side states share the checkpoint slot with conv/temporal,
|
||||
# so a host round trip that skips them leaves the previous occupant's rows.
|
||||
self.slot_state_device_tensors = [
|
||||
state
|
||||
for sibling in device_pool._slot_siblings
|
||||
for _, state, _, _ in sibling.iter_transfer_state_entries()
|
||||
]
|
||||
self.slot_state_device_tensors = list(buffer_info.extra_checkpoint_buffers)
|
||||
self.slot_state_buffers = []
|
||||
self.size_per_token = self.get_size_per_token()
|
||||
|
||||
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
|
||||
if device_capacity is None:
|
||||
device_capacity = device_pool.size
|
||||
if host_size > 0:
|
||||
self.size = sync_fixed_hicache_size(
|
||||
int(host_size * 1e9 // self.size_per_token), host_size
|
||||
@@ -161,19 +159,19 @@ class MambaPoolHost(HostKVCache):
|
||||
|
||||
self.temporal_device_ptrs = torch.tensor(
|
||||
[
|
||||
device_pool.mamba_cache.temporal[i].data_ptr()
|
||||
self.buffer_info.temporal[i].data_ptr()
|
||||
for i in range(self.num_mamba_layers)
|
||||
],
|
||||
dtype=torch.uint64,
|
||||
device=self.device_pool.device,
|
||||
device=self._buffer_device,
|
||||
)
|
||||
self.conv_device_ptrs = [
|
||||
torch.tensor(
|
||||
[conv_state[i].data_ptr() for i in range(self.num_mamba_layers)],
|
||||
dtype=torch.uint64,
|
||||
device=self.device_pool.device,
|
||||
device=self._buffer_device,
|
||||
)
|
||||
for conv_state in device_pool.mamba_cache.conv
|
||||
for conv_state in self.buffer_info.conv
|
||||
]
|
||||
|
||||
self.kv_buffer = self.init_kv_buffer()
|
||||
@@ -208,7 +206,7 @@ class MambaPoolHost(HostKVCache):
|
||||
logger.info("NPU HiCache Mamba state transfer mode: native async.")
|
||||
|
||||
def init_kv_buffer(self):
|
||||
_host_alloc = ALLOC_MEMORY_FUNCS[self.device_pool.device]
|
||||
_host_alloc = ALLOC_MEMORY_FUNCS[self._buffer_device.type]
|
||||
|
||||
def alloc_func(dims, *, dtype, device, pin_memory, allocator):
|
||||
# conv-only linear attention has no ssm state: mmap can't map the
|
||||
@@ -662,7 +660,7 @@ class MambaPoolHost(HostKVCache):
|
||||
# transfer errors, same guard as the per-layer path below
|
||||
if self.temporal_state_elem_size > 0:
|
||||
transfer_mamba_state(
|
||||
device_buf=device_pool.mamba_cache.temporal,
|
||||
device_buf=self.buffer_info.temporal,
|
||||
host_buf=self.temporal_buffer,
|
||||
device_indices=device_indices,
|
||||
host_indices=host_indices,
|
||||
@@ -670,7 +668,7 @@ class MambaPoolHost(HostKVCache):
|
||||
)
|
||||
for conv_idx in range(len(self.conv_state_shapes)):
|
||||
transfer_mamba_state(
|
||||
device_buf=device_pool.mamba_cache.conv[conv_idx],
|
||||
device_buf=self.buffer_info.conv[conv_idx],
|
||||
host_buf=self.conv_buffer[conv_idx],
|
||||
device_indices=device_indices,
|
||||
host_indices=host_indices,
|
||||
@@ -681,7 +679,7 @@ class MambaPoolHost(HostKVCache):
|
||||
if self.temporal_state_elem_size > 0:
|
||||
self._copy_tensor_pf_lf(
|
||||
src=self.temporal_buffer,
|
||||
dst=device_pool.mamba_cache.temporal[layer_id],
|
||||
dst=self.buffer_info.temporal[layer_id],
|
||||
src_indices=host_indices,
|
||||
dst_indices=device_indices,
|
||||
layer_id=layer_id,
|
||||
@@ -691,7 +689,7 @@ class MambaPoolHost(HostKVCache):
|
||||
for conv_idx in range(len(self.conv_state_shapes)):
|
||||
self._copy_tensor_pf_lf(
|
||||
src=self.conv_buffer[conv_idx],
|
||||
dst=device_pool.mamba_cache.conv[conv_idx][layer_id],
|
||||
dst=self.buffer_info.conv[conv_idx][layer_id],
|
||||
src_indices=host_indices,
|
||||
dst_indices=device_indices,
|
||||
layer_id=layer_id,
|
||||
@@ -701,7 +699,7 @@ class MambaPoolHost(HostKVCache):
|
||||
else:
|
||||
self._copy_tensor(
|
||||
self.temporal_buffer[layer_id],
|
||||
device_pool.mamba_cache.temporal[layer_id],
|
||||
self.buffer_info.temporal[layer_id],
|
||||
host_indices,
|
||||
device_indices,
|
||||
io_backend,
|
||||
@@ -709,7 +707,7 @@ class MambaPoolHost(HostKVCache):
|
||||
for conv_idx in range(len(self.conv_state_shapes)):
|
||||
self._copy_tensor(
|
||||
self.conv_buffer[conv_idx][layer_id],
|
||||
device_pool.mamba_cache.conv[conv_idx][layer_id],
|
||||
self.buffer_info.conv[conv_idx][layer_id],
|
||||
host_indices,
|
||||
device_indices,
|
||||
io_backend,
|
||||
@@ -727,7 +725,7 @@ class MambaPoolHost(HostKVCache):
|
||||
# no ssm state on conv-only models: a 0-size batched memcpy errors
|
||||
if self.temporal_state_elem_size > 0:
|
||||
self._copy_tensor_all_layers_lf_pf(
|
||||
src_layers=device_pool.mamba_cache.temporal,
|
||||
src_layers=self.buffer_info.temporal,
|
||||
dst=self.temporal_buffer,
|
||||
src_indices=device_indices,
|
||||
dst_indices=host_indices,
|
||||
@@ -739,7 +737,7 @@ class MambaPoolHost(HostKVCache):
|
||||
)
|
||||
for conv_idx in range(len(self.conv_state_shapes)):
|
||||
self._copy_tensor_all_layers_lf_pf(
|
||||
src_layers=device_pool.mamba_cache.conv[conv_idx],
|
||||
src_layers=self.buffer_info.conv[conv_idx],
|
||||
dst=self.conv_buffer[conv_idx],
|
||||
src_indices=device_indices,
|
||||
dst_indices=host_indices,
|
||||
@@ -752,7 +750,7 @@ class MambaPoolHost(HostKVCache):
|
||||
else:
|
||||
for layer_id in range(self.num_mamba_layers):
|
||||
self._copy_tensor(
|
||||
device_pool.mamba_cache.temporal[layer_id],
|
||||
self.buffer_info.temporal[layer_id],
|
||||
self.temporal_buffer[layer_id],
|
||||
device_indices,
|
||||
host_indices,
|
||||
@@ -760,7 +758,7 @@ class MambaPoolHost(HostKVCache):
|
||||
)
|
||||
for conv_idx in range(len(self.conv_state_shapes)):
|
||||
self._copy_tensor(
|
||||
device_pool.mamba_cache.conv[conv_idx][layer_id],
|
||||
self.buffer_info.conv[conv_idx][layer_id],
|
||||
self.conv_buffer[conv_idx][layer_id],
|
||||
device_indices,
|
||||
host_indices,
|
||||
|
||||
Reference in New Issue
Block a user