Co-authored-by: Lu Fang <116399278+luccafong@users.noreply.github.com> Co-authored-by: Shiyan Deng <20773414+842974287@users.noreply.github.com>
2535 lines
100 KiB
Python
2535 lines
100 KiB
Python
# Copyright 2023-2026 SGLang Team
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ==============================================================================
|
|
"""UnifiedKVPool -- one physical `uint8` byte buffer shared by N sub-pools.
|
|
|
|
Two END `MultiEndedAllocator`s grow inward from opposite ends; optional
|
|
"float" MIDDLE pools live between their frontiers (chain order
|
|
`[up end, floats..., down end]`). Eager- or lazy-compacting `free` keeps each
|
|
pool's byte range reclaimable. Layout is envelope-major (a slot's data for all
|
|
its layers in one contiguous byte envelope) so a freed slot vacates a region a
|
|
neighbor can grow into. Everything above the allocator stores virtual slot
|
|
IDs; the allocator owns the per-sub-pool virtual<->physical tables and
|
|
compaction only mutates those (no reference rewriting).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import logging
|
|
from abc import ABC, abstractmethod
|
|
from dataclasses import dataclass
|
|
from typing import ClassVar, Dict, List, NamedTuple, Optional, Tuple
|
|
|
|
import torch
|
|
from torch.profiler import record_function
|
|
|
|
from sglang.kernels.ops.kvcache.copy_pages import copy_pages
|
|
from sglang.kernels.ops.kvcache.zero_pages import zero_pages
|
|
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
|
|
from sglang.srt.mem_cache.layout.fused_draft import (
|
|
DenseDraftRegion,
|
|
DraftRegion,
|
|
DraftStateRegion,
|
|
FusedDraftPlacement,
|
|
)
|
|
from sglang.srt.mem_cache.layout.token_major import (
|
|
ROW_ALIGN_BYTES,
|
|
DenseEntryLayout,
|
|
DensePart,
|
|
align_entry_bytes,
|
|
align_part_offset,
|
|
build_dense_views,
|
|
build_mamba_entry_views,
|
|
)
|
|
from sglang.srt.mem_cache.memory_pool import (
|
|
HybridLinearKVPool,
|
|
HybridReqToTokenPool,
|
|
KvBufferDesc,
|
|
KVWriteLoc,
|
|
MambaPool,
|
|
MHATokenToKVPool,
|
|
MLATokenToKVPool,
|
|
unwrap_write_loc,
|
|
write_loc_is_physical,
|
|
)
|
|
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
|
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
GB = 1024 * 1024 * 1024
|
|
|
|
|
|
def _prod(iterable) -> int:
|
|
out = 1
|
|
for x in iterable:
|
|
out *= int(x)
|
|
return out
|
|
|
|
|
|
def _check_row_alignment(sub_pool: str, **row_bytes: int) -> None:
|
|
"""Entry parts are laid out in ROW_ALIGN_BYTES units; a model whose
|
|
per-rank rows are not cannot use the unified pool."""
|
|
for part, nbytes in row_bytes.items():
|
|
if nbytes % ROW_ALIGN_BYTES:
|
|
raise ValueError(
|
|
f"--enable-unified-memory needs every per-rank KV row to be a "
|
|
f"multiple of {ROW_ALIGN_BYTES} bytes; sub-pool {sub_pool!r} has a "
|
|
f"{nbytes}-byte {part} row. Run this model without "
|
|
"--enable-unified-memory."
|
|
)
|
|
|
|
|
|
def _store_dtype_for(kv_cache_dtype: torch.dtype) -> torch.dtype:
|
|
if kv_cache_dtype in (
|
|
torch.float8_e5m2,
|
|
torch.float8_e4m3fn,
|
|
torch.float8_e4m3fnuz,
|
|
):
|
|
return torch.uint8
|
|
return kv_cache_dtype
|
|
|
|
|
|
@dataclass(frozen=True, kw_only=True)
|
|
class SubPoolSpec(ABC):
|
|
"""Abstract per-slot layout of one sub-pool in a `UnifiedKVPool`.
|
|
|
|
``grow_direction`` places the sub-pool in the buffer's chain: the two
|
|
``"up"``/``"down"`` END pools own the buffer's two ends and grow inward;
|
|
``"float"`` middles live between the ends' frontiers (a float's position is
|
|
carried entirely by its allocator's two watermarks — every view spans the
|
|
whole buffer, so relocation never rebuilds views).
|
|
"""
|
|
|
|
# Grow directions this subclass accepts. Scratch-class specs (e.g. the
|
|
# spec-decode band) narrow this to ("float",).
|
|
_allowed_grow_directions: ClassVar[Tuple[str, ...]] = ("up", "down", "float")
|
|
|
|
name: str
|
|
layer_num: int
|
|
grow_direction: str # "up" | "down" | "float"
|
|
# Fused draft region riding in this sub-pool's entries; None = unfused.
|
|
# Each kind accepts the region kind its entry can lay out, after the host
|
|
# block at `draft_offset_in_entry()`.
|
|
draft_region: Optional[DraftRegion] = None
|
|
|
|
def __post_init__(self):
|
|
assert self.grow_direction in self._allowed_grow_directions, (
|
|
f"{type(self).__name__}.grow_direction must be one of "
|
|
f"{self._allowed_grow_directions}; got {self.grow_direction!r}"
|
|
)
|
|
assert self.layer_num > 0, f"layer_num must be positive; got {self.layer_num}"
|
|
|
|
@abstractmethod
|
|
def entry_bytes(self) -> int:
|
|
"""Bytes for one slot across all `layer_num` layers."""
|
|
raise NotImplementedError
|
|
|
|
@abstractmethod
|
|
def get_dtype(self) -> torch.dtype:
|
|
"""Storage dtype (informational). Multi-dtype subclasses return the dominant buffer's."""
|
|
raise NotImplementedError
|
|
|
|
|
|
@dataclass(frozen=True, kw_only=True)
|
|
class MHASubPoolSpec(SubPoolSpec):
|
|
"""Per-slot layout of one MHA-shaped sub-pool. `v_head_dim` defaults to `head_dim`.
|
|
|
|
With `draft_region` set, the draft model's K and V rows are two more
|
|
parts of every slot's entry, after the host parts:
|
|
|
|
[ K_0 | V_0 | ... | K_{Lh-1} | V_{Lh-1} | dK_0 | dV_0 | ... | pad ]
|
|
|
|
The slot stride stays the one entry, so host and draft views are indexed
|
|
by the same physical token id; only their offsets differ.
|
|
"""
|
|
|
|
head_num: int
|
|
head_dim: int
|
|
store_dtype: torch.dtype
|
|
kv_cache_dtype: Optional[torch.dtype] = None
|
|
v_head_dim: Optional[int] = None
|
|
|
|
def __post_init__(self):
|
|
super().__post_init__()
|
|
assert self.head_num > 0, f"head_num must be positive; got {self.head_num}"
|
|
assert self.head_dim > 0, f"head_dim must be positive; got {self.head_dim}"
|
|
if self.kv_cache_dtype is None:
|
|
object.__setattr__(self, "kv_cache_dtype", self.store_dtype)
|
|
expected_store_dtype = _store_dtype_for(self.kv_cache_dtype)
|
|
assert self.store_dtype == expected_store_dtype, (
|
|
"MHASubPoolSpec.store_dtype must match the storage dtype required by "
|
|
f"kv_cache_dtype; got kv_cache_dtype={self.kv_cache_dtype}, "
|
|
f"store_dtype={self.store_dtype}, expected={expected_store_dtype}"
|
|
)
|
|
if self.v_head_dim is None:
|
|
object.__setattr__(self, "v_head_dim", self.head_dim)
|
|
assert self.v_head_dim > 0, (
|
|
f"v_head_dim must be positive; got {self.v_head_dim}"
|
|
)
|
|
if self.draft_region is not None:
|
|
assert isinstance(self.draft_region, DenseDraftRegion), (
|
|
"dense entries carry dense draft K/V rows, never a state region"
|
|
)
|
|
self.draft_region.validate()
|
|
|
|
def k_row_bytes(self) -> int:
|
|
return self.head_num * self.head_dim * self.store_dtype.itemsize
|
|
|
|
def v_row_bytes(self) -> int:
|
|
return self.head_num * self.v_head_dim * self.store_dtype.itemsize
|
|
|
|
def host_entry_bytes(self) -> int:
|
|
"""Host (target-only) bytes for one slot, before any draft parts."""
|
|
return self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
|
|
|
|
def draft_offset_in_entry(self) -> int:
|
|
"""Byte offset of the fused draft parts inside one slot's entry."""
|
|
assert self.draft_region is not None
|
|
return align_part_offset(self.host_entry_bytes())
|
|
|
|
def entry_bytes(self) -> int:
|
|
if self.draft_region is None:
|
|
return align_entry_bytes(self.host_entry_bytes())
|
|
return align_entry_bytes(
|
|
self.draft_offset_in_entry() + self.draft_region.entry_bytes()
|
|
)
|
|
|
|
def page_bytes(self, page_size: int) -> int:
|
|
return page_size * self.entry_bytes()
|
|
|
|
def layout(self) -> DenseEntryLayout:
|
|
"""Token-major entry ``[K_0 | V_0 | K_1 | V_1 | ...]`` per slot; a page
|
|
is ``page_size`` such entries back to back."""
|
|
_check_row_alignment(self.name, K=self.k_row_bytes(), V=self.v_row_bytes())
|
|
layer_stride = self.k_row_bytes() + self.v_row_bytes()
|
|
parts = (
|
|
DensePart(
|
|
name="k",
|
|
offset_bytes=0,
|
|
layer_stride_bytes=layer_stride,
|
|
layer_num=self.layer_num,
|
|
row_shape=(self.head_num, self.head_dim),
|
|
dtype=self.store_dtype,
|
|
),
|
|
DensePart(
|
|
name="v",
|
|
offset_bytes=self.k_row_bytes(),
|
|
layer_stride_bytes=layer_stride,
|
|
layer_num=self.layer_num,
|
|
row_shape=(self.head_num, self.v_head_dim),
|
|
dtype=self.store_dtype,
|
|
),
|
|
)
|
|
if self.draft_region is not None:
|
|
parts += self.draft_region.parts(self.draft_offset_in_entry())
|
|
return DenseEntryLayout(entry_bytes=self.entry_bytes(), parts=parts)
|
|
|
|
def get_dtype(self) -> torch.dtype:
|
|
return self.store_dtype
|
|
|
|
|
|
@dataclass(frozen=True, kw_only=True)
|
|
class MLASubPoolSpec(SubPoolSpec):
|
|
"""Per-slot layout of one MLA-shaped sub-pool.
|
|
|
|
One latent row (``kv_lora_rank + qk_rope_head_dim``) per token per layer; V
|
|
is a prefix slice of the same row, so there is no separate V region. Not a
|
|
subclass of ``MHASubPoolSpec`` — the K+V byte math and the ``v_head_dim > 0``
|
|
invariant there do not apply. With `draft_region` set, the draft's K and V
|
|
parts follow the latent rows inside every slot's entry, as on an MHA host.
|
|
"""
|
|
|
|
kv_lora_rank: int
|
|
qk_rope_head_dim: int
|
|
store_dtype: torch.dtype
|
|
|
|
def __post_init__(self):
|
|
super().__post_init__()
|
|
assert self.kv_lora_rank > 0, (
|
|
f"kv_lora_rank must be positive; got {self.kv_lora_rank}"
|
|
)
|
|
assert self.qk_rope_head_dim > 0, (
|
|
f"qk_rope_head_dim must be positive; got {self.qk_rope_head_dim}"
|
|
)
|
|
if self.draft_region is not None:
|
|
assert isinstance(self.draft_region, DenseDraftRegion), (
|
|
"dense entries carry dense draft K/V rows, never a state region"
|
|
)
|
|
self.draft_region.validate()
|
|
|
|
@property
|
|
def kv_cache_dim(self) -> int:
|
|
return self.kv_lora_rank + self.qk_rope_head_dim
|
|
|
|
def row_bytes(self) -> int:
|
|
return self.kv_cache_dim * self.store_dtype.itemsize
|
|
|
|
def host_entry_bytes(self) -> int:
|
|
"""Host (target-only) bytes for one slot, before any draft parts."""
|
|
return self.layer_num * self.row_bytes()
|
|
|
|
def draft_offset_in_entry(self) -> int:
|
|
"""Byte offset of the fused draft parts inside one slot's entry."""
|
|
assert self.draft_region is not None
|
|
return align_part_offset(self.host_entry_bytes())
|
|
|
|
def entry_bytes(self) -> int:
|
|
if self.draft_region is None:
|
|
return align_entry_bytes(self.host_entry_bytes())
|
|
return align_entry_bytes(
|
|
self.draft_offset_in_entry() + self.draft_region.entry_bytes()
|
|
)
|
|
|
|
def layout(self) -> DenseEntryLayout:
|
|
_check_row_alignment(self.name, latent=self.row_bytes())
|
|
parts = (
|
|
DensePart(
|
|
name="kv",
|
|
offset_bytes=0,
|
|
layer_stride_bytes=self.row_bytes(),
|
|
layer_num=self.layer_num,
|
|
row_shape=(1, self.kv_cache_dim),
|
|
dtype=self.store_dtype,
|
|
),
|
|
)
|
|
if self.draft_region is not None:
|
|
parts += self.draft_region.parts(self.draft_offset_in_entry())
|
|
return DenseEntryLayout(entry_bytes=self.entry_bytes(), parts=parts)
|
|
|
|
def get_dtype(self) -> torch.dtype:
|
|
return self.store_dtype
|
|
|
|
|
|
@dataclass(frozen=True, kw_only=True)
|
|
class MambaSubPoolSpec(SubPoolSpec):
|
|
"""Per-slot layout of one Mamba-shaped sub-pool.
|
|
|
|
With `draft_region` (a `DraftStateRegion`) set, the draft's state block
|
|
follows the host's inside every slot's entry:
|
|
|
|
[ conv[0] x L | ... | temporal x L | draft conv[0] x Ld | ... | pad ]
|
|
|
|
Unfused, the entry keeps its raw (unaligned) size, byte-identical to before.
|
|
"""
|
|
|
|
conv_state_shapes: Tuple[Tuple[int, ...], ...] # one shape per conv tensor
|
|
conv_dtype: torch.dtype
|
|
temporal_state_shape: Tuple[int, ...]
|
|
temporal_dtype: torch.dtype
|
|
conv_slice_axis: int = 0
|
|
|
|
def __post_init__(self):
|
|
super().__post_init__()
|
|
assert len(self.conv_state_shapes) > 0, "conv_state_shapes must be non-empty"
|
|
if self.draft_region is not None:
|
|
assert isinstance(self.draft_region, DraftStateRegion), (
|
|
"mamba state entries carry a draft STATE region, never dense K/V rows"
|
|
)
|
|
self.draft_region.validate()
|
|
|
|
def conv_row_bytes(self, idx: int) -> int:
|
|
return _prod(self.conv_state_shapes[idx]) * self.conv_dtype.itemsize
|
|
|
|
def temporal_row_bytes(self) -> int:
|
|
return _prod(self.temporal_state_shape) * self.temporal_dtype.itemsize
|
|
|
|
def host_entry_bytes(self) -> int:
|
|
"""Host (target-only) bytes for one slot, before any draft block."""
|
|
total = 0
|
|
for i in range(len(self.conv_state_shapes)):
|
|
total += self.layer_num * self.conv_row_bytes(i)
|
|
total += self.layer_num * self.temporal_row_bytes()
|
|
return total
|
|
|
|
def draft_offset_in_entry(self) -> int:
|
|
"""Byte offset of the fused draft state block inside one slot's entry."""
|
|
assert self.draft_region is not None
|
|
return align_part_offset(self.host_entry_bytes())
|
|
|
|
def entry_bytes(self) -> int:
|
|
if self.draft_region is None:
|
|
return self.host_entry_bytes()
|
|
return align_entry_bytes(
|
|
self.draft_offset_in_entry() + self.draft_region.entry_bytes()
|
|
)
|
|
|
|
def get_dtype(self) -> torch.dtype:
|
|
return self.conv_dtype # representative state dtype; matches MambaPool.dtype
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# UnifiedKVPool — the byte buffer + the per-sub-pool views
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _assert_physical_id_bound(*, sub_pool_name: str, n_rows: int) -> None:
|
|
"""Physical ids must fit the int32 read-index buffers."""
|
|
assert n_rows < 2**31, (
|
|
f"sub-pool {sub_pool_name!r}: {n_rows} physical ids exceed the int32 "
|
|
f"bound (2^31) that read-index buffers assume. "
|
|
"Reduce max_total_num_tokens."
|
|
)
|
|
|
|
|
|
def _reserved_floor_bytes(sub_pool_specs: List[SubPoolSpec], page_size: int) -> int:
|
|
"""Bytes at the bottom of the buffer reserved as the slot-0 padding sink.
|
|
|
|
Slot-0 dummy writes for every sub-pool land here; each sub-pool's first
|
|
allocatable slot is chosen so real data starts past it. A slot-0 write
|
|
touches only slot 0's entry, but a PAGE-AWARE sub-pool allocates whole
|
|
pages, so its first allocatable page starts past all of page 0
|
|
(page_size * entry_bytes) -- while a mamba sub-pool is page_size=1, so its
|
|
entry is charged ONCE. Charging a mamba entry per page would reserve
|
|
page_size * ~100 MB of buffer that the sink never touches.
|
|
|
|
Single source of truth: `UnifiedKVPool` reserves exactly this, and the
|
|
factories' bs=1 feasibility floors charge exactly this.
|
|
"""
|
|
return max(
|
|
[max(s.entry_bytes() for s in sub_pool_specs)]
|
|
+ [
|
|
page_size * s.entry_bytes()
|
|
for s in sub_pool_specs
|
|
if not isinstance(s, MambaSubPoolSpec) # mamba is page_size=1
|
|
]
|
|
)
|
|
|
|
|
|
class UnifiedKVPool:
|
|
"""One physical `uint8` byte buffer shared by N sub-pools, each exposing
|
|
per-layer strided views over its own byte range. Two END pools (one
|
|
grow-up, one grow-down) own the buffer's ends; optional "float" MIDDLE
|
|
pools live between their frontiers. Allocators keep byte ranges disjoint;
|
|
no usage tracking here.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
total_bytes: int,
|
|
sub_pool_specs: List[SubPoolSpec],
|
|
device: str,
|
|
enable_memory_saver: bool,
|
|
page_size: int = 1,
|
|
post_capture_active: bool = False,
|
|
bs1_floor_terms: Optional[List[Tuple[str, int]]] = None,
|
|
fused_draft: Optional[FusedDraftPlacement] = None,
|
|
):
|
|
assert page_size >= 1, f"page_size must be >= 1; got {page_size}"
|
|
assert len(sub_pool_specs) >= 2, (
|
|
f"UnifiedKVPool needs >= 2 sub-pools; got {len(sub_pool_specs)}"
|
|
)
|
|
names = [s.name for s in sub_pool_specs]
|
|
assert len(set(names)) == len(names), (
|
|
f"sub-pool names must be unique; got {names}"
|
|
)
|
|
# Per-spec direction validity already ran in each spec's __post_init__.
|
|
up_specs = [s for s in sub_pool_specs if s.grow_direction == "up"]
|
|
down_specs = [s for s in sub_pool_specs if s.grow_direction == "down"]
|
|
float_specs = [s for s in sub_pool_specs if s.grow_direction == "float"]
|
|
assert len(up_specs) == 1 and len(down_specs) == 1, (
|
|
f"UnifiedKVPool needs exactly one grow-up and one grow-down END "
|
|
f"sub-pool; got directions "
|
|
f"{[s.grow_direction for s in sub_pool_specs]}"
|
|
)
|
|
|
|
self.device = device
|
|
self.total_bytes = total_bytes
|
|
# Canonical chain order, low byte end -> high: grow-up end, float
|
|
# middles (input order preserved), grow-down end. The allocators'
|
|
# neighbour wiring follows it; all other access is by-name.
|
|
self.sub_pool_specs: List[SubPoolSpec] = [
|
|
up_specs[0],
|
|
*float_specs,
|
|
down_specs[0],
|
|
]
|
|
self._page_size = page_size
|
|
self._specs_by_name: Dict[str, SubPoolSpec] = {
|
|
s.name: s for s in sub_pool_specs
|
|
}
|
|
self.post_capture_active = post_capture_active
|
|
self._post_capture_owner: Optional[KvVmmBufferOwner] = None
|
|
self._bs1_floor_terms = bs1_floor_terms or []
|
|
self.fused_draft = fused_draft
|
|
for spec in sub_pool_specs:
|
|
expected = None if fused_draft is None else fused_draft.region(spec.name)
|
|
assert spec.draft_region is expected, (
|
|
f"sub-pool {spec.name!r}: draft_region {spec.draft_region} does "
|
|
f"not match the fused draft placement's {expected}"
|
|
)
|
|
|
|
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
|
|
enable=enable_memory_saver
|
|
)
|
|
# Slot-0 dummy writes for both pools land in the reserved low-byte sink;
|
|
# each pool's first allocatable slot is chosen so real data starts past it.
|
|
# A page-aware sub-pool allocates whole pages, so the sink is all of page
|
|
# 0 (up to page_size * entry_bytes), not one slot entry -- reserve the
|
|
# max of both.
|
|
reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size)
|
|
self._reserved_floor = reserved_floor
|
|
|
|
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
|
|
if post_capture_active:
|
|
dev = torch.device(device)
|
|
if dev.type != "cuda":
|
|
raise ValueError(
|
|
"UnifiedKVPool post-capture backing requires a CUDA device"
|
|
)
|
|
device_id = (
|
|
dev.index if dev.index is not None else torch.cuda.current_device()
|
|
)
|
|
self._post_capture_owner = KvVmmBufferOwner(
|
|
device=device,
|
|
device_id=device_id,
|
|
store_dtype=torch.uint8,
|
|
page_size=1,
|
|
reserved_num_tokens=total_bytes,
|
|
buffer_descs=[
|
|
KvBufferDesc(
|
|
"unified_raw",
|
|
(total_bytes,),
|
|
row_bytes=1,
|
|
tokens_per_row=1,
|
|
)
|
|
],
|
|
)
|
|
self._raw = self._post_capture_owner.tensors[0]
|
|
# Capture routes every dummy KV write to slot 0, whose page
|
|
# envelope is the whole reserved sink: back it before any graph
|
|
# is captured.
|
|
self._post_capture_owner.ensure_prefix(reserved_floor)
|
|
accessible_raw = self._raw[:reserved_floor]
|
|
self._accessible_bytes = reserved_floor
|
|
else:
|
|
self._raw = torch.empty(total_bytes, dtype=torch.uint8, device=device)
|
|
accessible_raw = self._raw
|
|
self._accessible_bytes = total_bytes
|
|
self._initialize_raw(accessible_raw)
|
|
|
|
self._max_slots: Dict[str, int] = {}
|
|
self._anchor_bytes: Dict[str, int] = {}
|
|
self._min_slot_index: Dict[str, int] = {}
|
|
# MHA: (k_buffer, v_buffer); MLA: [per-layer views];
|
|
# Mamba: (conv_state_list, temporal_state)
|
|
self._mha_views: Dict[str, Tuple[List[torch.Tensor], List[torch.Tensor]]] = {}
|
|
self._mla_views: Dict[str, List[torch.Tensor]] = {}
|
|
self._mamba_views: Dict[str, Tuple[List[torch.Tensor], torch.Tensor]] = {}
|
|
|
|
for spec in self.sub_pool_specs:
|
|
entry_bytes = spec.entry_bytes()
|
|
max_slots = total_bytes // entry_bytes
|
|
min_slot_index = (reserved_floor + entry_bytes - 1) // entry_bytes # ceil
|
|
if max_slots <= min_slot_index:
|
|
raise RuntimeError(
|
|
f"UnifiedKVPool: sub-pool {spec.name!r} fits only {max_slots} "
|
|
f"slots in {total_bytes} bytes, but min_slot_index={min_slot_index} "
|
|
f"leaves no room for real data. Increase total_bytes."
|
|
)
|
|
anchor = 0
|
|
self._max_slots[spec.name] = max_slots
|
|
self._anchor_bytes[spec.name] = anchor
|
|
self._min_slot_index[spec.name] = min_slot_index
|
|
if isinstance(spec, MHASubPoolSpec):
|
|
self._mha_views[spec.name] = self._build_mha_views(
|
|
spec,
|
|
anchor,
|
|
max_slots,
|
|
page_size=page_size,
|
|
)
|
|
elif isinstance(spec, MLASubPoolSpec):
|
|
self._mla_views[spec.name] = self._build_mla_views(
|
|
spec,
|
|
anchor,
|
|
max_slots,
|
|
page_size=page_size,
|
|
)
|
|
elif isinstance(spec, MambaSubPoolSpec):
|
|
self._mamba_views[spec.name] = self._build_mamba_views(
|
|
spec, anchor, max_slots
|
|
)
|
|
else: # pragma: no cover
|
|
raise TypeError(f"unsupported SubPoolSpec type: {type(spec)}")
|
|
|
|
logger.info(
|
|
"[unified-memory-pool] UnifiedKVPool %s: total_bytes=%.2f GB (=%d B), "
|
|
"%d sub-pool(s)",
|
|
"reserved VA upper bound" if post_capture_active else "allocated",
|
|
total_bytes / GB,
|
|
total_bytes,
|
|
len(self.sub_pool_specs),
|
|
)
|
|
for s in self.sub_pool_specs:
|
|
logger.info(
|
|
"[unified-memory-pool] sub-pool %r: kind=%s, layer_num=%d, grow=%s, "
|
|
"entry_bytes=%d, max_slots=%d, min_slot_index=%d (slots [0,%d) reserved)",
|
|
s.name,
|
|
type(s).__name__,
|
|
s.layer_num,
|
|
s.grow_direction,
|
|
s.entry_bytes(),
|
|
self._max_slots[s.name],
|
|
self._min_slot_index[s.name],
|
|
self._min_slot_index[s.name],
|
|
)
|
|
|
|
def _initialize_raw(self, raw: torch.Tensor) -> None:
|
|
if envs.SGLANG_DEBUG_POISON_POOL.get():
|
|
# Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes
|
|
# fail deterministically.
|
|
raw.view(torch.int16).fill_(0x7FC1)
|
|
raw[: self._reserved_floor].zero_()
|
|
logger.warning(
|
|
"[unified-memory-pool] POISONED: backed pool bytes filled with "
|
|
"bf16-NaN patterns (SGLANG_DEBUG_POISON_POOL)"
|
|
)
|
|
else:
|
|
raw.zero_() # unset slots must read as zeros (matches non-shared)
|
|
|
|
def _max_slots_for_total_bytes(self, total_bytes: int) -> Dict[str, int]:
|
|
max_slots_by_name = {}
|
|
for spec in self.sub_pool_specs:
|
|
max_slots = total_bytes // spec.entry_bytes()
|
|
min_page_index = (
|
|
self._min_slot_index[spec.name] + self._page_size - 1
|
|
) // self._page_size
|
|
if max_slots // self._page_size <= min_page_index:
|
|
raise ValueError(
|
|
f"unified pool size {total_bytes} leaves no allocatable page "
|
|
f"for sub-pool {spec.name!r} after its reserved slot-0 sink"
|
|
)
|
|
max_slots_by_name[spec.name] = max_slots
|
|
return max_slots_by_name
|
|
|
|
@property
|
|
def post_capture_backed_bytes(self) -> int:
|
|
if self._post_capture_owner is None:
|
|
return 0
|
|
return self._post_capture_owner.backed_bytes
|
|
|
|
@property
|
|
def active_allocation_bytes(self) -> int:
|
|
"""Bytes that are safe to access/register in the shared raw tensor."""
|
|
return self._accessible_bytes
|
|
|
|
def finalize_backing(self, config) -> None:
|
|
"""Back the final shared byte budget without changing any tensor address."""
|
|
if self._post_capture_owner is None:
|
|
raise RuntimeError("UnifiedKVPool post-capture backing is not active")
|
|
final_total_bytes = config.unified_memory_pool_bytes
|
|
if final_total_bytes is None:
|
|
raise ValueError(
|
|
"UnifiedKVPool post-capture sizing requires "
|
|
"config.unified_memory_pool_bytes"
|
|
)
|
|
final_total_bytes = int(final_total_bytes)
|
|
_check_bs1_feasibility_floor(
|
|
total_bytes=final_total_bytes,
|
|
floor_terms=self._bs1_floor_terms,
|
|
factory="UnifiedKVPool.finalize_backing",
|
|
)
|
|
final_max_slots = self._max_slots_for_total_bytes(final_total_bytes)
|
|
self._post_capture_owner.ensure_prefix(final_total_bytes)
|
|
self._initialize_raw(self._raw[:final_total_bytes])
|
|
self.total_bytes = final_total_bytes
|
|
self._accessible_bytes = final_total_bytes
|
|
self._max_slots = final_max_slots
|
|
|
|
# -- introspection --
|
|
|
|
def spec(self, name: str) -> SubPoolSpec:
|
|
return self._specs_by_name[name]
|
|
|
|
def mha_spec(self, name: str) -> MHASubPoolSpec:
|
|
s = self._specs_by_name[name]
|
|
assert isinstance(s, MHASubPoolSpec), (
|
|
f"sub-pool {name!r} is {type(s).__name__}, expected MHASubPoolSpec"
|
|
)
|
|
return s
|
|
|
|
def mla_spec(self, name: str) -> MLASubPoolSpec:
|
|
s = self._specs_by_name[name]
|
|
assert isinstance(s, MLASubPoolSpec), (
|
|
f"sub-pool {name!r} is {type(s).__name__}, expected MLASubPoolSpec"
|
|
)
|
|
return s
|
|
|
|
def mamba_spec(self, name: str) -> MambaSubPoolSpec:
|
|
s = self._specs_by_name[name]
|
|
assert isinstance(s, MambaSubPoolSpec), (
|
|
f"sub-pool {name!r} is {type(s).__name__}, expected MambaSubPoolSpec"
|
|
)
|
|
return s
|
|
|
|
def require_draft_host_spec(self, name: str) -> SubPoolSpec:
|
|
"""The sub-pool spec whose entries carry a fused draft region.
|
|
|
|
Kind-agnostic: any spec that resolves a `draft_region` also lays the
|
|
draft parts out in its `layout()`."""
|
|
s = self._specs_by_name[name]
|
|
assert s.draft_region is not None, (
|
|
f"sub-pool {name!r} carries no fused draft region"
|
|
)
|
|
return s
|
|
|
|
def max_slots(self, name: str) -> int:
|
|
return self._max_slots[name]
|
|
|
|
def min_slot_index(self, name: str) -> int:
|
|
return self._min_slot_index[name]
|
|
|
|
def anchor_bytes(self, name: str) -> int:
|
|
anchor = self._anchor_bytes[name]
|
|
assert anchor == 0, f"current design assumes all anchors are 0; got {anchor}"
|
|
return anchor
|
|
|
|
def mha_views_for(self, name: str) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
|
return self._mha_views[name]
|
|
|
|
def mla_views_for(self, name: str) -> List[torch.Tensor]:
|
|
return self._mla_views[name]
|
|
|
|
def mamba_views_for(self, name: str) -> Tuple[List[torch.Tensor], torch.Tensor]:
|
|
return self._mamba_views[name]
|
|
|
|
def _build_mha_views(
|
|
self,
|
|
spec: MHASubPoolSpec,
|
|
anchor_bytes: int,
|
|
max_slots: int,
|
|
page_size: int,
|
|
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
|
num_slots = max_slots // page_size * page_size
|
|
_assert_physical_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
|
|
layout = spec.layout()
|
|
k_views, v_views = (
|
|
build_dense_views(
|
|
self._raw,
|
|
layout=layout,
|
|
part_name=name,
|
|
num_slots=num_slots,
|
|
anchor_bytes=anchor_bytes,
|
|
)
|
|
for name in ("k", "v")
|
|
)
|
|
return k_views, v_views
|
|
|
|
def build_dense_draft_views(
|
|
self, sub_pool_name: str
|
|
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
|
"""Per-layer K/V views of the DRAFT parts fused into ``sub_pool_name``'s
|
|
entries: same pages, same slot ids, same v2p table as the host."""
|
|
spec = self.require_draft_host_spec(sub_pool_name)
|
|
layout = spec.layout()
|
|
page_size = self._page_size
|
|
num_slots = self.max_slots(sub_pool_name) // page_size * page_size
|
|
k_views, v_views = (
|
|
build_dense_views(
|
|
self._raw,
|
|
layout=layout,
|
|
part_name=name,
|
|
num_slots=num_slots,
|
|
anchor_bytes=self._anchor_bytes[sub_pool_name],
|
|
)
|
|
for name in ("draft_k", "draft_v")
|
|
)
|
|
return k_views, v_views
|
|
|
|
def _build_mla_views(
|
|
self,
|
|
spec: MLASubPoolSpec,
|
|
anchor_bytes: int,
|
|
max_slots: int,
|
|
page_size: int,
|
|
) -> List[torch.Tensor]:
|
|
num_slots = max_slots // page_size * page_size
|
|
_assert_physical_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
|
|
layout = spec.layout()
|
|
return build_dense_views(
|
|
self._raw,
|
|
layout=layout,
|
|
part_name="kv",
|
|
num_slots=num_slots,
|
|
anchor_bytes=anchor_bytes,
|
|
)
|
|
|
|
def _build_mamba_views(
|
|
self, spec: MambaSubPoolSpec, anchor_bytes: int, max_slots: int
|
|
) -> Tuple[List[torch.Tensor], torch.Tensor]:
|
|
return build_mamba_entry_views(
|
|
self._raw,
|
|
layer_num=spec.layer_num,
|
|
conv_state_shapes=spec.conv_state_shapes,
|
|
conv_dtype=spec.conv_dtype,
|
|
temporal_state_shape=spec.temporal_state_shape,
|
|
temporal_dtype=spec.temporal_dtype,
|
|
max_slots=max_slots,
|
|
anchor_bytes=anchor_bytes,
|
|
entry_bytes=spec.entry_bytes(),
|
|
)
|
|
|
|
def build_draft_state_views(
|
|
self, sub_pool_name: str
|
|
) -> Tuple[List[torch.Tensor], torch.Tensor]:
|
|
"""Per-layer conv/temporal views of the DRAFT state block fused into
|
|
``sub_pool_name``'s entries: same slots, same v2p table as the host."""
|
|
spec = self.mamba_spec(sub_pool_name)
|
|
region = spec.draft_region
|
|
assert isinstance(region, DraftStateRegion), (
|
|
f"sub-pool {sub_pool_name!r} carries no fused draft state region"
|
|
)
|
|
return build_mamba_entry_views(
|
|
self._raw,
|
|
layer_num=region.lane_num,
|
|
conv_state_shapes=region.state.conv_state_shapes,
|
|
conv_dtype=region.state.conv_dtype,
|
|
temporal_state_shape=region.state.temporal_state_shape,
|
|
temporal_dtype=region.state.temporal_dtype,
|
|
max_slots=self.max_slots(sub_pool_name),
|
|
anchor_bytes=self._anchor_bytes[sub_pool_name],
|
|
entry_bytes=spec.entry_bytes(),
|
|
offset_bytes=spec.draft_offset_in_entry(),
|
|
)
|
|
|
|
|
|
class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
|
"""MHA KV pool whose per-layer `k_buffer`/`v_buffer` are token-major views
|
|
into a `UnifiedKVPool`: `(num_pages * ps, head_num, head_dim)` with the whole
|
|
entry as slot stride, indexed by the physical token id. `move_physical_pages`
|
|
relocates whole page envelopes.
|
|
"""
|
|
|
|
requires_physical_write_loc = True
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
unified_buffer: UnifiedKVPool,
|
|
sub_pool_name: str,
|
|
page_size: int = 1,
|
|
start_layer: Optional[int] = None,
|
|
end_layer: Optional[int] = None,
|
|
enable_alt_stream: bool = True,
|
|
):
|
|
spec = unified_buffer.mha_spec(sub_pool_name)
|
|
k_views, v_views = unified_buffer.mha_views_for(sub_pool_name)
|
|
max_slots = unified_buffer.max_slots(sub_pool_name)
|
|
|
|
self._unified_buffer = unified_buffer
|
|
self._sub_pool_name = sub_pool_name
|
|
self._k_views = k_views
|
|
self._v_views = v_views
|
|
self._num_pages = max_slots // page_size
|
|
self._page_bytes = page_size * spec.entry_bytes()
|
|
view_rows = self._num_pages * page_size
|
|
|
|
super().__init__(
|
|
size=view_rows - page_size,
|
|
page_size=page_size,
|
|
dtype=spec.kv_cache_dtype,
|
|
head_num=spec.head_num,
|
|
head_dim=spec.head_dim,
|
|
layer_num=spec.layer_num,
|
|
device=unified_buffer.device,
|
|
enable_memory_saver=False, # buffer owned by UnifiedKVPool
|
|
v_head_dim=spec.v_head_dim,
|
|
start_layer=start_layer,
|
|
end_layer=end_layer,
|
|
enable_alt_stream=enable_alt_stream,
|
|
enable_kv_cache_copy=False,
|
|
kv_cache_layout="page_major",
|
|
)
|
|
|
|
def _kv_tokens_per_row(self) -> int:
|
|
return 1
|
|
|
|
def _create_buffers(self):
|
|
self.k_buffer = self._k_views
|
|
self.v_buffer = self._v_views
|
|
# This override must initialize the pointers and row strides used by HiCache.
|
|
self._init_data_ptrs_and_strides()
|
|
|
|
def _clear_buffers(self):
|
|
# Lifetime owned by UnifiedKVPool; do not delete the views.
|
|
pass
|
|
|
|
def get_kv_size_bytes(self):
|
|
return 0, 0 # UnifiedKVPool logs the total; per-sub-pool would double-count
|
|
|
|
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
|
"""Relocate slots by whole page envelope (`tgt_loc`/`src_loc` are
|
|
physical token ids, in whole page-major runs)."""
|
|
if tgt_loc.numel() == 0:
|
|
return
|
|
ps = self.page_size
|
|
self.move_physical_pages(
|
|
tgt_loc.view(-1, ps)[:, 0] // ps, src_loc.view(-1, ps)[:, 0] // ps, ps
|
|
)
|
|
|
|
def move_physical_pages(
|
|
self, tgt_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
|
|
):
|
|
"""Copy whole page envelopes `src_pages` -> `tgt_pages`: disjoint sets of
|
|
physical page ids, in pages of `page_size` slots (compaction moves)."""
|
|
assert page_size == self.page_size, (
|
|
f"page ids of {page_size}-slot pages passed to a pool whose envelope "
|
|
f"holds {self.page_size} slots"
|
|
)
|
|
# The envelope view below starts at byte 0, so this sub-pool must be
|
|
# anchored there; a non-zero anchor moves another sub-pool's bytes, and
|
|
# the ids stay in range so nothing downstream notices.
|
|
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
|
with record_function("UnifiedMHA.move_physical_pages"):
|
|
copy_pages(
|
|
self._unified_buffer._raw,
|
|
tgt_pages,
|
|
src_pages,
|
|
self._num_pages,
|
|
self._page_bytes,
|
|
)
|
|
|
|
def zero_physical_pages(self, phys_pages: torch.Tensor) -> None:
|
|
"""Zero whole page envelopes (PHYSICAL page ids) on allocator
|
|
hand-out."""
|
|
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
|
zero_pages(
|
|
self._unified_buffer._raw,
|
|
phys_pages,
|
|
self._num_pages,
|
|
self._page_bytes,
|
|
)
|
|
|
|
def get_contiguous_buf_infos(self):
|
|
"""Register the raw buffer as physical page envelopes for PD transfer.
|
|
|
|
Full and SWA expose the same allocation with different envelope sizes;
|
|
the transfer backend preserves both logical entries while deduplicating
|
|
the underlying memory registration.
|
|
"""
|
|
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
|
raw = self._unified_buffer._raw
|
|
return (
|
|
[raw.data_ptr()],
|
|
[self._unified_buffer.active_allocation_bytes],
|
|
[self._page_bytes],
|
|
)
|
|
|
|
def get_page_envelope_buffer(self) -> torch.Tensor:
|
|
"""Return this sub-pool's page-strided view of the shared allocation."""
|
|
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
|
raw = self._unified_buffer._raw
|
|
page_count = (
|
|
self._unified_buffer.max_slots(self._sub_pool_name) // self.page_size
|
|
)
|
|
return raw[: page_count * self._page_bytes].view(page_count, self._page_bytes)
|
|
|
|
@property
|
|
def grow_direction(self) -> str:
|
|
"""Physical growth direction used by this sub-pool's L1 allocator."""
|
|
return self._unified_buffer.spec(self._sub_pool_name).grow_direction
|
|
|
|
def set_kv_buffer_prefix_valid(self, *args, **kwargs):
|
|
raise NotImplementedError(
|
|
"prefix-valid commit is unsupported under the unified layout "
|
|
"(_set_kv_buffer_prefix_valid_impl assumes token-id indexing)."
|
|
)
|
|
|
|
|
|
class UnifiedMLATokenToKVPool(MLATokenToKVPool):
|
|
"""MLA KV pool whose per-layer `kv_buffer` entries are token-major views
|
|
into a `UnifiedKVPool`: `(num_pages * ps, 1, kv_cache_dim)` with the whole
|
|
entry as slot stride, indexed by the physical token id (`translate_kv_loc`
|
|
of the virtual id), so the stock `MLATokenToKVPool` read/write methods work
|
|
on them unmodified. `move_physical_pages` relocates whole page envelopes.
|
|
"""
|
|
|
|
requires_physical_write_loc = True
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
unified_buffer: UnifiedKVPool,
|
|
sub_pool_name: str,
|
|
kv_cache_dtype: torch.dtype,
|
|
page_size: int = 1,
|
|
):
|
|
spec = unified_buffer.mla_spec(sub_pool_name)
|
|
store_dtype = _store_dtype_for(kv_cache_dtype)
|
|
assert spec.store_dtype == store_dtype, (
|
|
f"sub-pool {sub_pool_name!r} store dtype {spec.store_dtype} does not "
|
|
f"match kv cache dtype {kv_cache_dtype} (store {store_dtype})"
|
|
)
|
|
|
|
self._unified_buffer = unified_buffer
|
|
self._sub_pool_name = sub_pool_name
|
|
self._kv_views = unified_buffer.mla_views_for(sub_pool_name)
|
|
max_slots = unified_buffer.max_slots(sub_pool_name)
|
|
self._num_pages = max_slots // page_size
|
|
self._page_bytes = page_size * spec.entry_bytes()
|
|
self._view_rows = self._num_pages * page_size
|
|
|
|
super().__init__(
|
|
# OOB checks bound locs by `size + page_size`; physical ids run to
|
|
# `_view_rows` (page 0 is the reserved padding sink).
|
|
size=self._view_rows - page_size,
|
|
page_size=page_size,
|
|
dtype=kv_cache_dtype,
|
|
kv_lora_rank=spec.kv_lora_rank,
|
|
qk_rope_head_dim=spec.qk_rope_head_dim,
|
|
layer_num=spec.layer_num,
|
|
device=unified_buffer.device,
|
|
enable_memory_saver=False, # buffer owned by UnifiedKVPool
|
|
)
|
|
|
|
def _create_buffers(self):
|
|
self.kv_buffer = self._kv_views
|
|
|
|
def _clear_buffers(self):
|
|
# Lifetime owned by UnifiedKVPool; do not delete the views.
|
|
pass
|
|
|
|
# The plan's write translation (`translate_write_loc`) already collapsed
|
|
# the widened id and sent the rows this rank does not own to the padding
|
|
# sink.
|
|
write_loc_is_dcp_resolved = True
|
|
|
|
def get_kv_size_bytes(self):
|
|
return 0 # UnifiedKVPool logs the total; per-sub-pool would double-count
|
|
|
|
def get_contiguous_buf_infos(self):
|
|
"""PD-transfer registration: ONE entry, the raw buffer, addressed as
|
|
``raw_ptr + physical_page_id * page_envelope_bytes``.
|
|
|
|
The transfer item is the whole page envelope (all layers of one page)
|
|
rather than a per-layer region: the layers interleave inside each
|
|
slot's entry. Both sides must therefore build the pool with identical
|
|
specs.
|
|
"""
|
|
# The address formula omits the anchor; a nonzero one would mis-address.
|
|
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
|
raw = self._unified_buffer._raw
|
|
return [raw.data_ptr()], [raw.numel()], [self._page_bytes]
|
|
|
|
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
|
"""Relocate whole page envelopes.
|
|
|
|
`tgt_loc`/`src_loc` are physical token ids in whole page-major runs
|
|
(`pages[:, None] * ps + offsets`), relied on here to recover the page
|
|
lists.
|
|
"""
|
|
if tgt_loc.numel() == 0:
|
|
return
|
|
ps = self.page_size
|
|
self.move_physical_pages(
|
|
tgt_loc.view(-1, ps)[:, 0] // ps, src_loc.view(-1, ps)[:, 0] // ps, ps
|
|
)
|
|
|
|
def move_physical_pages(
|
|
self, tgt_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
|
|
):
|
|
"""Copy whole page envelopes `src_pages` -> `tgt_pages`: disjoint sets of
|
|
physical page ids, in pages of `page_size` slots (compaction moves). One
|
|
contiguous envelope copy replaces the per-layer strided moves."""
|
|
assert page_size == self.page_size, (
|
|
f"page ids of {page_size}-slot pages passed to a pool whose envelope "
|
|
f"holds {self.page_size} slots"
|
|
)
|
|
with record_function("UnifiedMLA.move_physical_pages"):
|
|
copy_pages(
|
|
self._unified_buffer._raw,
|
|
tgt_pages,
|
|
src_pages,
|
|
self._num_pages,
|
|
self._page_bytes,
|
|
)
|
|
|
|
def zero_physical_pages(self, phys_pages: torch.Tensor) -> None:
|
|
"""Zero whole page envelopes (PHYSICAL page ids) on allocator
|
|
hand-out."""
|
|
zero_pages(
|
|
self._unified_buffer._raw,
|
|
phys_pages,
|
|
self._num_pages,
|
|
self._page_bytes,
|
|
)
|
|
|
|
|
|
class UnifiedMambaPool(MambaPool):
|
|
"""Mamba state pool whose conv/temporal state are strided views into a `UnifiedKVPool`.
|
|
|
|
Pure PHYSICAL store: slot lifecycle and the v<->p mapping live in the attached
|
|
`UnifiedMambaSlotAllocator`. Does NOT call `super().__init__()` — replicates the
|
|
minimal `MambaPool` state against the unified buffer so inherited methods work.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
unified_buffer: UnifiedKVPool,
|
|
sub_pool_name: str,
|
|
spec_state_size: int,
|
|
mamba_layer_ids: List[int],
|
|
enable_memory_saver: bool = False,
|
|
speculative_num_draft_tokens: Optional[int] = None,
|
|
):
|
|
spec = unified_buffer.mamba_spec(sub_pool_name)
|
|
assert spec.layer_num == len(mamba_layer_ids)
|
|
# PP disagg state transfer maps entries by global layer id.
|
|
self.mamba_layer_ids = list(mamba_layer_ids)
|
|
conv_views, temporal_view = unified_buffer.mamba_views_for(sub_pool_name)
|
|
max_slots = unified_buffer.max_slots(sub_pool_name)
|
|
|
|
self._unified_buffer = unified_buffer
|
|
self._sub_pool_name = sub_pool_name
|
|
|
|
# Replicate the state MambaPool.__init__ would have set.
|
|
self._max_size = max_slots - 1 # -1 for reserved slot 0
|
|
self.size = self._max_size
|
|
self.device = unified_buffer.device
|
|
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
|
|
enable=enable_memory_saver
|
|
)
|
|
self.enable_custom_mem_pool = False
|
|
self.custom_mem_pool = None
|
|
self.num_mamba_layers = spec.layer_num
|
|
# GDN/KDA ReplaySSM / spec unsupported; replicate parent's disabled-state
|
|
# attrs so unconditional reads (e.g. `replayssm_cache_base is not None` in
|
|
# the req-slot alloc path) and `... is not None` guards don't AttributeError.
|
|
self.enable_linear_replayssm = False
|
|
self.linear_replayssm_cache_len = 16
|
|
self.replayssm_write_pos = None
|
|
self.replayssm_is_kda = False
|
|
self.enable_linear_replayssm_spec = False
|
|
self.replayssm_spec_fold = False
|
|
self.replayssm_spec_write_pos = None
|
|
self.replayssm_cache_base = None
|
|
self.replayssm_is_flush = None
|
|
self.debug_memory_pool = False
|
|
self.conv_shard_groups = None
|
|
self.conv_slice_axis = spec.conv_slice_axis
|
|
|
|
assert conv_views[0].shape[0] == self.num_mamba_layers, (
|
|
f"conv_views layers={conv_views[0].shape[0]} vs expected {self.num_mamba_layers}"
|
|
)
|
|
assert conv_views[0].shape[1] == self._max_size + 1, (
|
|
f"conv_views slots={conv_views[0].shape[1]} vs expected {self._max_size + 1}"
|
|
)
|
|
|
|
# Per-draft-token intermediate buffers have a different outer size
|
|
# (spec_state_size+1), so they're NOT in the shared buffer; allocate locally.
|
|
temporal_state_shape = spec.temporal_state_shape
|
|
conv_state_shape = spec.conv_state_shapes
|
|
conv_dtype = spec.conv_dtype
|
|
ssm_dtype = spec.temporal_dtype
|
|
if speculative_num_draft_tokens is not None:
|
|
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
|
|
intermediate_ssm_state_cache = torch.zeros(
|
|
size=(
|
|
self.num_mamba_layers,
|
|
spec_state_size + 1,
|
|
speculative_num_draft_tokens,
|
|
temporal_state_shape[0],
|
|
temporal_state_shape[1],
|
|
temporal_state_shape[2],
|
|
),
|
|
dtype=ssm_dtype,
|
|
device=unified_buffer.device,
|
|
)
|
|
intermediate_conv_window_cache = [
|
|
torch.zeros(
|
|
size=(
|
|
self.num_mamba_layers,
|
|
spec_state_size + 1,
|
|
speculative_num_draft_tokens,
|
|
cshape[0],
|
|
cshape[1],
|
|
),
|
|
dtype=conv_dtype,
|
|
device=unified_buffer.device,
|
|
)
|
|
for cshape in conv_state_shape
|
|
]
|
|
self.mamba_cache = self.SpeculativeState(
|
|
conv=list(conv_views),
|
|
temporal=temporal_view,
|
|
intermediate_ssm=intermediate_ssm_state_cache,
|
|
intermediate_conv_window=intermediate_conv_window_cache,
|
|
)
|
|
else:
|
|
self.mamba_cache = self.State(conv=list(conv_views), temporal=temporal_view)
|
|
|
|
self.mem_usage = unified_buffer.total_bytes / GB
|
|
logger.info(
|
|
"[unified-memory-pool] UnifiedMambaPool(%s) wrapped unified buffer: max_slots=%d, "
|
|
"num_mamba_layers=%d",
|
|
sub_pool_name,
|
|
max_slots,
|
|
self.num_mamba_layers,
|
|
)
|
|
|
|
# State ops take PHYSICAL slot ids and act on whole entries, so a fused
|
|
# draft state block is cleared, copied and moved with the host streams.
|
|
|
|
def _entry_view(self) -> torch.Tensor:
|
|
spec = self._unified_buffer.mamba_spec(self._sub_pool_name)
|
|
entry_bytes = spec.entry_bytes()
|
|
anchor = self._unified_buffer.anchor_bytes(self._sub_pool_name)
|
|
num_slots = self._max_size + 1
|
|
raw = self._unified_buffer._raw
|
|
entries = raw[anchor : anchor + num_slots * entry_bytes]
|
|
return entries.view(num_slots, entry_bytes)
|
|
|
|
def clear_slots(self, indices: torch.Tensor):
|
|
self._entry_view()[indices] = 0
|
|
for sibling in self._slot_siblings:
|
|
sibling.reset_slots(indices)
|
|
|
|
def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor):
|
|
assert self.replayssm_write_pos is None
|
|
entries = self._entry_view()
|
|
entries[dst_indices] = entries[src_indices]
|
|
for sibling in self._slot_siblings:
|
|
sibling.copy_slots(src_indices, dst_indices)
|
|
|
|
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
|
# Compaction moves disjoint physical slots. Copy their whole envelopes
|
|
# without the conv/temporal gather temporary used by copy_from (COW).
|
|
# Sibling callbacks retain their own copy and scratch-allocation contracts.
|
|
if tgt_loc.numel() == 0:
|
|
return
|
|
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
|
copy_pages(
|
|
self._unified_buffer._raw,
|
|
tgt_loc,
|
|
src_loc,
|
|
self._unified_buffer.max_slots(self._sub_pool_name),
|
|
self._unified_buffer.mamba_spec(self._sub_pool_name).entry_bytes(),
|
|
)
|
|
if self.replayssm_write_pos is not None:
|
|
self.replayssm_write_pos[tgt_loc] = 0
|
|
for sibling in self._slot_siblings:
|
|
sibling.copy_slots(src_loc, tgt_loc)
|
|
|
|
def move_physical_pages(
|
|
self, tgt_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
|
|
):
|
|
# Cross-pool physical-move contract, implemented by every pool the
|
|
# MultiEndedAllocator wraps. A state slot is one page.
|
|
assert page_size == 1, f"mamba state pages are single slots, got {page_size}"
|
|
self.move_kv_cache(tgt_pages, src_pages)
|
|
|
|
# -- PD state transfer (StateType.MAMBA) --
|
|
# The transfer item is the whole per-slot envelope, addressed as
|
|
# `raw_ptr + physical_slot * entry_bytes`. An envelope cannot be TP-resliced
|
|
# or PP-subset, so the per-tensor metadata below stays empty and both sides
|
|
# must build identical mamba specs (equal attn TP, pp=1).
|
|
|
|
def get_contiguous_buf_infos(self):
|
|
# The address formula omits the anchor; a nonzero one would mis-address.
|
|
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
|
spec = self._unified_buffer.mamba_spec(self._sub_pool_name)
|
|
raw = self._unified_buffer._raw
|
|
return [raw.data_ptr()], [raw.numel()], [spec.entry_bytes()]
|
|
|
|
def get_state_dim_per_tensor(self):
|
|
return []
|
|
|
|
def get_state_layer_ids(self):
|
|
return []
|
|
|
|
def get_state_slice_outer_counts(self):
|
|
return []
|
|
|
|
def get_state_conv_shard_groups(self):
|
|
return []
|
|
|
|
|
|
class UnifiedMambaSlotAllocator:
|
|
"""Mamba slot allocator (PHYSICAL view) for the unified memory pool.
|
|
|
|
Owns slot alloc/free, sizing, and the v<->p mapping (``translate``), presenting the
|
|
upstream ``MambaSlotAllocator`` interface. ``alloc()`` returns VIRTUAL ids and does
|
|
NOT clear state — clearing is deferred to ``UnifiedMambaPool.clear_slots``.
|
|
"""
|
|
|
|
def __init__(self, mea, max_size: int, device: str):
|
|
self._multi_ended_allocator = mea
|
|
self._max_size = max_size # excludes reserved slot 0
|
|
self._device = device
|
|
self._alloc_iter = None # active alloc_group batch iterator
|
|
|
|
# -- translation (owns the v<->p mapping) --
|
|
|
|
def translate(self, virtual_ids: torch.Tensor) -> torch.Tensor:
|
|
# VIRTUAL -> PHYSICAL slot ids; page_size==1, so a direct v2p gather.
|
|
return self._multi_ended_allocator.virtual_to_physical[virtual_ids]
|
|
|
|
@property
|
|
def virtual_to_physical(self) -> torch.Tensor:
|
|
return self._multi_ended_allocator.virtual_to_physical
|
|
|
|
# -- sizing / free-list --
|
|
|
|
@property
|
|
def size(self) -> int:
|
|
return self._max_size
|
|
|
|
def available_size(self) -> int:
|
|
# Slot-conservation count (max - allocated): the leak-check view, NOT the
|
|
# planner value (use schedulable_available_size for that).
|
|
return self._max_size - self._multi_ended_allocator.allocated_count()
|
|
|
|
def schedulable_available_size(self) -> int:
|
|
# Byte-coordinated count (>= N => alloc(N) succeeds); credits the peer's
|
|
# drainable holes since alloc flushes the peer before extending.
|
|
return self._multi_ended_allocator.schedulable_available_size()
|
|
|
|
@property
|
|
def free_slots(self) -> torch.Tensor:
|
|
# Watermark-derived physical free-list for the invariant checker.
|
|
a = self._multi_ended_allocator
|
|
assert a.page_size == 1, (
|
|
"UnifiedMambaSlotAllocator.free_slots assumes page_size==1; got "
|
|
f"{a.page_size}. Mamba state is per-request, orthogonal to paging."
|
|
)
|
|
if a.grow_direction == "up":
|
|
start, end = a.watermark_physical, a.num_pages
|
|
else:
|
|
start, end = a.min_page_index, a.watermark_physical + 1
|
|
if start >= end:
|
|
return torch.empty((0,), dtype=torch.int64, device=self._device)
|
|
return torch.arange(start, end, dtype=torch.int64, device=self._device)
|
|
|
|
# -- slot management (delegates to the MultiEndedAllocator) --
|
|
|
|
def alloc(self, need_size: int):
|
|
# alloc_group fast path: single-slot draws from the prefetched batch.
|
|
if self._alloc_iter is not None and need_size == 1:
|
|
slot = next(self._alloc_iter, None)
|
|
if slot is not None:
|
|
return slot
|
|
return self._multi_ended_allocator.alloc(need_size) # VIRTUAL ids
|
|
|
|
def free(self, free_index: torch.Tensor):
|
|
return self._multi_ended_allocator.free(free_index)
|
|
|
|
def clear(self):
|
|
self._alloc_iter = None
|
|
return self._multi_ended_allocator.clear()
|
|
|
|
def alloc_group_begin(self, num_reqs: int):
|
|
"""Pre-allocate a batch that ``alloc(1)`` then draws from."""
|
|
self._alloc_iter = None
|
|
if num_reqs > 0:
|
|
result = self._multi_ended_allocator.alloc(num_reqs)
|
|
if result is not None:
|
|
self._alloc_iter = iter(result.split(1))
|
|
|
|
def alloc_group_end(self):
|
|
"""Return any unused pre-allocated slots from the current group."""
|
|
if self._alloc_iter is not None:
|
|
remaining = list(self._alloc_iter)
|
|
if remaining:
|
|
self._multi_ended_allocator.free(torch.cat(remaining))
|
|
self._alloc_iter = None
|
|
|
|
def is_slot_allocated(self, slot) -> bool:
|
|
return self._multi_ended_allocator.is_slot_allocated(int(slot))
|
|
|
|
def allocator_state_str(self) -> str:
|
|
return self._multi_ended_allocator.allocator_state_str()
|
|
|
|
|
|
class UnifiedHybridReqToTokenPool(HybridReqToTokenPool):
|
|
"""`HybridReqToTokenPool` whose `mamba_pool` is a `UnifiedMambaPool`. The inherited
|
|
mamba-id state now holds VIRTUAL ids; adds `translate_mamba_indices` for v->p."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
unified_buffer: UnifiedKVPool,
|
|
mamba_sub_pool_name: str,
|
|
size: int,
|
|
mamba_spec_state_size: int,
|
|
max_context_len: int,
|
|
device: str,
|
|
enable_memory_saver: bool,
|
|
cache_params,
|
|
mamba_layer_ids: List[int],
|
|
enable_mamba_extra_buffer: bool,
|
|
enable_mamba_extra_buffer_lazy: bool,
|
|
speculative_num_draft_tokens: Optional[int] = None,
|
|
enable_overlap_schedule: bool = True,
|
|
start_layer: Optional[int] = None,
|
|
pre_alloc_size: int = 0,
|
|
):
|
|
self._unified_buffer = unified_buffer
|
|
self._mamba_sub_pool_name = mamba_sub_pool_name
|
|
self._shared_mamba_size = (
|
|
unified_buffer.max_slots(mamba_sub_pool_name) - 1
|
|
) # reserve slot 0
|
|
super().__init__(
|
|
# `DecodeReqToTokenPool` semantics: rows cover the preallocated
|
|
# requests too, while `self.size` (rebound below) stays the
|
|
# running-request cap the scheduler and leak invariant expect.
|
|
size=size + pre_alloc_size,
|
|
mamba_size=self._shared_mamba_size,
|
|
mamba_spec_state_size=mamba_spec_state_size,
|
|
max_context_len=max_context_len,
|
|
device=device,
|
|
enable_memory_saver=enable_memory_saver,
|
|
cache_params=cache_params,
|
|
mamba_layer_ids=mamba_layer_ids,
|
|
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
|
|
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
|
|
speculative_num_draft_tokens=speculative_num_draft_tokens,
|
|
enable_overlap_schedule=enable_overlap_schedule,
|
|
start_layer=start_layer,
|
|
)
|
|
self.size = size
|
|
self.pre_alloc_size = pre_alloc_size
|
|
|
|
def _init_mamba_pool(
|
|
self,
|
|
mamba_size: int,
|
|
mamba_spec_state_size: int,
|
|
cache_params,
|
|
mamba_layer_ids: List[int],
|
|
device: str,
|
|
enable_mamba_extra_buffer: bool,
|
|
speculative_num_draft_tokens: Optional[int] = None,
|
|
speculative_eagle_topk: Optional[int] = None,
|
|
mamba_envelope_layout: bool = False,
|
|
enable_linear_replayssm: bool = False,
|
|
linear_replayssm_cache_len: int = 16,
|
|
enable_linear_replayssm_spec: bool = False,
|
|
short_conv_layer_ids: Optional[List[int]] = None,
|
|
short_conv_state_shape=None,
|
|
ngram_context_len: int = 0,
|
|
ngram_eos_token_id: int = 0,
|
|
):
|
|
# mamba_envelope_layout / speculative_eagle_topk / enable_linear_replayssm /
|
|
# linear_replayssm_cache_len / enable_linear_replayssm_spec: accepted to match
|
|
# the parent signature but NOT forwarded — the shared pool's conv/temporal
|
|
# state are fixed-shape views (replayssm/spec are gated off under unified).
|
|
if short_conv_layer_ids or ngram_context_len:
|
|
raise ValueError(
|
|
"Qwen4-Exp PLE side states are not supported with "
|
|
"--enable-unified-memory"
|
|
)
|
|
from sglang.srt.mem_cache.ple_state_pool import NGramPool, ShortConvPool
|
|
|
|
self.short_conv_pool = ShortConvPool(
|
|
size=0,
|
|
state_shape=None,
|
|
layer_ids=[],
|
|
dtype=torch.bfloat16,
|
|
device=device,
|
|
)
|
|
self.ngram_pool = NGramPool(
|
|
size=0,
|
|
context_len=0,
|
|
eos_token_id=0,
|
|
device=device,
|
|
)
|
|
assert mamba_size == self._shared_mamba_size, (
|
|
f"UnifiedHybridReqToTokenPool._init_mamba_pool: mamba_size={mamba_size} "
|
|
f"!= unified_buffer.max_slots({self._mamba_sub_pool_name!r}) - 1 "
|
|
f"= {self._shared_mamba_size}"
|
|
)
|
|
assert len(cache_params.layers) >= len(mamba_layer_ids), (
|
|
f"cache_params.layers ({len(cache_params.layers)}) cannot supply "
|
|
f"{len(mamba_layer_ids)} mamba layer ids"
|
|
)
|
|
self.mamba_pool = UnifiedMambaPool(
|
|
unified_buffer=self._unified_buffer,
|
|
sub_pool_name=self._mamba_sub_pool_name,
|
|
spec_state_size=mamba_spec_state_size,
|
|
mamba_layer_ids=mamba_layer_ids,
|
|
enable_memory_saver=self.enable_memory_saver,
|
|
speculative_num_draft_tokens=speculative_num_draft_tokens,
|
|
)
|
|
# Wired in by init_unified_mamba_pools once the mamba allocator exists.
|
|
self.mamba_allocator = None
|
|
self.mamba_map = {layer_id: i for i, layer_id in enumerate(mamba_layer_ids)}
|
|
self.mamba_ckpt_pool = None # int8 ckpt pool unused; None = feature off
|
|
self.device = device
|
|
# Sized by req_to_token's first dim (size + 1; row 0 is padding); self.size
|
|
# would under-size by one row.
|
|
req_pool_size = self.req_to_token.shape[0]
|
|
self.req_index_to_mamba_index_mapping: torch.Tensor = torch.zeros(
|
|
req_pool_size, dtype=torch.int32, device=self.device
|
|
)
|
|
if enable_mamba_extra_buffer:
|
|
self.req_index_to_mamba_ping_pong_track_buffer_mapping: torch.Tensor = (
|
|
torch.zeros(
|
|
(req_pool_size, self.mamba_ping_pong_track_buffer_size),
|
|
# int64 to match the parent's uncast index_put source (int32 dest
|
|
# would dtype-mismatch on the first radix prefill).
|
|
dtype=torch.int64,
|
|
device=self.device,
|
|
)
|
|
)
|
|
|
|
def clone_for_fused_draft(
|
|
self, *, layer_lanes: Dict[int, int]
|
|
) -> UnifiedHybridReqToTokenPool:
|
|
"""Shallow copy for a draft runner whose recurrent state is fused into
|
|
this pool's state entries: it shares the request->slot mappings and
|
|
the slot allocator, and its `mamba_pool` is a view of the draft block
|
|
keyed by the runner's own layer ids."""
|
|
from sglang.srt.mem_cache.unified_draft_pool import UnifiedDraftMambaPool
|
|
|
|
clone = copy.copy(self)
|
|
clone.mamba_pool = UnifiedDraftMambaPool(
|
|
unified_buffer=self._unified_buffer,
|
|
sub_pool_name=self._mamba_sub_pool_name,
|
|
layer_lanes=layer_lanes,
|
|
)
|
|
clone.mamba_map = {
|
|
layer_id: idx for idx, layer_id in enumerate(sorted(layer_lanes))
|
|
}
|
|
clone.mamba_ckpt_pool = None
|
|
return clone
|
|
|
|
@property
|
|
def mamba_v2p_table(self) -> Optional[torch.Tensor]:
|
|
"""This pool's ids ARE virtual; page_size is 1, so the translate is the
|
|
plain gather this table serves, which keeps `mamba_translate_is_fusable`
|
|
true despite the override."""
|
|
if self.mamba_allocator is None:
|
|
return None
|
|
return self.mamba_allocator.virtual_to_physical
|
|
|
|
def translate_mamba_indices(self, virtual_ids: torch.Tensor) -> torch.Tensor:
|
|
"""Virtual mamba ids -> physical slot ids."""
|
|
return self.mamba_allocator.translate(virtual_ids).to(torch.int32)
|
|
|
|
|
|
class UnifiedHybridLinearKVPool(HybridLinearKVPool):
|
|
"""`HybridLinearKVPool` over unified sub-pools (full = Unified{MLA,MHA},
|
|
mamba = UnifiedMambaPool)."""
|
|
|
|
def get_kv_layer_ids(self):
|
|
# Empty: the KV component is one whole-envelope entry, so there are no
|
|
# per-layer entries to pair by layer id (the sender falls back to
|
|
# positional pairing).
|
|
return []
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Factory
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class UnifiedPoolBundle(NamedTuple):
|
|
unified_memory_pool: UnifiedKVPool
|
|
token_to_kv_pool: object # HybridLinearKVPool
|
|
token_to_kv_pool_allocator: object # UnifiedMambaTokenToKVPoolAllocator
|
|
req_to_token_pool: object # UnifiedHybridReqToTokenPool
|
|
|
|
|
|
def _check_bs1_feasibility_floor(
|
|
*,
|
|
total_bytes: int,
|
|
floor_terms: List[Tuple[str, int]],
|
|
factory: str,
|
|
) -> None:
|
|
"""bs=1 feasibility FLOOR — the retract loop's terminal guarantee.
|
|
|
|
The scheduler retracts requests until the LAST one fits; if one worst-case
|
|
request running ALONE does not fit in the buffer, under-sizing is a retract
|
|
LIVELOCK at runtime, not a perf bug. Fail loud at boot, before any pool
|
|
construction, with the itemized requirement.
|
|
"""
|
|
floor = sum(b for _, b in floor_terms)
|
|
if total_bytes >= floor:
|
|
return
|
|
detail = " + ".join(f"{name}={b}" for name, b in floor_terms)
|
|
raise RuntimeError(
|
|
f"[unified-memory-pool] {factory}: byte budget {total_bytes} cannot fit "
|
|
f"ONE worst-case request (bs=1 floor {floor} = {detail}). A pool this "
|
|
f"size retract-livelocks at runtime. Raise --mem-fraction-static, lower "
|
|
f"the model context length, or reduce reserved memory."
|
|
)
|
|
|
|
|
|
def _wire_mamba_slot_allocator(
|
|
*,
|
|
mamba_end,
|
|
req_to_token_pool,
|
|
device,
|
|
) -> UnifiedMambaSlotAllocator:
|
|
"""Install Mamba slot allocation, host capacities, and transfer translation."""
|
|
slot_allocator = UnifiedMambaSlotAllocator(
|
|
mamba_end,
|
|
max_size=req_to_token_pool._shared_mamba_size,
|
|
device=device,
|
|
)
|
|
req_to_token_pool.mamba_allocator = slot_allocator
|
|
state_pool = req_to_token_pool.mamba_pool
|
|
state_pool.host_transfer_translate = slot_allocator.translate
|
|
state_pool.host_capacity_tokens = req_to_token_pool._shared_mamba_size
|
|
state_pool.host_capacity_bytes = (
|
|
state_pool.host_capacity_tokens * mamba_end.entry_bytes
|
|
)
|
|
return slot_allocator
|
|
|
|
|
|
def init_unified_mamba_pools(
|
|
*,
|
|
device: str,
|
|
kv_cache_dtype: torch.dtype,
|
|
head_num: int,
|
|
head_dim: int,
|
|
page_size: int,
|
|
start_layer: int,
|
|
end_layer: int,
|
|
is_draft_worker: bool,
|
|
use_mla_backend: bool,
|
|
v_head_dim: Optional[int] = None,
|
|
kv_lora_rank: Optional[int] = None,
|
|
qk_rope_head_dim: Optional[int] = None,
|
|
fused_draft: Optional[FusedDraftPlacement] = None,
|
|
mamba_layer_ids: List[int],
|
|
full_attention_layer_ids: List[int],
|
|
mamba2_cache_params,
|
|
model_context_len: int,
|
|
extra_max_context_len: int,
|
|
max_total_num_tokens: int,
|
|
max_mamba_cache_size: int,
|
|
max_num_reqs: int,
|
|
enable_memory_saver: bool,
|
|
enable_mamba_extra_buffer: bool,
|
|
enable_mamba_extra_buffer_lazy: bool,
|
|
speculative_num_draft_tokens: Optional[int],
|
|
disable_overlap_schedule: bool,
|
|
need_sort: bool,
|
|
mamba_full_memory_ratio: Optional[float] = None, # informational only
|
|
forward_stream: Optional[torch.cuda.Stream] = None,
|
|
lazy_compaction: bool = False,
|
|
decode_pre_alloc_size: int = 0,
|
|
unified_total_bytes: Optional[int] = None,
|
|
) -> UnifiedPoolBundle:
|
|
"""Build the Mamba-hybrid unified-memory-pool stack."""
|
|
from sglang.srt.mem_cache.allocator.unified_mamba import (
|
|
UnifiedMambaTokenToKVPoolAllocator,
|
|
)
|
|
|
|
# Full sub-pool is page-aware; mamba stays page=1 (state is per-request).
|
|
assert page_size >= 1, f"page_size must be >= 1, got {page_size}"
|
|
|
|
store_dtype = _store_dtype_for(kv_cache_dtype)
|
|
# full-attn at the high-byte end (grow-down), mamba at the low-byte end (grow-up).
|
|
if use_mla_backend:
|
|
assert kv_lora_rank and qk_rope_head_dim, (
|
|
"init_unified_mamba_pools: MLA-hybrid-Mamba needs kv_lora_rank and "
|
|
f"qk_rope_head_dim; got {kv_lora_rank} / {qk_rope_head_dim}"
|
|
)
|
|
assert not is_draft_worker, (
|
|
"init_unified_mamba_pools: a draft worker binds views over the "
|
|
"target's buffer instead of building one of its own"
|
|
)
|
|
full_spec = MLASubPoolSpec(
|
|
name="full",
|
|
layer_num=len(full_attention_layer_ids),
|
|
kv_lora_rank=kv_lora_rank,
|
|
qk_rope_head_dim=qk_rope_head_dim,
|
|
store_dtype=store_dtype,
|
|
grow_direction="down",
|
|
draft_region=None if fused_draft is None else fused_draft.region("full"),
|
|
)
|
|
else:
|
|
full_spec = MHASubPoolSpec(
|
|
name="full",
|
|
layer_num=len(full_attention_layer_ids),
|
|
head_num=head_num,
|
|
head_dim=head_dim,
|
|
v_head_dim=v_head_dim,
|
|
store_dtype=store_dtype,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
grow_direction="down",
|
|
draft_region=None if fused_draft is None else fused_draft.region("full"),
|
|
)
|
|
cp = mamba2_cache_params
|
|
mamba_spec = MambaSubPoolSpec(
|
|
name="mamba",
|
|
layer_num=len(mamba_layer_ids),
|
|
conv_state_shapes=tuple(tuple(int(x) for x in s) for s in cp.shape.conv),
|
|
conv_dtype=cp.dtype.conv,
|
|
temporal_state_shape=tuple(int(x) for x in cp.shape.temporal),
|
|
temporal_dtype=cp.dtype.temporal,
|
|
conv_slice_axis=getattr(cp.shape, "conv_slice_axis", 0),
|
|
grow_direction="up",
|
|
draft_region=None if fused_draft is None else fused_draft.region("mamba"),
|
|
)
|
|
if unified_total_bytes is not None:
|
|
# PROFILED byte budget for the token side (captured pre-ratio-floor);
|
|
# the state pool's bytes ride on top. The token counts stay boot
|
|
# labels / conserve caps -- the runtime split floats.
|
|
total_bytes = (
|
|
unified_total_bytes + max_mamba_cache_size * mamba_spec.entry_bytes()
|
|
)
|
|
else:
|
|
total_bytes = (
|
|
max_total_num_tokens * full_spec.entry_bytes()
|
|
+ max_mamba_cache_size * mamba_spec.entry_bytes()
|
|
)
|
|
# bs=1 floor: the state slots one running request locks (1 active + 2 radix
|
|
# checkpoints, a FLOOR not headroom) + the slot-0 sink. The token side is
|
|
# not charged -- `TpModelWorker.get_worker_info` already clamps max_req_len
|
|
# to the pool, so a too-long request is refused at admission, not livelocked.
|
|
_check_bs1_feasibility_floor(
|
|
total_bytes=total_bytes,
|
|
floor_terms=[
|
|
("bs1_state_slots", 3 * mamba_spec.entry_bytes()),
|
|
("sink", _reserved_floor_bytes([full_spec, mamba_spec], page_size)),
|
|
],
|
|
factory="init_unified_mamba_pools",
|
|
)
|
|
shared_pool = UnifiedKVPool(
|
|
total_bytes=total_bytes,
|
|
sub_pool_specs=[full_spec, mamba_spec],
|
|
device=device,
|
|
enable_memory_saver=enable_memory_saver,
|
|
page_size=page_size,
|
|
fused_draft=fused_draft,
|
|
)
|
|
req_to_token_pool = UnifiedHybridReqToTokenPool(
|
|
unified_buffer=shared_pool,
|
|
mamba_sub_pool_name="mamba",
|
|
size=max_num_reqs,
|
|
mamba_spec_state_size=max_num_reqs, # outer dim of spec-decode intermediates
|
|
max_context_len=model_context_len + extra_max_context_len,
|
|
device=device,
|
|
enable_memory_saver=enable_memory_saver,
|
|
cache_params=mamba2_cache_params,
|
|
mamba_layer_ids=mamba_layer_ids,
|
|
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
|
|
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
|
|
speculative_num_draft_tokens=speculative_num_draft_tokens,
|
|
enable_overlap_schedule=not disable_overlap_schedule,
|
|
start_layer=start_layer,
|
|
pre_alloc_size=decode_pre_alloc_size,
|
|
)
|
|
if use_mla_backend:
|
|
# start_layer stays 0: HybridLinearKVPool patches layer ids to the contiguous
|
|
# 0..N-1 index via _transfer_id_context before every MLA pool call.
|
|
unified_full_kv_pool = UnifiedMLATokenToKVPool(
|
|
unified_buffer=shared_pool,
|
|
sub_pool_name="full",
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
page_size=page_size,
|
|
)
|
|
else:
|
|
unified_full_kv_pool = UnifiedMHATokenToKVPool(
|
|
unified_buffer=shared_pool,
|
|
sub_pool_name="full",
|
|
page_size=page_size,
|
|
start_layer=start_layer,
|
|
end_layer=end_layer,
|
|
)
|
|
full_attn_layer_ids_for_pool = (
|
|
[0] if is_draft_worker else list(full_attention_layer_ids)
|
|
)
|
|
token_to_kv_pool = UnifiedHybridLinearKVPool(
|
|
page_size=page_size,
|
|
size=max_total_num_tokens,
|
|
dtype=kv_cache_dtype,
|
|
head_num=head_num,
|
|
head_dim=head_dim,
|
|
full_attention_layer_ids=full_attn_layer_ids_for_pool,
|
|
device=device,
|
|
mamba_pool=req_to_token_pool.mamba_pool,
|
|
enable_memory_saver=enable_memory_saver,
|
|
use_mla=use_mla_backend,
|
|
start_layer=start_layer,
|
|
full_kv_pool=unified_full_kv_pool,
|
|
)
|
|
allocator = UnifiedMambaTokenToKVPoolAllocator(
|
|
unified_buffer=shared_pool,
|
|
kvcache=token_to_kv_pool,
|
|
device=device,
|
|
page_size=page_size,
|
|
need_sort=need_sort,
|
|
forward_stream=forward_stream,
|
|
lazy_compaction=lazy_compaction,
|
|
)
|
|
# Size host storage from the configured token cap, not the dynamic buffer view.
|
|
full_pool = token_to_kv_pool.full_kv_pool
|
|
full_pool.host_capacity_tokens = max_total_num_tokens
|
|
full_pool.host_capacity_bytes = (
|
|
max_total_num_tokens * allocator.full_attn_allocator.entry_bytes
|
|
)
|
|
|
|
mamba_slot_allocator = _wire_mamba_slot_allocator(
|
|
mamba_end=allocator.mamba_allocator,
|
|
req_to_token_pool=req_to_token_pool,
|
|
device=device,
|
|
)
|
|
# Only HybridLinearKVPool's retraction CPU-copy path uses this hook.
|
|
token_to_kv_pool._mamba_translate = mamba_slot_allocator.translate
|
|
# No full-KV translate hook is wired: both MLA doors now receive
|
|
# PHYSICAL ids -- writes from the ForwardBatch rebind, reads
|
|
# translated at their production sites.
|
|
|
|
logger.info(
|
|
"[unified-memory-pool] ============================================================"
|
|
)
|
|
logger.info(
|
|
"[unified-memory-pool] UNIFIED MEMORY POOL ENABLED -- path=Mamba hybrid (%s full side)",
|
|
"MLA" if use_mla_backend else "MHA",
|
|
)
|
|
if use_mla_backend:
|
|
logger.info(
|
|
"[unified-memory-pool] full_layers=%d, mamba_layers=%d, kv_lora_rank=%d, "
|
|
"qk_rope_head_dim=%d, page_size=%d (token-major views, entry_bytes=%d B)",
|
|
len(full_attention_layer_ids),
|
|
len(mamba_layer_ids),
|
|
kv_lora_rank,
|
|
qk_rope_head_dim,
|
|
page_size,
|
|
full_spec.entry_bytes(),
|
|
)
|
|
else:
|
|
logger.info(
|
|
"[unified-memory-pool] full_layers=%d, mamba_layers=%d, head_num=%d, head_dim=%d, "
|
|
"page_size=%d, is_draft_worker=%s (%s)",
|
|
len(full_attention_layer_ids),
|
|
len(mamba_layer_ids),
|
|
head_num,
|
|
head_dim,
|
|
page_size,
|
|
is_draft_worker,
|
|
"token-major views, entry_bytes=%d B" % full_spec.entry_bytes(),
|
|
)
|
|
logger.info(
|
|
"[unified-memory-pool] total_bytes=%d, max_total_num_tokens=%d, max_mamba_cache_size=%d, "
|
|
"max_num_reqs=%d, speculative_num_draft_tokens=%s",
|
|
total_bytes,
|
|
max_total_num_tokens,
|
|
max_mamba_cache_size,
|
|
max_num_reqs,
|
|
speculative_num_draft_tokens,
|
|
)
|
|
if mamba_full_memory_ratio is not None:
|
|
logger.info(
|
|
"[unified-memory-pool] mamba_full_memory_ratio=%s governs the total budget only, "
|
|
"not the runtime split.",
|
|
mamba_full_memory_ratio,
|
|
)
|
|
logger.info(
|
|
"[unified-memory-pool] ============================================================"
|
|
)
|
|
return UnifiedPoolBundle(
|
|
unified_memory_pool=shared_pool,
|
|
token_to_kv_pool=token_to_kv_pool,
|
|
token_to_kv_pool_allocator=allocator,
|
|
req_to_token_pool=req_to_token_pool,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# UnifiedSWAKVPool — hybrid SWA on the shared byte buffer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class UnifiedSWAKVPool(SWAKVPool):
|
|
"""Shared-buffer replacement for `SWAKVPool`.
|
|
|
|
Composes two `UnifiedMHATokenToKVPool` instances (full + swa) aliasing the same
|
|
byte buffer. Inherits from `SWAKVPool` only for `isinstance`; does NOT call the
|
|
parent `__init__` (it would build static-partition pools). The per-sub-pool v2p
|
|
table IS the full->swa mapping, so `register_mapping` is a no-op.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
unified_buffer: UnifiedKVPool,
|
|
swa_attention_layer_ids: List[int],
|
|
full_attention_layer_ids: List[int],
|
|
page_size: int = 1,
|
|
start_layer: Optional[int] = None,
|
|
end_layer: Optional[int] = None,
|
|
enable_memory_saver: bool = False,
|
|
):
|
|
# Do NOT call super().__init__ — it would allocate static-partition pools.
|
|
self.unified_buffer = unified_buffer
|
|
self.swa_layer_nums = len(swa_attention_layer_ids)
|
|
self.full_layer_nums = len(full_attention_layer_ids)
|
|
self.layer_num = self.full_layer_nums + self.swa_layer_nums
|
|
self.start_layer = start_layer if start_layer is not None else 0
|
|
self.page_size = page_size
|
|
self.layer_transfer_counter = None
|
|
|
|
self.size = unified_buffer.max_slots("full") - 1
|
|
self.size_swa = unified_buffer.max_slots("swa") - 1
|
|
|
|
full_spec = unified_buffer.mha_spec("full")
|
|
swa_spec = unified_buffer.mha_spec("swa")
|
|
assert full_spec.store_dtype == swa_spec.store_dtype, (
|
|
"UnifiedSWAKVPool: full and swa sub-pools must share store_dtype; got "
|
|
f"full={full_spec.store_dtype}, swa={swa_spec.store_dtype}"
|
|
)
|
|
assert full_spec.kv_cache_dtype == swa_spec.kv_cache_dtype, (
|
|
"UnifiedSWAKVPool: full and swa sub-pools must share kv_cache_dtype; got "
|
|
f"full={full_spec.kv_cache_dtype}, swa={swa_spec.kv_cache_dtype}"
|
|
)
|
|
self.dtype = full_spec.kv_cache_dtype
|
|
self.head_num = full_spec.head_num
|
|
self.head_dim = full_spec.head_dim
|
|
self.device = unified_buffer.device
|
|
|
|
self.full_kv_pool = UnifiedMHATokenToKVPool(
|
|
unified_buffer=unified_buffer,
|
|
sub_pool_name="full",
|
|
page_size=page_size,
|
|
start_layer=start_layer,
|
|
end_layer=end_layer,
|
|
)
|
|
self.swa_kv_pool = UnifiedMHATokenToKVPool(
|
|
unified_buffer=unified_buffer,
|
|
sub_pool_name="swa",
|
|
page_size=page_size,
|
|
start_layer=start_layer,
|
|
end_layer=end_layer,
|
|
)
|
|
|
|
self.full_kv_pool.model_layer_ids = tuple(full_attention_layer_ids)
|
|
self.swa_kv_pool.model_layer_ids = tuple(swa_attention_layer_ids)
|
|
|
|
# disagg/nvlink disabled; keep attrs present to avoid AttributeError.
|
|
self.enable_custom_mem_pool = False
|
|
self.custom_mem_pool = None
|
|
|
|
# {global_layer_id: (per-pool index, is_swa_layer)}
|
|
self.layers_mapping: Dict[int, Tuple[int, bool]] = {}
|
|
for idx, gid in enumerate(full_attention_layer_ids):
|
|
self.layers_mapping[gid] = (idx, False)
|
|
for idx, gid in enumerate(swa_attention_layer_ids):
|
|
self.layers_mapping[gid] = (idx, True)
|
|
|
|
# None so dispatch routes through our v2p-table overrides, not a registered mapping.
|
|
self.full_to_swa_index_mapping: Optional[torch.Tensor] = None
|
|
|
|
self.mem_usage = 0.0 # cosmetic; UnifiedKVPool logs the real size
|
|
|
|
# Wired in via attach_allocators.
|
|
self._full_allocator = None
|
|
self._swa_allocator = None
|
|
|
|
logger.info(
|
|
"[unified-memory-pool] UnifiedSWAKVPool wrapped unified buffer: "
|
|
"full_layers=%d (max_slots=%d), swa_layers=%d (max_slots=%d), "
|
|
"head_num=%d, head_dim=%d",
|
|
self.full_layer_nums,
|
|
unified_buffer.max_slots("full"),
|
|
self.swa_layer_nums,
|
|
unified_buffer.max_slots("swa"),
|
|
self.head_num,
|
|
self.head_dim,
|
|
)
|
|
|
|
# -- allocator wiring --
|
|
|
|
def attach_allocators(self, *, full_allocator, swa_allocator) -> None:
|
|
"""Wire the two `MultiEndedAllocator`s whose v2p tables translate slot ids."""
|
|
self._full_allocator = full_allocator
|
|
self._swa_allocator = swa_allocator
|
|
|
|
# -- BaseSWAKVPool ABC surface --
|
|
|
|
def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None:
|
|
return # no-op in shared mode (the swa-side v2p IS the mapping)
|
|
|
|
def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor):
|
|
"""Virtual token ids -> swa-physical token ids (int64)."""
|
|
assert self._swa_allocator is not None, (
|
|
"UnifiedSWAKVPool.translate_loc_from_full_to_swa called before "
|
|
"attach_allocators"
|
|
)
|
|
return self._swa_allocator.translate_kv_loc(kv_indices)
|
|
|
|
def get_state_buf_infos(self):
|
|
return self.swa_kv_pool.get_contiguous_buf_infos()
|
|
|
|
# -- size/info --
|
|
|
|
def get_kv_size_bytes(self):
|
|
return 0, 0 # UnifiedKVPool logs the total; per-side would double-count
|
|
|
|
@property
|
|
def post_capture_active(self) -> bool:
|
|
return self.unified_buffer.post_capture_active
|
|
|
|
@property
|
|
def post_capture_backed_bytes(self) -> int:
|
|
return self.unified_buffer.post_capture_backed_bytes
|
|
|
|
def finalize_backing(self, config) -> None:
|
|
self.unified_buffer.finalize_backing(config)
|
|
|
|
def get_contiguous_buf_infos(self):
|
|
return self.full_kv_pool.get_contiguous_buf_infos()
|
|
|
|
# -- buffer accessors --
|
|
|
|
def get_key_buffer(self, layer_id: int):
|
|
self._wait_for_layer(layer_id)
|
|
pool_layer_id, is_swa = self.layers_mapping[layer_id]
|
|
pool = self.swa_kv_pool if is_swa else self.full_kv_pool
|
|
return pool.get_key_buffer(pool_layer_id)
|
|
|
|
def get_value_buffer(self, layer_id: int):
|
|
self._wait_for_layer(layer_id)
|
|
pool_layer_id, is_swa = self.layers_mapping[layer_id]
|
|
pool = self.swa_kv_pool if is_swa else self.full_kv_pool
|
|
return pool.get_value_buffer(pool_layer_id)
|
|
|
|
def get_kv_buffer(self, layer_id: int):
|
|
self._wait_for_layer(layer_id)
|
|
pool_layer_id, is_swa = self.layers_mapping[layer_id]
|
|
pool = self.swa_kv_pool if is_swa else self.full_kv_pool
|
|
return pool.get_kv_buffer(pool_layer_id)
|
|
|
|
# -- kv writing --
|
|
|
|
def set_kv_buffer(
|
|
self,
|
|
layer,
|
|
loc_info,
|
|
cache_k: torch.Tensor,
|
|
cache_v: torch.Tensor,
|
|
k_scale: float = 1.0,
|
|
v_scale: float = 1.0,
|
|
):
|
|
"""Route to the right sub-pool. Both `swa_loc` and `full_loc` are PHYSICAL
|
|
(pre-translated once per forward by the attention backend); never translates here.
|
|
"""
|
|
loc, swa_loc, full_loc = unwrap_write_loc(loc_info)
|
|
physical = write_loc_is_physical(loc_info)
|
|
layer_id = layer.layer_id
|
|
pool_layer_id, is_swa = self.layers_mapping[layer_id]
|
|
if is_swa:
|
|
# swa_loc is ALREADY swa-physical. Routed through the UnifiedMHATokenToKVPool
|
|
# override (its 4-D layer-major view can't take the parent's view(-1, row_dim)).
|
|
assert swa_loc is not None, (
|
|
"UnifiedSWAKVPool.set_kv_buffer: SWA layer received no swa_loc; the "
|
|
"attention backend must bundle forward_metadata.swa_out_cache_loc."
|
|
)
|
|
self.swa_kv_pool.set_kv_buffer(
|
|
None,
|
|
KVWriteLoc(swa_loc, physical=physical),
|
|
cache_k,
|
|
cache_v,
|
|
k_scale,
|
|
v_scale,
|
|
layer_id_override=pool_layer_id,
|
|
)
|
|
return
|
|
# Full layer: `loc` is already the full-side physical id, so an
|
|
# explicit full_loc is a same-space alias -- only triton's captured path
|
|
# passes one (its capture-stable buffer).
|
|
if full_loc is None:
|
|
full_loc = loc
|
|
self.full_kv_pool.set_kv_buffer(
|
|
None,
|
|
KVWriteLoc(full_loc, physical=physical),
|
|
cache_k,
|
|
cache_v,
|
|
k_scale,
|
|
v_scale,
|
|
layer_id_override=pool_layer_id,
|
|
)
|
|
|
|
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
|
# Never called on the composite — compaction runs per-sub-pool via
|
|
# UnifiedMHATokenToKVPool.move_kv_cache.
|
|
raise NotImplementedError(
|
|
"UnifiedSWAKVPool.move_kv_cache should not be called; compaction "
|
|
"operates per-sub-pool via UnifiedMHATokenToKVPool.move_kv_cache."
|
|
)
|
|
|
|
# -- HiCache shims (translate virtual->physical, then delegate) --
|
|
|
|
@staticmethod
|
|
def _virt_tokens_to_phys_tokens(
|
|
virt_tokens: torch.Tensor, allocator
|
|
) -> torch.Tensor:
|
|
"""Virtual TOKEN ids -> physical TOKEN ids (page-aware). Unbound pages yield
|
|
negatives; callers filter via `swa_phys >= 0`."""
|
|
ps = allocator.page_size
|
|
if ps == 1:
|
|
return allocator.virtual_to_physical[virt_tokens]
|
|
virt_pages = virt_tokens // ps
|
|
offsets = virt_tokens % ps
|
|
phys_pages = allocator.virtual_to_physical[virt_pages]
|
|
return phys_pages * ps + offsets
|
|
|
|
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
|
|
assert self._full_allocator is not None
|
|
assert self._swa_allocator is not None
|
|
# `indices` are virtual TOKEN ids; translate per sub-pool.
|
|
full_phys = self._virt_tokens_to_phys_tokens(indices, self._full_allocator)
|
|
swa_phys = self._virt_tokens_to_phys_tokens(indices, self._swa_allocator)
|
|
full_cpu = self.full_kv_pool.get_cpu_copy(
|
|
full_phys, req_pool_index=req_pool_index
|
|
)
|
|
valid = swa_phys >= 0
|
|
swa_cpu = None
|
|
if bool(valid.any().item()):
|
|
swa_cpu = self.swa_kv_pool.get_cpu_copy(swa_phys[valid])
|
|
return {"full": full_cpu, "swa": swa_cpu, "swa_mask": valid.cpu()}
|
|
|
|
def load_cpu_copy(
|
|
self, kv_cache_cpu, indices, mamba_indices=None, req_pool_index=None
|
|
):
|
|
assert self._full_allocator is not None
|
|
full_phys = self._virt_tokens_to_phys_tokens(indices, self._full_allocator)
|
|
self.full_kv_pool.load_cpu_copy(
|
|
kv_cache_cpu["full"], full_phys, req_pool_index=req_pool_index
|
|
)
|
|
if kv_cache_cpu.get("swa") is not None:
|
|
assert self._swa_allocator is not None
|
|
swa_phys = self._virt_tokens_to_phys_tokens(indices, self._swa_allocator)
|
|
old_swa_mask = kv_cache_cpu["swa_mask"].to(indices.device)
|
|
assert old_swa_mask.shape == indices.shape
|
|
row_mask = (swa_phys >= 0)[old_swa_mask].cpu()
|
|
swa_phys = swa_phys[old_swa_mask][row_mask.to(indices.device)]
|
|
if swa_phys.numel() == 0:
|
|
return
|
|
swa_cpu = self._filter_swa_cpu_copy(
|
|
swa_kv_cpu=kv_cache_cpu["swa"], row_mask=row_mask
|
|
)
|
|
self.swa_kv_pool.load_cpu_copy(swa_cpu, swa_phys)
|
|
|
|
|
|
class UnifiedSWAPoolBundle(NamedTuple):
|
|
unified_memory_pool: UnifiedKVPool
|
|
token_to_kv_pool: object # UnifiedSWAKVPool
|
|
token_to_kv_pool_allocator: object # UnifiedSWATokenToKVPoolAllocator
|
|
|
|
|
|
def init_unified_swa_pools(
|
|
*,
|
|
device: str,
|
|
kv_cache_dtype: torch.dtype,
|
|
head_num: int,
|
|
head_dim: int,
|
|
v_head_dim: int,
|
|
swa_head_num: int,
|
|
swa_head_dim: int,
|
|
swa_v_head_dim: int,
|
|
page_size: int,
|
|
start_layer: int,
|
|
end_layer: int,
|
|
swa_attention_layer_ids: List[int],
|
|
full_attention_layer_ids: List[int],
|
|
full_max_total_num_tokens: Optional[int] = None,
|
|
swa_max_total_num_tokens: Optional[int] = None,
|
|
total_bytes: Optional[int] = None,
|
|
enable_memory_saver: bool,
|
|
post_capture_active: bool = False,
|
|
need_sort: bool,
|
|
forward_stream: Optional[torch.cuda.Stream] = None,
|
|
lazy_compaction: bool = False,
|
|
model_context_len: Optional[int] = None,
|
|
sliding_window_size: Optional[int] = None,
|
|
fused_draft: Optional[FusedDraftPlacement] = None,
|
|
) -> UnifiedSWAPoolBundle:
|
|
"""Build the SWA-hybrid unified-memory-pool stack.
|
|
|
|
With ``fused_draft``, every entry of a sub-pool the placement names
|
|
carries the draft model's K/V parts after the host parts, and each draft
|
|
runner binds a `UnifiedDraftKVPool` over its own lanes instead of
|
|
allocating a pool of its own.
|
|
"""
|
|
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
|
UnifiedSWATokenToKVPoolAllocator,
|
|
)
|
|
|
|
# Both sub-allocators are page-aware: one virtual ID space at PAGE granularity,
|
|
# two physical sub-pools compacting pages independently.
|
|
assert page_size >= 1, f"page_size must be >= 1, got {page_size}"
|
|
assert len(full_attention_layer_ids) > 0, (
|
|
"SWA-hybrid with zero full-attention layers is degenerate"
|
|
)
|
|
assert len(swa_attention_layer_ids) > 0, (
|
|
"SWA-hybrid with zero SWA-attention layers is degenerate"
|
|
)
|
|
|
|
store_dtype = _store_dtype_for(kv_cache_dtype)
|
|
# full-attn at the high-byte end (grow-down), swa at the low-byte end (grow-up).
|
|
full_spec = MHASubPoolSpec(
|
|
name="full",
|
|
layer_num=len(full_attention_layer_ids),
|
|
head_num=head_num,
|
|
head_dim=head_dim,
|
|
v_head_dim=v_head_dim,
|
|
store_dtype=store_dtype,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
grow_direction="down",
|
|
draft_region=None if fused_draft is None else fused_draft.region("full"),
|
|
)
|
|
swa_spec = MHASubPoolSpec(
|
|
name="swa",
|
|
layer_num=len(swa_attention_layer_ids),
|
|
head_num=swa_head_num,
|
|
head_dim=swa_head_dim,
|
|
v_head_dim=swa_v_head_dim,
|
|
store_dtype=store_dtype,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
grow_direction="up",
|
|
draft_region=None if fused_draft is None else fused_draft.region("swa"),
|
|
)
|
|
legacy_allocator_capacities = {}
|
|
if total_bytes is None:
|
|
if full_max_total_num_tokens is None or swa_max_total_num_tokens is None:
|
|
raise ValueError(
|
|
"total_bytes or both legacy full/SWA capacities must be provided"
|
|
)
|
|
total_bytes = (
|
|
full_max_total_num_tokens * full_spec.entry_bytes()
|
|
+ swa_max_total_num_tokens * swa_spec.entry_bytes()
|
|
)
|
|
legacy_allocator_capacities = {
|
|
"full_max_total_num_tokens": full_max_total_num_tokens,
|
|
"swa_max_total_num_tokens": swa_max_total_num_tokens,
|
|
}
|
|
bs1_floor_terms = []
|
|
if model_context_len is not None:
|
|
# bs=1 floor: ONE sliding window of swa KV (+ a page of slack for the
|
|
# page-granular walk) + the slot-0 sink. The full side is not charged
|
|
# (max_req_len clamps it); the swa sub-pool is sized independently of
|
|
# that clamp, which is why the window term stays.
|
|
swa_bs1_tokens = (
|
|
min(model_context_len, sliding_window_size + page_size)
|
|
if sliding_window_size is not None
|
|
else model_context_len
|
|
)
|
|
bs1_floor_terms = [
|
|
("swa_window_kv", swa_bs1_tokens * swa_spec.entry_bytes()),
|
|
("sink", _reserved_floor_bytes([full_spec, swa_spec], page_size)),
|
|
]
|
|
_check_bs1_feasibility_floor(
|
|
total_bytes=total_bytes,
|
|
floor_terms=bs1_floor_terms,
|
|
factory="init_unified_swa_pools",
|
|
)
|
|
if total_bytes <= 0:
|
|
raise ValueError(f"total_bytes must be positive, got {total_bytes}")
|
|
shared_pool = UnifiedKVPool(
|
|
total_bytes=total_bytes,
|
|
sub_pool_specs=[full_spec, swa_spec],
|
|
device=device,
|
|
enable_memory_saver=enable_memory_saver,
|
|
page_size=page_size,
|
|
post_capture_active=post_capture_active,
|
|
bs1_floor_terms=bs1_floor_terms,
|
|
fused_draft=fused_draft,
|
|
)
|
|
token_to_kv_pool = UnifiedSWAKVPool(
|
|
unified_buffer=shared_pool,
|
|
swa_attention_layer_ids=swa_attention_layer_ids,
|
|
full_attention_layer_ids=full_attention_layer_ids,
|
|
page_size=page_size,
|
|
start_layer=start_layer,
|
|
end_layer=end_layer,
|
|
enable_memory_saver=enable_memory_saver,
|
|
)
|
|
allocator = UnifiedSWATokenToKVPoolAllocator(
|
|
unified_buffer=shared_pool,
|
|
kvcache=token_to_kv_pool,
|
|
device=device,
|
|
page_size=page_size,
|
|
need_sort=need_sort,
|
|
forward_stream=forward_stream,
|
|
lazy_compaction=lazy_compaction,
|
|
**legacy_allocator_capacities,
|
|
)
|
|
|
|
logger.info(
|
|
"[unified-memory-pool] ============================================================"
|
|
)
|
|
logger.info("[unified-memory-pool] UNIFIED MEMORY POOL ENABLED -- path=SWA hybrid")
|
|
logger.info(
|
|
"[unified-memory-pool] %s",
|
|
"token-major views, entry_bytes full=%d swa=%d B"
|
|
% (full_spec.entry_bytes(), swa_spec.entry_bytes()),
|
|
)
|
|
logger.info(
|
|
"[unified-memory-pool] full_layers=%d, swa_layers=%d, head_num=%d, head_dim=%d, "
|
|
"v_head_dim=%d, swa_head_num=%d, swa_head_dim=%d, swa_v_head_dim=%d, "
|
|
"page_size=%d",
|
|
len(full_attention_layer_ids),
|
|
len(swa_attention_layer_ids),
|
|
head_num,
|
|
head_dim,
|
|
v_head_dim,
|
|
swa_head_num,
|
|
swa_head_dim,
|
|
swa_v_head_dim,
|
|
page_size,
|
|
)
|
|
logger.info(
|
|
"[unified-memory-pool] total_bytes=%d (=%.2f GB), "
|
|
"full_capacity=%d, swa_capacity=%d, joint_available=%d slots",
|
|
total_bytes,
|
|
total_bytes / GB,
|
|
allocator.size_full,
|
|
allocator.size_swa,
|
|
allocator.available_size(),
|
|
)
|
|
logger.info(
|
|
"[unified-memory-pool] ============================================================"
|
|
)
|
|
return UnifiedSWAPoolBundle(
|
|
unified_memory_pool=shared_pool,
|
|
token_to_kv_pool=token_to_kv_pool,
|
|
token_to_kv_pool_allocator=allocator,
|
|
)
|
|
|
|
|
|
def init_unified_mamba_swa_pools(
|
|
*,
|
|
device: str,
|
|
kv_cache_dtype: torch.dtype,
|
|
head_num: int,
|
|
head_dim: int,
|
|
v_head_dim: int,
|
|
swa_head_num: int,
|
|
swa_head_dim: int,
|
|
swa_v_head_dim: int,
|
|
page_size: int,
|
|
start_layer: int,
|
|
end_layer: int,
|
|
swa_attention_layer_ids: List[int],
|
|
full_attention_layer_ids: List[int],
|
|
mamba_layer_ids: List[int],
|
|
mamba2_cache_params,
|
|
full_max_total_num_tokens: int,
|
|
swa_max_total_num_tokens: int,
|
|
max_mamba_cache_size: int,
|
|
model_context_len: int,
|
|
extra_max_context_len: int,
|
|
max_num_reqs: int,
|
|
enable_memory_saver: bool,
|
|
enable_mamba_extra_buffer: bool,
|
|
enable_mamba_extra_buffer_lazy: bool,
|
|
disable_overlap_schedule: bool,
|
|
need_sort: bool,
|
|
speculative_num_draft_tokens: Optional[int] = None,
|
|
forward_stream: Optional[torch.cuda.Stream] = None,
|
|
lazy_compaction: bool = False,
|
|
unified_total_bytes: Optional[int] = None,
|
|
sliding_window_size: Optional[int] = None,
|
|
decode_pre_alloc_size: int = 0,
|
|
fused_draft: Optional[FusedDraftPlacement] = None,
|
|
) -> UnifiedPoolBundle:
|
|
"""Build the TRI-pool unified-memory-pool stack for models with full KV +
|
|
SWA KV + mamba/conv state (Inkling-class: `mambaish_config` AND
|
|
`is_hybrid_swa` simultaneously — Inkling's SConv state is conv-only but
|
|
rides the mamba machinery, so "mamba" here == the conv state pool).
|
|
|
|
Chain: ``[mamba (up END) | swa (FLOAT) | full (down END)]``. The KV side
|
|
is a `UnifiedSWAKVPool` (per-layer full/swa routing, asymmetric head
|
|
geometry supported); the state side is a `UnifiedHybridReqToTokenPool`
|
|
whose `mamba_pool` the model reads directly (sconv:
|
|
`req_to_token_pool.mamba2_layer_cache(layer).conv[...]` with
|
|
`translate_mamba_indices` for v->p).
|
|
|
|
Sizing inputs are the same token counts the 2-pool factories take (ratio-
|
|
fed until the byte configurator lands); the buffer budget is their byte
|
|
sum and the runtime split floats. With ``fused_draft``, every entry of a
|
|
sub-pool the placement names carries the draft's parts, as in the 2-pool
|
|
factories.
|
|
"""
|
|
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
|
UnifiedMambaSWATokenToKVPoolAllocator,
|
|
)
|
|
|
|
assert page_size >= 1, f"page_size must be >= 1, got {page_size}"
|
|
assert len(full_attention_layer_ids) > 0, (
|
|
"tri-pool with zero full-attention layers is degenerate"
|
|
)
|
|
assert len(swa_attention_layer_ids) > 0, (
|
|
"tri-pool with zero SWA-attention layers is degenerate"
|
|
)
|
|
assert len(mamba_layer_ids) > 0, "tri-pool with zero state layers is degenerate"
|
|
|
|
store_dtype = _store_dtype_for(kv_cache_dtype)
|
|
# mamba/conv at the LOWEST bytes, full KV at the HIGHEST, SWA floating
|
|
# between: ends never relocate, and SWA's window-capped span is cheapest to move.
|
|
full_spec = MHASubPoolSpec(
|
|
name="full",
|
|
layer_num=len(full_attention_layer_ids),
|
|
head_num=head_num,
|
|
head_dim=head_dim,
|
|
v_head_dim=v_head_dim,
|
|
store_dtype=store_dtype,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
grow_direction="down",
|
|
draft_region=None if fused_draft is None else fused_draft.region("full"),
|
|
)
|
|
swa_spec = MHASubPoolSpec(
|
|
name="swa",
|
|
layer_num=len(swa_attention_layer_ids),
|
|
head_num=swa_head_num,
|
|
head_dim=swa_head_dim,
|
|
v_head_dim=swa_v_head_dim,
|
|
store_dtype=store_dtype,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
grow_direction="float",
|
|
draft_region=None if fused_draft is None else fused_draft.region("swa"),
|
|
)
|
|
cp = mamba2_cache_params
|
|
mamba_spec = MambaSubPoolSpec(
|
|
name="mamba",
|
|
layer_num=len(mamba_layer_ids),
|
|
conv_state_shapes=tuple(tuple(int(x) for x in s) for s in cp.shape.conv),
|
|
conv_dtype=cp.dtype.conv,
|
|
temporal_state_shape=tuple(int(x) for x in cp.shape.temporal),
|
|
temporal_dtype=cp.dtype.temporal,
|
|
grow_direction="up",
|
|
draft_region=None if fused_draft is None else fused_draft.region("mamba"),
|
|
)
|
|
if unified_total_bytes is not None:
|
|
# PROFILED byte budget for the token side (captured pre-ratio-floor);
|
|
# the state pool's bytes ride on top. The token counts stay boot
|
|
# labels / conserve caps -- the runtime split floats.
|
|
total_bytes = (
|
|
unified_total_bytes + max_mamba_cache_size * mamba_spec.entry_bytes()
|
|
)
|
|
else:
|
|
total_bytes = (
|
|
full_max_total_num_tokens * full_spec.entry_bytes()
|
|
+ swa_max_total_num_tokens * swa_spec.entry_bytes()
|
|
+ max_mamba_cache_size * mamba_spec.entry_bytes()
|
|
)
|
|
# bs=1 floor: ONE sliding window of swa KV (+ a page of slack, clamped to
|
|
# the context) + the state slots one running request locks (1 active + 2
|
|
# radix checkpoints, a FLOOR not headroom) + the slot-0 sink. The
|
|
# full-attention side is not charged: `max_req_len` already clamps to the pool.
|
|
swa_bs1_tokens = (
|
|
min(model_context_len, sliding_window_size + page_size)
|
|
if sliding_window_size is not None
|
|
else model_context_len
|
|
)
|
|
_check_bs1_feasibility_floor(
|
|
total_bytes=total_bytes,
|
|
floor_terms=[
|
|
("swa_window_kv", swa_bs1_tokens * swa_spec.entry_bytes()),
|
|
("bs1_state_slots", 3 * mamba_spec.entry_bytes()),
|
|
(
|
|
"sink",
|
|
_reserved_floor_bytes([full_spec, swa_spec, mamba_spec], page_size),
|
|
),
|
|
],
|
|
factory="init_unified_mamba_swa_pools",
|
|
)
|
|
shared_pool = UnifiedKVPool(
|
|
total_bytes=total_bytes,
|
|
sub_pool_specs=[full_spec, swa_spec, mamba_spec],
|
|
device=device,
|
|
enable_memory_saver=enable_memory_saver,
|
|
page_size=page_size,
|
|
fused_draft=fused_draft,
|
|
)
|
|
token_to_kv_pool = UnifiedSWAKVPool(
|
|
unified_buffer=shared_pool,
|
|
swa_attention_layer_ids=swa_attention_layer_ids,
|
|
full_attention_layer_ids=full_attention_layer_ids,
|
|
page_size=page_size,
|
|
start_layer=start_layer,
|
|
end_layer=end_layer,
|
|
enable_memory_saver=enable_memory_saver,
|
|
)
|
|
req_to_token_pool = UnifiedHybridReqToTokenPool(
|
|
unified_buffer=shared_pool,
|
|
mamba_sub_pool_name="mamba",
|
|
size=max_num_reqs,
|
|
mamba_spec_state_size=max_num_reqs,
|
|
max_context_len=model_context_len + extra_max_context_len,
|
|
device=device,
|
|
enable_memory_saver=enable_memory_saver,
|
|
cache_params=mamba2_cache_params,
|
|
mamba_layer_ids=mamba_layer_ids,
|
|
enable_mamba_extra_buffer=enable_mamba_extra_buffer,
|
|
enable_mamba_extra_buffer_lazy=enable_mamba_extra_buffer_lazy,
|
|
speculative_num_draft_tokens=speculative_num_draft_tokens,
|
|
enable_overlap_schedule=not disable_overlap_schedule,
|
|
start_layer=start_layer,
|
|
pre_alloc_size=decode_pre_alloc_size,
|
|
)
|
|
allocator = UnifiedMambaSWATokenToKVPoolAllocator(
|
|
unified_buffer=shared_pool,
|
|
kvcache=token_to_kv_pool,
|
|
mamba_kvcache=req_to_token_pool.mamba_pool,
|
|
device=device,
|
|
full_max_total_num_tokens=full_max_total_num_tokens,
|
|
swa_max_total_num_tokens=swa_max_total_num_tokens,
|
|
page_size=page_size,
|
|
need_sort=need_sort,
|
|
forward_stream=forward_stream,
|
|
lazy_compaction=lazy_compaction,
|
|
)
|
|
_wire_mamba_slot_allocator(
|
|
mamba_end=allocator.mamba_allocator,
|
|
req_to_token_pool=req_to_token_pool,
|
|
device=device,
|
|
)
|
|
|
|
logger.info(
|
|
"[unified-memory-pool] ============================================================"
|
|
)
|
|
logger.info(
|
|
"[unified-memory-pool] UNIFIED MEMORY POOL ENABLED -- path=SWA+Mamba tri-pool"
|
|
)
|
|
logger.info(
|
|
"[unified-memory-pool] full_layers=%d, swa_layers=%d, state_layers=%d, "
|
|
"head_num=%d/%d, head_dim=%d/%d, page_size=%d",
|
|
len(full_attention_layer_ids),
|
|
len(swa_attention_layer_ids),
|
|
len(mamba_layer_ids),
|
|
head_num,
|
|
swa_head_num,
|
|
head_dim,
|
|
swa_head_dim,
|
|
page_size,
|
|
)
|
|
logger.info(
|
|
"[unified-memory-pool] total_bytes=%d (=%.2f GB), full_max=%d, swa_max=%d, "
|
|
"max_mamba_cache_size=%d, max_num_reqs=%d, joint_available=%d",
|
|
total_bytes,
|
|
total_bytes / GB,
|
|
full_max_total_num_tokens,
|
|
swa_max_total_num_tokens,
|
|
max_mamba_cache_size,
|
|
max_num_reqs,
|
|
allocator.available_size(),
|
|
)
|
|
logger.info(
|
|
"[unified-memory-pool] ============================================================"
|
|
)
|
|
return UnifiedPoolBundle(
|
|
unified_memory_pool=shared_pool,
|
|
token_to_kv_pool=token_to_kv_pool,
|
|
token_to_kv_pool_allocator=allocator,
|
|
req_to_token_pool=req_to_token_pool,
|
|
)
|