Share MLA host capacity initialization (#43595)

This commit is contained in:
Ke Bao
2026-10-11 22:36:29 +08:00
committed by GitHub
parent c938b6e454
commit b0a75c8196
5 changed files with 175 additions and 122 deletions
+24 -23
View File
@@ -12,7 +12,6 @@ from typing import Optional, TypeGuard
import torch
from sglang.srt.mem_cache.host_memory import available_host_memory_bytes
from sglang.srt.mem_cache.memory_pool import KVCache
from sglang.srt.mem_cache.pool_host.common import (
_cuda_host_unregister,
get_allocator_from_storage,
@@ -171,7 +170,12 @@ class HostKVCache(abc.ABC):
def __init__(
self,
device_pool: KVCache,
*,
dtype: torch.dtype,
device_capacity_tokens: int,
size_per_token: int,
start_layer: int,
end_layer: int,
host_to_device_ratio: float,
host_size: int,
page_size: int,
@@ -181,10 +185,10 @@ class HostKVCache(abc.ABC):
allocator_type: str = "default",
dcp_size: int = 1,
dcp_rank: int = 0,
*,
pool_label: str = "kv",
is_dummy: bool = False,
):
self.device_pool = device_pool
"""Allocate host pages from geometry prepared by the concrete host pool."""
self.pool_label = pool_label
# page_size arrives widened (x dcp_size); size/page_size/page_num are physical.
self.dcp_size = dcp_size
@@ -201,33 +205,34 @@ class HostKVCache(abc.ABC):
self.allocator = get_allocator_from_storage(allocator_type)
self.can_use_write_back_jit = False
self.dtype = device_pool.store_dtype
self.size_per_token = self.get_size_per_token()
# Unified pools report token capacity separately from their buffer-row count.
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
if device_capacity is None:
device_capacity = device_pool.size
self.device_capacity_tokens = device_capacity
self.dtype = dtype
self.size_per_token = size_per_token
self.device_capacity_tokens = device_capacity_tokens
if host_size > 0:
self.size = sync_fixed_hicache_size(
int(host_size * 1e9 // self.size_per_token), host_size
)
else:
self.size = int(device_capacity * host_to_device_ratio)
self.size = int(device_capacity_tokens * host_to_device_ratio)
# Align up the host memory pool size to the page size
self.page_num = self.size // self.page_size + 1
self.size = self.page_num * self.page_size
self.start_layer = device_pool.start_layer
self.end_layer = device_pool.end_layer
self.start_layer = start_layer
self.end_layer = end_layer
if not is_dummy:
self._allocate_host_buffer()
self.lock = threading.RLock()
self.clear()
if self.size <= device_capacity:
def _allocate_host_buffer(self) -> None:
if self.size <= self.device_capacity_tokens:
logger.warning(
"HiCache %s host pool (%d tokens) is smaller than the device pool (%d tokens);"
"L2 cache effectiveness is reduced."
"Consider increasing --hicache-ratio (or --hicache-size) for higher L2 cache hit rate.",
pool_label,
self.pool_label,
self.size,
device_capacity,
self.device_capacity_tokens,
)
# Verify there is enough available host memory.
@@ -247,7 +252,7 @@ class HostKVCache(abc.ABC):
"Allocating %s hierarchical KV host pool: %d tokens, "
"%.2f GB host memory, packed MTP KV layers: "
"target_layers=%d, draft_layers=%d, total_layers=%d.",
pool_label,
self.pool_label,
self.size,
requested_bytes / 1e9,
self.target_layer_num,
@@ -257,7 +262,7 @@ class HostKVCache(abc.ABC):
else:
logger.info(
"Allocating %s hierarchical KV host pool: %d tokens, %.2f GB host memory.",
pool_label,
self.pool_label,
self.size,
requested_bytes / 1e9,
)
@@ -265,10 +270,6 @@ class HostKVCache(abc.ABC):
self.kv_buffer = self.init_kv_buffer()
self.fd = getattr(self.allocator, "fd", None)
# A lock for synchronized operations on memory allocation and state transitions.
self.lock = threading.RLock()
self.clear()
def destroy(self):
"""Unregister pinned host buffers in userspace before process exit.
+19 -8
View File
@@ -142,15 +142,26 @@ class MHATokenToKVPoolHost(HostKVCache):
):
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
self.target_layer_num = device_pool.layer_num
self.device_pool = device_pool
self.dtype = device_pool.store_dtype
# Scale-bearing subclasses need page coverage when computing token bytes.
self.page_size = page_size
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
super().__init__(
device_pool,
host_to_device_ratio,
host_size,
page_size,
layout,
pin_memory,
device,
allocator_type,
dtype=self.dtype,
device_capacity_tokens=(
device_pool.size if device_capacity is None else device_capacity
),
size_per_token=self.get_size_per_token(),
start_layer=device_pool.start_layer,
end_layer=device_pool.end_layer,
host_to_device_ratio=host_to_device_ratio,
host_size=host_size,
page_size=page_size,
layout=layout,
pin_memory=pin_memory,
device=device,
allocator_type=allocator_type,
pool_label=pool_label,
)
self.element_dim = self.head_num * self.head_dim
+38 -91
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import logging
import threading
from typing import Optional, Sequence
import torch
@@ -24,11 +23,9 @@ from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
from sglang.srt.mem_cache.pool_host.base import (
_WRITE_BACK_STAGING_PAGE_CHUNK,
HostKVCache,
sync_fixed_hicache_size,
)
from sglang.srt.mem_cache.pool_host.common import (
ALLOC_MEMORY_FUNCS,
get_allocator_from_storage,
make_kernel_ptr_table,
)
from sglang.srt.mem_cache.pool_host.hisparse import HiSparseHostPoolMixin
@@ -90,35 +87,50 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
self._is_dummy = is_dummy
self.device_pool = device_pool
self.dtype = device_pool.store_dtype
self.layout = layout
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
if is_dummy:
self._init_dummy(
device_pool,
host_to_device_ratio,
host_size,
page_size,
layout,
pin_memory,
device,
allocator_type,
dcp_size,
dcp_rank,
pool_label,
)
return
device_capacity = device_capacity or device_pool.size
super().__init__(
device_pool,
host_to_device_ratio,
host_size,
page_size,
layout,
pin_memory,
device,
allocator_type,
dtype=self.dtype,
device_capacity_tokens=(
device_pool.size if device_capacity is None else device_capacity
),
size_per_token=self.get_size_per_token(),
start_layer=device_pool.start_layer,
end_layer=device_pool.end_layer,
host_to_device_ratio=host_to_device_ratio,
host_size=host_size,
page_size=page_size,
layout=layout,
pin_memory=pin_memory,
device=device,
allocator_type=allocator_type,
dcp_size=dcp_size,
dcp_rank=dcp_rank,
pool_label=pool_label,
is_dummy=is_dummy,
)
if is_dummy:
self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize
self.layout_dim = self.token_stride_size * self.layer_num
self.can_use_jit = False
self.staging_page_capacity = 0
self.staging_token_capacity = 0
self.staging_buffer = None
self.kv_buffer = None
self.data_refs = None
self.data_ptrs = None
logger.info(
"MLATokenToKVPoolHost dummy mode: allocator-only, size=%d tokens, "
"saving %.2f GB host memory",
self.size,
self.size * self.size_per_token / 1e9,
)
return
# The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only
# helpers in hicache.cuh are guarded by USE_ROCM and the staged
# write-back kernel has a ROCm path, so enable them on HIP too. This
@@ -156,71 +168,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
)
self._init_write_back_staging_buffers()
def _init_dummy(
self,
device_pool: MLATokenToKVPool,
host_to_device_ratio: float,
host_size: int,
page_size: int,
layout: str,
pin_memory: bool,
device: str,
allocator_type: str,
dcp_size: int,
dcp_rank: int,
pool_label: str,
) -> None:
self.device_pool = device_pool
self.pool_label = pool_label
self.dcp_size = dcp_size
self.dcp_rank = dcp_rank
assert page_size % dcp_size == 0, (
f"HiCache host pool page_size ({page_size}) must be a multiple of "
f"dcp_size ({dcp_size})."
)
self.page_size = page_size // dcp_size
self.layout = layout
self.pin_memory = pin_memory
self.device = device
self.allocator = get_allocator_from_storage(allocator_type)
self.dtype = device_pool.store_dtype
self.size_per_token = self.get_size_per_token()
if host_size > 0:
self.size = sync_fixed_hicache_size(
int(host_size * 1e9 // self.size_per_token), host_size
)
else:
self.size = int(
(getattr(device_pool, "host_capacity_tokens", None) or device_pool.size)
* host_to_device_ratio
)
self.page_num = self.size // self.page_size + 1
self.size = self.page_num * self.page_size
self.start_layer = device_pool.start_layer
self.end_layer = device_pool.end_layer
self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize
self.layout_dim = self.token_stride_size * self.layer_num
self.can_use_jit = False
self.can_use_write_back_jit = False
self.staging_page_capacity = 0
self.staging_token_capacity = 0
self.staging_buffer = None
self.kv_buffer = None
self.data_refs = None
self.data_ptrs = None
logger.info(
"MLATokenToKVPoolHost dummy mode: allocator-only, size=%d tokens, "
"saving %.2f GB host memory",
self.size,
self.size * self.size_per_token / 1e9,
)
self.lock = threading.RLock()
self.clear()
def get_contiguous_buf_infos(self):
"""Return (data_ptrs, data_lens, item_lens) in the same format as device pool,
for registering host memory with the disaggregation transfer engine."""