Share MLA host capacity initialization (#43595)
This commit is contained in:
@@ -12,7 +12,6 @@ from typing import Optional, TypeGuard
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.host_memory import available_host_memory_bytes
|
||||
from sglang.srt.mem_cache.memory_pool import KVCache
|
||||
from sglang.srt.mem_cache.pool_host.common import (
|
||||
_cuda_host_unregister,
|
||||
get_allocator_from_storage,
|
||||
@@ -171,7 +170,12 @@ class HostKVCache(abc.ABC):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool: KVCache,
|
||||
*,
|
||||
dtype: torch.dtype,
|
||||
device_capacity_tokens: int,
|
||||
size_per_token: int,
|
||||
start_layer: int,
|
||||
end_layer: int,
|
||||
host_to_device_ratio: float,
|
||||
host_size: int,
|
||||
page_size: int,
|
||||
@@ -181,10 +185,10 @@ class HostKVCache(abc.ABC):
|
||||
allocator_type: str = "default",
|
||||
dcp_size: int = 1,
|
||||
dcp_rank: int = 0,
|
||||
*,
|
||||
pool_label: str = "kv",
|
||||
is_dummy: bool = False,
|
||||
):
|
||||
self.device_pool = device_pool
|
||||
"""Allocate host pages from geometry prepared by the concrete host pool."""
|
||||
self.pool_label = pool_label
|
||||
# page_size arrives widened (x dcp_size); size/page_size/page_num are physical.
|
||||
self.dcp_size = dcp_size
|
||||
@@ -201,33 +205,34 @@ class HostKVCache(abc.ABC):
|
||||
self.allocator = get_allocator_from_storage(allocator_type)
|
||||
self.can_use_write_back_jit = False
|
||||
|
||||
self.dtype = device_pool.store_dtype
|
||||
self.size_per_token = self.get_size_per_token()
|
||||
# Unified pools report token capacity separately from their buffer-row count.
|
||||
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
|
||||
if device_capacity is None:
|
||||
device_capacity = device_pool.size
|
||||
self.device_capacity_tokens = device_capacity
|
||||
self.dtype = dtype
|
||||
self.size_per_token = size_per_token
|
||||
self.device_capacity_tokens = device_capacity_tokens
|
||||
if host_size > 0:
|
||||
self.size = sync_fixed_hicache_size(
|
||||
int(host_size * 1e9 // self.size_per_token), host_size
|
||||
)
|
||||
else:
|
||||
self.size = int(device_capacity * host_to_device_ratio)
|
||||
self.size = int(device_capacity_tokens * host_to_device_ratio)
|
||||
# Align up the host memory pool size to the page size
|
||||
self.page_num = self.size // self.page_size + 1
|
||||
self.size = self.page_num * self.page_size
|
||||
self.start_layer = device_pool.start_layer
|
||||
self.end_layer = device_pool.end_layer
|
||||
self.start_layer = start_layer
|
||||
self.end_layer = end_layer
|
||||
if not is_dummy:
|
||||
self._allocate_host_buffer()
|
||||
self.lock = threading.RLock()
|
||||
self.clear()
|
||||
|
||||
if self.size <= device_capacity:
|
||||
def _allocate_host_buffer(self) -> None:
|
||||
if self.size <= self.device_capacity_tokens:
|
||||
logger.warning(
|
||||
"HiCache %s host pool (%d tokens) is smaller than the device pool (%d tokens);"
|
||||
"L2 cache effectiveness is reduced."
|
||||
"Consider increasing --hicache-ratio (or --hicache-size) for higher L2 cache hit rate.",
|
||||
pool_label,
|
||||
self.pool_label,
|
||||
self.size,
|
||||
device_capacity,
|
||||
self.device_capacity_tokens,
|
||||
)
|
||||
|
||||
# Verify there is enough available host memory.
|
||||
@@ -247,7 +252,7 @@ class HostKVCache(abc.ABC):
|
||||
"Allocating %s hierarchical KV host pool: %d tokens, "
|
||||
"%.2f GB host memory, packed MTP KV layers: "
|
||||
"target_layers=%d, draft_layers=%d, total_layers=%d.",
|
||||
pool_label,
|
||||
self.pool_label,
|
||||
self.size,
|
||||
requested_bytes / 1e9,
|
||||
self.target_layer_num,
|
||||
@@ -257,7 +262,7 @@ class HostKVCache(abc.ABC):
|
||||
else:
|
||||
logger.info(
|
||||
"Allocating %s hierarchical KV host pool: %d tokens, %.2f GB host memory.",
|
||||
pool_label,
|
||||
self.pool_label,
|
||||
self.size,
|
||||
requested_bytes / 1e9,
|
||||
)
|
||||
@@ -265,10 +270,6 @@ class HostKVCache(abc.ABC):
|
||||
self.kv_buffer = self.init_kv_buffer()
|
||||
self.fd = getattr(self.allocator, "fd", None)
|
||||
|
||||
# A lock for synchronized operations on memory allocation and state transitions.
|
||||
self.lock = threading.RLock()
|
||||
self.clear()
|
||||
|
||||
def destroy(self):
|
||||
"""Unregister pinned host buffers in userspace before process exit.
|
||||
|
||||
|
||||
@@ -142,15 +142,26 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
):
|
||||
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
|
||||
self.target_layer_num = device_pool.layer_num
|
||||
self.device_pool = device_pool
|
||||
self.dtype = device_pool.store_dtype
|
||||
# Scale-bearing subclasses need page coverage when computing token bytes.
|
||||
self.page_size = page_size
|
||||
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
|
||||
super().__init__(
|
||||
device_pool,
|
||||
host_to_device_ratio,
|
||||
host_size,
|
||||
page_size,
|
||||
layout,
|
||||
pin_memory,
|
||||
device,
|
||||
allocator_type,
|
||||
dtype=self.dtype,
|
||||
device_capacity_tokens=(
|
||||
device_pool.size if device_capacity is None else device_capacity
|
||||
),
|
||||
size_per_token=self.get_size_per_token(),
|
||||
start_layer=device_pool.start_layer,
|
||||
end_layer=device_pool.end_layer,
|
||||
host_to_device_ratio=host_to_device_ratio,
|
||||
host_size=host_size,
|
||||
page_size=page_size,
|
||||
layout=layout,
|
||||
pin_memory=pin_memory,
|
||||
device=device,
|
||||
allocator_type=allocator_type,
|
||||
pool_label=pool_label,
|
||||
)
|
||||
self.element_dim = self.head_num * self.head_dim
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
from typing import Optional, Sequence
|
||||
|
||||
import torch
|
||||
@@ -24,11 +23,9 @@ from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
||||
from sglang.srt.mem_cache.pool_host.base import (
|
||||
_WRITE_BACK_STAGING_PAGE_CHUNK,
|
||||
HostKVCache,
|
||||
sync_fixed_hicache_size,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.common import (
|
||||
ALLOC_MEMORY_FUNCS,
|
||||
get_allocator_from_storage,
|
||||
make_kernel_ptr_table,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.hisparse import HiSparseHostPoolMixin
|
||||
@@ -90,35 +87,50 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
|
||||
self._is_dummy = is_dummy
|
||||
|
||||
self.device_pool = device_pool
|
||||
self.dtype = device_pool.store_dtype
|
||||
self.layout = layout
|
||||
device_capacity = getattr(device_pool, "host_capacity_tokens", None)
|
||||
if is_dummy:
|
||||
self._init_dummy(
|
||||
device_pool,
|
||||
host_to_device_ratio,
|
||||
host_size,
|
||||
page_size,
|
||||
layout,
|
||||
pin_memory,
|
||||
device,
|
||||
allocator_type,
|
||||
dcp_size,
|
||||
dcp_rank,
|
||||
pool_label,
|
||||
)
|
||||
return
|
||||
|
||||
device_capacity = device_capacity or device_pool.size
|
||||
super().__init__(
|
||||
device_pool,
|
||||
host_to_device_ratio,
|
||||
host_size,
|
||||
page_size,
|
||||
layout,
|
||||
pin_memory,
|
||||
device,
|
||||
allocator_type,
|
||||
dtype=self.dtype,
|
||||
device_capacity_tokens=(
|
||||
device_pool.size if device_capacity is None else device_capacity
|
||||
),
|
||||
size_per_token=self.get_size_per_token(),
|
||||
start_layer=device_pool.start_layer,
|
||||
end_layer=device_pool.end_layer,
|
||||
host_to_device_ratio=host_to_device_ratio,
|
||||
host_size=host_size,
|
||||
page_size=page_size,
|
||||
layout=layout,
|
||||
pin_memory=pin_memory,
|
||||
device=device,
|
||||
allocator_type=allocator_type,
|
||||
dcp_size=dcp_size,
|
||||
dcp_rank=dcp_rank,
|
||||
pool_label=pool_label,
|
||||
is_dummy=is_dummy,
|
||||
)
|
||||
if is_dummy:
|
||||
self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize
|
||||
self.layout_dim = self.token_stride_size * self.layer_num
|
||||
self.can_use_jit = False
|
||||
self.staging_page_capacity = 0
|
||||
self.staging_token_capacity = 0
|
||||
self.staging_buffer = None
|
||||
self.kv_buffer = None
|
||||
self.data_refs = None
|
||||
self.data_ptrs = None
|
||||
logger.info(
|
||||
"MLATokenToKVPoolHost dummy mode: allocator-only, size=%d tokens, "
|
||||
"saving %.2f GB host memory",
|
||||
self.size,
|
||||
self.size * self.size_per_token / 1e9,
|
||||
)
|
||||
return
|
||||
|
||||
# The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only
|
||||
# helpers in hicache.cuh are guarded by USE_ROCM and the staged
|
||||
# write-back kernel has a ROCm path, so enable them on HIP too. This
|
||||
@@ -156,71 +168,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
)
|
||||
self._init_write_back_staging_buffers()
|
||||
|
||||
def _init_dummy(
|
||||
self,
|
||||
device_pool: MLATokenToKVPool,
|
||||
host_to_device_ratio: float,
|
||||
host_size: int,
|
||||
page_size: int,
|
||||
layout: str,
|
||||
pin_memory: bool,
|
||||
device: str,
|
||||
allocator_type: str,
|
||||
dcp_size: int,
|
||||
dcp_rank: int,
|
||||
pool_label: str,
|
||||
) -> None:
|
||||
self.device_pool = device_pool
|
||||
self.pool_label = pool_label
|
||||
self.dcp_size = dcp_size
|
||||
self.dcp_rank = dcp_rank
|
||||
assert page_size % dcp_size == 0, (
|
||||
f"HiCache host pool page_size ({page_size}) must be a multiple of "
|
||||
f"dcp_size ({dcp_size})."
|
||||
)
|
||||
self.page_size = page_size // dcp_size
|
||||
self.layout = layout
|
||||
self.pin_memory = pin_memory
|
||||
self.device = device
|
||||
self.allocator = get_allocator_from_storage(allocator_type)
|
||||
|
||||
self.dtype = device_pool.store_dtype
|
||||
self.size_per_token = self.get_size_per_token()
|
||||
if host_size > 0:
|
||||
self.size = sync_fixed_hicache_size(
|
||||
int(host_size * 1e9 // self.size_per_token), host_size
|
||||
)
|
||||
else:
|
||||
self.size = int(
|
||||
(getattr(device_pool, "host_capacity_tokens", None) or device_pool.size)
|
||||
* host_to_device_ratio
|
||||
)
|
||||
self.page_num = self.size // self.page_size + 1
|
||||
self.size = self.page_num * self.page_size
|
||||
self.start_layer = device_pool.start_layer
|
||||
self.end_layer = device_pool.end_layer
|
||||
|
||||
self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize
|
||||
self.layout_dim = self.token_stride_size * self.layer_num
|
||||
self.can_use_jit = False
|
||||
self.can_use_write_back_jit = False
|
||||
self.staging_page_capacity = 0
|
||||
self.staging_token_capacity = 0
|
||||
self.staging_buffer = None
|
||||
self.kv_buffer = None
|
||||
self.data_refs = None
|
||||
self.data_ptrs = None
|
||||
|
||||
logger.info(
|
||||
"MLATokenToKVPoolHost dummy mode: allocator-only, size=%d tokens, "
|
||||
"saving %.2f GB host memory",
|
||||
self.size,
|
||||
self.size * self.size_per_token / 1e9,
|
||||
)
|
||||
|
||||
self.lock = threading.RLock()
|
||||
self.clear()
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
"""Return (data_ptrs, data_lens, item_lens) in the same format as device pool,
|
||||
for registering host memory with the disaggregation transfer engine."""
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPoolMXFP8
|
||||
from sglang.srt.mem_cache.pool_host.mha_mxfp8 import MHATokenToKVPoolMXFP8Host
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-small")
|
||||
|
||||
|
||||
class TestMXFP8HostAllocation(CustomTestCase):
|
||||
def test_constructor_counts_payload_and_scale_pages(self):
|
||||
pool = MHATokenToKVPoolMXFP8(
|
||||
size=256,
|
||||
page_size=128,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
head_num=1,
|
||||
head_dim=128,
|
||||
layer_num=2,
|
||||
device="cuda",
|
||||
enable_memory_saver=False,
|
||||
)
|
||||
host = MHATokenToKVPoolMXFP8Host(
|
||||
pool,
|
||||
host_to_device_ratio=2,
|
||||
host_size=0,
|
||||
page_size=pool.page_size,
|
||||
layout="page_first",
|
||||
pin_memory=False,
|
||||
)
|
||||
self.addCleanup(host.destroy)
|
||||
expected_token_bytes = (
|
||||
2
|
||||
* pool.layer_num
|
||||
* pool.head_num
|
||||
* (pool.head_dim + pool.head_dim // pool.MXFP8_SCALE_BLOCK_SIZE)
|
||||
)
|
||||
self.assertEqual(host.size_per_token, expected_token_bytes)
|
||||
self.assertEqual(host.page_size, pool.page_size)
|
||||
self.assertTrue(host.can_use_write_back_jit)
|
||||
self.assertEqual(
|
||||
host.kv_buffer.nbytes + host.k_scale_host.nbytes + host.v_scale_host.nbytes,
|
||||
host.size * expected_token_bytes,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -103,6 +103,50 @@ class TestMLAHostDedupPrimitives(unittest.TestCase):
|
||||
dsa_device_pool, slots, slots, layer_id=0, io_backend="kernel"
|
||||
)
|
||||
|
||||
def test_dummy_and_physical_hosts_share_capacity_geometry(self):
|
||||
"""Dedup peers must allocate identical IDs despite different host payloads."""
|
||||
for dcp_size in (1, 2):
|
||||
for host_size in (0, 0.000002):
|
||||
with self.subTest(dcp_size=dcp_size, host_size=host_size):
|
||||
pool = _device_pool_stub(
|
||||
layer_num=2,
|
||||
store_dtype=torch.float16,
|
||||
kv_lora_rank=4,
|
||||
qk_rope_head_dim=2,
|
||||
size=8,
|
||||
host_capacity_tokens=16,
|
||||
start_layer=7,
|
||||
end_layer=9,
|
||||
device="cpu",
|
||||
kv_buffer=[torch.zeros(10, 1, 6) for _ in range(2)],
|
||||
)
|
||||
kwargs = dict(
|
||||
host_to_device_ratio=2,
|
||||
host_size=host_size,
|
||||
page_size=4,
|
||||
layout="page_first",
|
||||
pin_memory=False,
|
||||
dcp_size=dcp_size,
|
||||
)
|
||||
host = MLATokenToKVPoolHost(pool, **kwargs)
|
||||
dummy = MLATokenToKVPoolHost(pool, **kwargs, is_dummy=True)
|
||||
self.addCleanup(host.destroy)
|
||||
self.addCleanup(dummy.destroy)
|
||||
self.assertEqual(
|
||||
(host.size, host.page_num, host.page_size, host.size_per_token),
|
||||
(
|
||||
dummy.size,
|
||||
dummy.page_num,
|
||||
dummy.page_size,
|
||||
dummy.size_per_token,
|
||||
),
|
||||
)
|
||||
self.assertIsNone(dummy.kv_buffer)
|
||||
count = host.logical_size // 4 * 4
|
||||
self.assertEqual(
|
||||
host.alloc(count).tolist(), dummy.alloc(count).tolist()
|
||||
)
|
||||
|
||||
def test_layer_broadcast_reuses_full_staging_capacity(self):
|
||||
broadcaster = MLAHostDedupBroadcaster.__new__(MLAHostDedupBroadcaster)
|
||||
broadcaster.is_src = True
|
||||
|
||||
Reference in New Issue
Block a user