diff --git a/python/sglang/srt/mem_cache/pool_host/base.py b/python/sglang/srt/mem_cache/pool_host/base.py index 56588886ec..630e1f8564 100644 --- a/python/sglang/srt/mem_cache/pool_host/base.py +++ b/python/sglang/srt/mem_cache/pool_host/base.py @@ -12,7 +12,6 @@ from typing import Optional, TypeGuard import torch from sglang.srt.mem_cache.host_memory import available_host_memory_bytes -from sglang.srt.mem_cache.memory_pool import KVCache from sglang.srt.mem_cache.pool_host.common import ( _cuda_host_unregister, get_allocator_from_storage, @@ -171,7 +170,12 @@ class HostKVCache(abc.ABC): def __init__( self, - device_pool: KVCache, + *, + dtype: torch.dtype, + device_capacity_tokens: int, + size_per_token: int, + start_layer: int, + end_layer: int, host_to_device_ratio: float, host_size: int, page_size: int, @@ -181,10 +185,10 @@ class HostKVCache(abc.ABC): allocator_type: str = "default", dcp_size: int = 1, dcp_rank: int = 0, - *, pool_label: str = "kv", + is_dummy: bool = False, ): - self.device_pool = device_pool + """Allocate host pages from geometry prepared by the concrete host pool.""" self.pool_label = pool_label # page_size arrives widened (x dcp_size); size/page_size/page_num are physical. self.dcp_size = dcp_size @@ -201,33 +205,34 @@ class HostKVCache(abc.ABC): self.allocator = get_allocator_from_storage(allocator_type) self.can_use_write_back_jit = False - self.dtype = device_pool.store_dtype - self.size_per_token = self.get_size_per_token() - # Unified pools report token capacity separately from their buffer-row count. - device_capacity = getattr(device_pool, "host_capacity_tokens", None) - if device_capacity is None: - device_capacity = device_pool.size - self.device_capacity_tokens = device_capacity + self.dtype = dtype + self.size_per_token = size_per_token + self.device_capacity_tokens = device_capacity_tokens if host_size > 0: self.size = sync_fixed_hicache_size( int(host_size * 1e9 // self.size_per_token), host_size ) else: - self.size = int(device_capacity * host_to_device_ratio) + self.size = int(device_capacity_tokens * host_to_device_ratio) # Align up the host memory pool size to the page size self.page_num = self.size // self.page_size + 1 self.size = self.page_num * self.page_size - self.start_layer = device_pool.start_layer - self.end_layer = device_pool.end_layer + self.start_layer = start_layer + self.end_layer = end_layer + if not is_dummy: + self._allocate_host_buffer() + self.lock = threading.RLock() + self.clear() - if self.size <= device_capacity: + def _allocate_host_buffer(self) -> None: + if self.size <= self.device_capacity_tokens: logger.warning( "HiCache %s host pool (%d tokens) is smaller than the device pool (%d tokens);" "L2 cache effectiveness is reduced." "Consider increasing --hicache-ratio (or --hicache-size) for higher L2 cache hit rate.", - pool_label, + self.pool_label, self.size, - device_capacity, + self.device_capacity_tokens, ) # Verify there is enough available host memory. @@ -247,7 +252,7 @@ class HostKVCache(abc.ABC): "Allocating %s hierarchical KV host pool: %d tokens, " "%.2f GB host memory, packed MTP KV layers: " "target_layers=%d, draft_layers=%d, total_layers=%d.", - pool_label, + self.pool_label, self.size, requested_bytes / 1e9, self.target_layer_num, @@ -257,7 +262,7 @@ class HostKVCache(abc.ABC): else: logger.info( "Allocating %s hierarchical KV host pool: %d tokens, %.2f GB host memory.", - pool_label, + self.pool_label, self.size, requested_bytes / 1e9, ) @@ -265,10 +270,6 @@ class HostKVCache(abc.ABC): self.kv_buffer = self.init_kv_buffer() self.fd = getattr(self.allocator, "fd", None) - # A lock for synchronized operations on memory allocation and state transitions. - self.lock = threading.RLock() - self.clear() - def destroy(self): """Unregister pinned host buffers in userspace before process exit. diff --git a/python/sglang/srt/mem_cache/pool_host/mha.py b/python/sglang/srt/mem_cache/pool_host/mha.py index f612e57115..331eb32398 100644 --- a/python/sglang/srt/mem_cache/pool_host/mha.py +++ b/python/sglang/srt/mem_cache/pool_host/mha.py @@ -142,15 +142,26 @@ class MHATokenToKVPoolHost(HostKVCache): ): self.mtp_draft_device_pools = tuple(mtp_draft_device_pools) self.target_layer_num = device_pool.layer_num + self.device_pool = device_pool + self.dtype = device_pool.store_dtype + # Scale-bearing subclasses need page coverage when computing token bytes. + self.page_size = page_size + device_capacity = getattr(device_pool, "host_capacity_tokens", None) super().__init__( - device_pool, - host_to_device_ratio, - host_size, - page_size, - layout, - pin_memory, - device, - allocator_type, + dtype=self.dtype, + device_capacity_tokens=( + device_pool.size if device_capacity is None else device_capacity + ), + size_per_token=self.get_size_per_token(), + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + host_to_device_ratio=host_to_device_ratio, + host_size=host_size, + page_size=page_size, + layout=layout, + pin_memory=pin_memory, + device=device, + allocator_type=allocator_type, pool_label=pool_label, ) self.element_dim = self.head_num * self.head_dim diff --git a/python/sglang/srt/mem_cache/pool_host/mla.py b/python/sglang/srt/mem_cache/pool_host/mla.py index 35e0bb2c2f..4688fd7a7b 100644 --- a/python/sglang/srt/mem_cache/pool_host/mla.py +++ b/python/sglang/srt/mem_cache/pool_host/mla.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -import threading from typing import Optional, Sequence import torch @@ -24,11 +23,9 @@ from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool from sglang.srt.mem_cache.pool_host.base import ( _WRITE_BACK_STAGING_PAGE_CHUNK, HostKVCache, - sync_fixed_hicache_size, ) from sglang.srt.mem_cache.pool_host.common import ( ALLOC_MEMORY_FUNCS, - get_allocator_from_storage, make_kernel_ptr_table, ) from sglang.srt.mem_cache.pool_host.hisparse import HiSparseHostPoolMixin @@ -90,35 +87,50 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): self.mtp_draft_device_pools = tuple(mtp_draft_device_pools) self._is_dummy = is_dummy + self.device_pool = device_pool + self.dtype = device_pool.store_dtype + self.layout = layout + device_capacity = getattr(device_pool, "host_capacity_tokens", None) if is_dummy: - self._init_dummy( - device_pool, - host_to_device_ratio, - host_size, - page_size, - layout, - pin_memory, - device, - allocator_type, - dcp_size, - dcp_rank, - pool_label, - ) - return - + device_capacity = device_capacity or device_pool.size super().__init__( - device_pool, - host_to_device_ratio, - host_size, - page_size, - layout, - pin_memory, - device, - allocator_type, + dtype=self.dtype, + device_capacity_tokens=( + device_pool.size if device_capacity is None else device_capacity + ), + size_per_token=self.get_size_per_token(), + start_layer=device_pool.start_layer, + end_layer=device_pool.end_layer, + host_to_device_ratio=host_to_device_ratio, + host_size=host_size, + page_size=page_size, + layout=layout, + pin_memory=pin_memory, + device=device, + allocator_type=allocator_type, dcp_size=dcp_size, dcp_rank=dcp_rank, pool_label=pool_label, + is_dummy=is_dummy, ) + if is_dummy: + self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize + self.layout_dim = self.token_stride_size * self.layer_num + self.can_use_jit = False + self.staging_page_capacity = 0 + self.staging_token_capacity = 0 + self.staging_buffer = None + self.kv_buffer = None + self.data_refs = None + self.data_ptrs = None + logger.info( + "MLATokenToKVPoolHost dummy mode: allocator-only, size=%d tokens, " + "saving %.2f GB host memory", + self.size, + self.size * self.size_per_token / 1e9, + ) + return + # The JIT HiCache kernels also build with hipcc (ROCm): the PTX-only # helpers in hicache.cuh are guarded by USE_ROCM and the staged # write-back kernel has a ROCm path, so enable them on HIP too. This @@ -156,71 +168,6 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): ) self._init_write_back_staging_buffers() - def _init_dummy( - self, - device_pool: MLATokenToKVPool, - host_to_device_ratio: float, - host_size: int, - page_size: int, - layout: str, - pin_memory: bool, - device: str, - allocator_type: str, - dcp_size: int, - dcp_rank: int, - pool_label: str, - ) -> None: - self.device_pool = device_pool - self.pool_label = pool_label - self.dcp_size = dcp_size - self.dcp_rank = dcp_rank - assert page_size % dcp_size == 0, ( - f"HiCache host pool page_size ({page_size}) must be a multiple of " - f"dcp_size ({dcp_size})." - ) - self.page_size = page_size // dcp_size - self.layout = layout - self.pin_memory = pin_memory - self.device = device - self.allocator = get_allocator_from_storage(allocator_type) - - self.dtype = device_pool.store_dtype - self.size_per_token = self.get_size_per_token() - if host_size > 0: - self.size = sync_fixed_hicache_size( - int(host_size * 1e9 // self.size_per_token), host_size - ) - else: - self.size = int( - (getattr(device_pool, "host_capacity_tokens", None) or device_pool.size) - * host_to_device_ratio - ) - self.page_num = self.size // self.page_size + 1 - self.size = self.page_num * self.page_size - self.start_layer = device_pool.start_layer - self.end_layer = device_pool.end_layer - - self.token_stride_size = self.kv_cache_dim * self.dtype.itemsize - self.layout_dim = self.token_stride_size * self.layer_num - self.can_use_jit = False - self.can_use_write_back_jit = False - self.staging_page_capacity = 0 - self.staging_token_capacity = 0 - self.staging_buffer = None - self.kv_buffer = None - self.data_refs = None - self.data_ptrs = None - - logger.info( - "MLATokenToKVPoolHost dummy mode: allocator-only, size=%d tokens, " - "saving %.2f GB host memory", - self.size, - self.size * self.size_per_token / 1e9, - ) - - self.lock = threading.RLock() - self.clear() - def get_contiguous_buf_infos(self): """Return (data_ptrs, data_lens, item_lens) in the same format as device pool, for registering host memory with the disaggregation transfer engine.""" diff --git a/test/registered/kernels/ops/kvcache/test_mxfp8_host_allocation.py b/test/registered/kernels/ops/kvcache/test_mxfp8_host_allocation.py new file mode 100644 index 0000000000..4b99821dbb --- /dev/null +++ b/test/registered/kernels/ops/kvcache/test_mxfp8_host_allocation.py @@ -0,0 +1,50 @@ +import unittest + +import torch + +from sglang.srt.mem_cache.memory_pool import MHATokenToKVPoolMXFP8 +from sglang.srt.mem_cache.pool_host.mha_mxfp8 import MHATokenToKVPoolMXFP8Host +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-small") + + +class TestMXFP8HostAllocation(CustomTestCase): + def test_constructor_counts_payload_and_scale_pages(self): + pool = MHATokenToKVPoolMXFP8( + size=256, + page_size=128, + dtype=torch.float8_e4m3fn, + head_num=1, + head_dim=128, + layer_num=2, + device="cuda", + enable_memory_saver=False, + ) + host = MHATokenToKVPoolMXFP8Host( + pool, + host_to_device_ratio=2, + host_size=0, + page_size=pool.page_size, + layout="page_first", + pin_memory=False, + ) + self.addCleanup(host.destroy) + expected_token_bytes = ( + 2 + * pool.layer_num + * pool.head_num + * (pool.head_dim + pool.head_dim // pool.MXFP8_SCALE_BLOCK_SIZE) + ) + self.assertEqual(host.size_per_token, expected_token_bytes) + self.assertEqual(host.page_size, pool.page_size) + self.assertTrue(host.can_use_write_back_jit) + self.assertEqual( + host.kv_buffer.nbytes + host.k_scale_host.nbytes + host.v_scale_host.nbytes, + host.size * expected_token_bytes, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py index 6970608de0..588d4066d6 100644 --- a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py +++ b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py @@ -103,6 +103,50 @@ class TestMLAHostDedupPrimitives(unittest.TestCase): dsa_device_pool, slots, slots, layer_id=0, io_backend="kernel" ) + def test_dummy_and_physical_hosts_share_capacity_geometry(self): + """Dedup peers must allocate identical IDs despite different host payloads.""" + for dcp_size in (1, 2): + for host_size in (0, 0.000002): + with self.subTest(dcp_size=dcp_size, host_size=host_size): + pool = _device_pool_stub( + layer_num=2, + store_dtype=torch.float16, + kv_lora_rank=4, + qk_rope_head_dim=2, + size=8, + host_capacity_tokens=16, + start_layer=7, + end_layer=9, + device="cpu", + kv_buffer=[torch.zeros(10, 1, 6) for _ in range(2)], + ) + kwargs = dict( + host_to_device_ratio=2, + host_size=host_size, + page_size=4, + layout="page_first", + pin_memory=False, + dcp_size=dcp_size, + ) + host = MLATokenToKVPoolHost(pool, **kwargs) + dummy = MLATokenToKVPoolHost(pool, **kwargs, is_dummy=True) + self.addCleanup(host.destroy) + self.addCleanup(dummy.destroy) + self.assertEqual( + (host.size, host.page_num, host.page_size, host.size_per_token), + ( + dummy.size, + dummy.page_num, + dummy.page_size, + dummy.size_per_token, + ), + ) + self.assertIsNone(dummy.kv_buffer) + count = host.logical_size // 4 * 4 + self.assertEqual( + host.alloc(count).tolist(), dummy.alloc(count).tolist() + ) + def test_layer_broadcast_reuses_full_staging_capacity(self): broadcaster = MLAHostDedupBroadcaster.__new__(MLAHostDedupBroadcaster) broadcaster.is_src = True