From 78996e3f02a88e914b1000fd734da5d3df8ebab8 Mon Sep 17 00:00:00 2001 From: weireweire Date: Fri, 9 Oct 2026 21:12:42 +0800 Subject: [PATCH] [Performance] Batch Mooncake hybrid-pool metadata and puts (#39696) Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang Co-authored-by: Claude Opus 5.5 --- .../storage/mooncake_store/mooncake_store.py | 178 ++++++++++++++---- .../test_mooncake_group_semantics.py | 167 ++++++++++++++++ 2 files changed, 310 insertions(+), 35 deletions(-) diff --git a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py index a30e4325bb..e66b9ed301 100644 --- a/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py +++ b/python/sglang/srt/mem_cache/storage/mooncake_store/mooncake_store.py @@ -874,14 +874,29 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore): # caller has to intersect these sets across ranks. restorable = list(range(1, kv_pages + 1)) + if not restorable: + return PoolTransferResult(0, hit_count, []) + + prepared = [] + all_component_keys = [] for transfer in pool_transfers or []: - if not restorable: - break component_keys, key_multiplier = self._get_hybrid_page_component_keys( keys, transfer ) component_keys = self._tag_keys(component_keys) - ex = self._batch_exist(component_keys, extra_info) + start = len(all_component_keys) + all_component_keys.extend(component_keys) + prepared.append((transfer, key_multiplier, start, len(all_component_keys))) + + all_exists = ( + self._batch_exist(all_component_keys, extra_info) + if all_component_keys + else [] + ) + for transfer, key_multiplier, start, end in prepared: + if not restorable: + break + ex = all_exists[start:end] if key_multiplier > 0: page_exists = [ all( @@ -923,13 +938,20 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore): return PoolTransferResult(final_pages, hit_count, restorable) def _batch_io_v2(self, transfers: List[PoolTransfer], is_set: bool): - # Unified v2 I/O path: each PoolTransfer can expand to one or more - # storage objects per logical page, but API still reports page-level result. - results: dict = {} + # Expand every pool first so one logical operation becomes one Mooncake + # RPC rather than one RPC per hybrid-cache component. + prepared = [] + all_key_strs = [] + all_ptrs = [] + all_sizes = [] + all_group_ids = [] + buffer_requests = [] for transfer in transfers: - host_pool = getattr(self, "registered_pools", {}).get(transfer.name) + host_pool = self.registered_pools.get(transfer.name) keys = transfer.keys - page_size = getattr(host_pool, "page_size", 1) or 1 + if host_pool is None: + raise ValueError(f"Unregistered Mooncake hybrid pool: {transfer.name}") + page_size = host_pool.page_size or 1 host_indices = transfer.host_indices assert len(keys) > 0 assert len(keys) == len(host_indices) // page_size @@ -939,39 +961,125 @@ class MooncakeStore(HiCacheStorage, MooncakeBaseStore): keys, transfer ) key_strs = self._tag_keys(key_strs) - ptr_list, element_size_list = host_pool.get_page_buffer_meta(host_indices) - if len(ptr_list) != len(key_strs): - ptr_list, element_size_list = self._pack_multi_buffer_meta( - key_strs, ptr_list, element_size_list - ) - - if is_set: - group_ids = ( + start = len(all_key_strs) + all_key_strs.extend(key_strs) + buffer_requests.append( + (host_pool, host_indices, key_strs, key_multiplier, start) + ) + if is_set and self._can_use_group_semantics(): + all_group_ids.extend( self._expand_group_ids(tagged_keys, key_multiplier) - if self._can_use_group_semantics() + ) + prepared.append((transfer.name, key_multiplier, start, len(all_key_strs))) + + if not prepared: + return {} + + exist_result = self._batch_exist(all_key_strs) if is_set else None + for host_pool, host_indices, key_strs, key_multiplier, start in buffer_requests: + pool_ptrs, pool_sizes = self._build_pool_buffer_meta( + host_pool=host_pool, + host_indices=host_indices, + key_strs=key_strs, + key_multiplier=key_multiplier, + exist_states=( + exist_result[start : start + len(key_strs)] + if exist_result is not None else None + ), + ) + all_ptrs.extend(pool_ptrs) + all_sizes.extend(pool_sizes) + + if any(isinstance(ptr, Sequence) for ptr in all_ptrs): + all_ptrs = [ + list(ptr) if isinstance(ptr, Sequence) else [ptr] for ptr in all_ptrs + ] + all_sizes = [ + list(size) if isinstance(size, Sequence) else [size] + for size in all_sizes + ] + + if is_set: + assert exist_result is not None + io_results = [0 if state == 1 else -1 for state in exist_result] + missing_idx = [i for i, state in enumerate(exist_result) if state != 1] + if missing_idx: + put_results = self._put_batch_zero_copy_impl( + [all_key_strs[i] for i in missing_idx], + [all_ptrs[i] for i in missing_idx], + [all_sizes[i] for i in missing_idx], + self._filter_group_ids( + all_group_ids if all_group_ids else None, missing_idx + ), ) - exist_result = self._batch_exist(key_strs) - io_results = [0 if state == 1 else -1 for state in exist_result] - missing_idx = [i for i, state in enumerate(exist_result) if state != 1] - if missing_idx: - put_results = self._put_batch_zero_copy_impl( - [key_strs[i] for i in missing_idx], - [ptr_list[i] for i in missing_idx], - [element_size_list[i] for i in missing_idx], - self._filter_group_ids(group_ids, missing_idx), - ) - for i, res in zip(missing_idx, put_results): - io_results[i] = res - else: - io_results = self._get_batch_zero_copy_impl( - key_strs, ptr_list, element_size_list - ) - results[transfer.name] = self._batch_postprocess( - io_results, is_set_operate=is_set, key_multiplier=key_multiplier + for i, res in zip(missing_idx, put_results): + io_results[i] = res + else: + io_results = self._get_batch_zero_copy_impl( + all_key_strs, all_ptrs, all_sizes + ) + + results: dict = {} + for name, key_multiplier, start, end in prepared: + results[name] = self._batch_postprocess( + io_results[start:end], + is_set_operate=is_set, + key_multiplier=key_multiplier, ) return results + def _build_pool_buffer_meta( + self, + host_pool: HostKVCache, + host_indices: torch.Tensor, + key_strs: List[str], + key_multiplier: int, + exist_states: Optional[List[int]], + ) -> Tuple[List[Any], List[Any]]: + # Only pages the store is missing need per-layer addresses. Zero + # placeholders keep result and group offsets aligned for the rest. + object_indices = list(range(len(key_strs))) + if exist_states is not None: + missing_pages = [ + page + for page in range(len(key_strs) // key_multiplier) + if any( + state != 1 + for state in exist_states[ + page * key_multiplier : (page + 1) * key_multiplier + ] + ) + ] + if not missing_pages: + return [0] * len(key_strs), [0] * len(key_strs) + if len(missing_pages) * key_multiplier < len(key_strs): + # Metadata generation consumes CPU indices anyway, so select + # the logical pages there rather than uploading a new index. + page_size = host_pool.page_size or 1 + host_indices = host_indices.detach().to(device="cpu") + host_indices = host_indices.reshape(-1, page_size)[ + missing_pages + ].reshape(-1) + object_indices = [ + page * key_multiplier + component + for page in missing_pages + for component in range(key_multiplier) + ] + + selected_keys = [key_strs[i] for i in object_indices] + ptr_list, element_size_list = host_pool.get_page_buffer_meta(host_indices) + if len(ptr_list) != len(selected_keys): + ptr_list, element_size_list = self._pack_multi_buffer_meta( + selected_keys, ptr_list, element_size_list + ) + pool_ptrs = [0] * len(key_strs) + pool_sizes = [0] * len(key_strs) + for index, ptr, size in zip(object_indices, ptr_list, element_size_list): + pool_ptrs[index] = ptr + pool_sizes[index] = size + return pool_ptrs, pool_sizes + def batch_get_v2( self, transfers: List[PoolTransfer], diff --git a/test/registered/unit/mem_cache/test_mooncake_group_semantics.py b/test/registered/unit/mem_cache/test_mooncake_group_semantics.py index 0e814df539..eead5d800c 100644 --- a/test/registered/unit/mem_cache/test_mooncake_group_semantics.py +++ b/test/registered/unit/mem_cache/test_mooncake_group_semantics.py @@ -79,6 +79,7 @@ def _fake_store_class(): def __init__(self): self.batch_put_calls = [] + self.batch_exist_calls = [] self.existing_keys = set() self.objects = {} type(self).instances.append(self) @@ -100,6 +101,7 @@ def _fake_store_class(): return self.objects.get(key) def batch_is_exist(self, keys): + self.batch_exist_calls.append(list(keys)) return [1 if key in self.existing_keys else 0 for key in keys] def batch_put_from(self, keys, ptrs, sizes, *args): @@ -452,6 +454,171 @@ class TestMooncakeGroupSemantics(CustomTestCase): ["sglang-hicache:tag_page0", "sglang-hicache:tag_page1"], ) + def test_v2_batches_scalar_and_multi_buffer_pools_in_one_rpc(self): + store, fake_store = _make_store(extra_backend_tag="tag", is_mla_model=True) + store.register_mem_host_pool_v2(FakeIndexerPool(), PoolName.INDEXER) + store.register_mem_host_pool_v2(FakeMultiBufferPool(), PoolName.DEEPSEEK_V4_C4) + + results = store.batch_set_v2( + [ + PoolTransfer( + name=PoolName.INDEXER, + keys=["page0"], + host_indices=torch.tensor([0]), + ), + PoolTransfer( + name=PoolName.DEEPSEEK_V4_C4, + keys=["page0"], + host_indices=torch.tensor([0]), + ), + ] + ) + + self.assertEqual(results[PoolName.INDEXER], [True]) + self.assertEqual(results[PoolName.DEEPSEEK_V4_C4], [True]) + self.assertEqual(len(fake_store.batch_exist_calls), 1) + self.assertEqual(len(fake_store.batch_put_calls), 1) + call = fake_store.batch_put_calls[0] + self.assertEqual(call["method"], "batch_put_from_multi_buffers") + self.assertEqual( + call["keys"], + ["tag_page0__indexer", "tag_page0__deepseek_v4_c4"], + ) + self.assertEqual(call["ptrs"], [[3000], [4000, 4001]]) + self.assertEqual(call["sizes"], [[8], [8, 16]]) + self.assertEqual( + call["args"][0].group_ids, + ["sglang-hicache:tag_page0", "sglang-hicache:tag_page0"], + ) + + def test_v2_mixed_hits_build_metadata_only_for_missing_pages(self): + """Pages already in the store must not have buffer metadata built for them.""" + + class RecordingIndexerPool(FakeIndexerPool): + def __init__(self): + super().__init__() + self.meta_calls = [] + + def get_page_buffer_meta(self, indices): + self.meta_calls.append(indices.tolist()) + return [3000 + int(i) for i in indices], [8] * len(indices) + + class RecordingMultiBufferPool(FakeMultiBufferPool): + def __init__(self): + super().__init__() + self.meta_calls = [] + + def get_page_buffer_meta(self, indices): + self.meta_calls.append(indices.tolist()) + ptrs, sizes = [], [] + for i in indices: + ptrs.extend([4000 + int(i) * 10, 4001 + int(i) * 10]) + sizes.extend([8, 16]) + return ptrs, sizes + + store, fake_store = _make_store(extra_backend_tag="tag", is_mla_model=True) + indexer_pool = RecordingIndexerPool() + c4_pool = RecordingMultiBufferPool() + store.register_mem_host_pool_v2(indexer_pool, PoolName.INDEXER) + store.register_mem_host_pool_v2(c4_pool, PoolName.DEEPSEEK_V4_C4) + fake_store.existing_keys.update( + {"tag_page1__indexer", "tag_page1__deepseek_v4_c4"} + ) + keys = ["page0", "page1", "page2"] + host_indices = torch.tensor([5, 6, 7]) + + results = store.batch_set_v2( + [ + PoolTransfer( + name=PoolName.INDEXER, keys=keys, host_indices=host_indices + ), + PoolTransfer( + name=PoolName.DEEPSEEK_V4_C4, keys=keys, host_indices=host_indices + ), + ] + ) + + self.assertEqual(results[PoolName.INDEXER], [True, True, True]) + self.assertEqual(results[PoolName.DEEPSEEK_V4_C4], [True, True, True]) + # Only the missing pages' host slots are materialized. + self.assertEqual(indexer_pool.meta_calls, [[5, 7]]) + self.assertEqual(c4_pool.meta_calls, [[5, 7]]) + self.assertEqual(len(fake_store.batch_put_calls), 1) + call = fake_store.batch_put_calls[0] + self.assertEqual( + call["keys"], + [ + "tag_page0__indexer", + "tag_page2__indexer", + "tag_page0__deepseek_v4_c4", + "tag_page2__deepseek_v4_c4", + ], + ) + self.assertEqual(call["ptrs"], [[3005], [3007], [4050, 4051], [4070, 4071]]) + self.assertEqual(call["sizes"], [[8], [8], [8, 16], [8, 16]]) + self.assertEqual( + call["args"][0].group_ids, + [ + "sglang-hicache:tag_page0", + "sglang-hicache:tag_page2", + "sglang-hicache:tag_page0", + "sglang-hicache:tag_page2", + ], + ) + + def test_v2_partly_stored_multi_object_page_puts_only_missing_objects(self): + """A page with some objects stored must put the rest with their own pointers.""" + + class RecordingKVDraftPool: + page_size = 1 + + def __init__(self): + self.kv_buffer = torch.empty((128,), dtype=torch.uint8) + self.meta_calls = [] + + def get_page_buffer_meta(self, indices): + self.meta_calls.append(indices.tolist()) + ptrs, sizes = [], [] + for i in indices: + ptrs.extend([5000 + int(i) * 10, 5001 + int(i) * 10]) + sizes.extend([8, 8]) + return ptrs, sizes + + store, fake_store = _make_store(extra_backend_tag="tag") + draft_pool = RecordingKVDraftPool() + store.register_mem_host_pool_v2(draft_pool, PoolName.DRAFT) + fake_store.existing_keys.update( + {"tag_page0_0_draft_k", "tag_page1_0_draft_k", "tag_page1_0_draft_v"} + ) + + results = store.batch_set_v2( + [ + PoolTransfer( + name=PoolName.DRAFT, + keys=["page0", "page1", "page2"], + host_indices=torch.tensor([5, 6, 7]), + ) + ] + ) + + self.assertEqual(results[PoolName.DRAFT], [True, True, True]) + self.assertEqual(draft_pool.meta_calls, [[5, 7]]) + self.assertEqual(len(fake_store.batch_put_calls), 1) + call = fake_store.batch_put_calls[0] + self.assertEqual( + call["keys"], + ["tag_page0_0_draft_v", "tag_page2_0_draft_k", "tag_page2_0_draft_v"], + ) + self.assertEqual(call["ptrs"], [5051, 5070, 5071]) + self.assertEqual( + call["args"][0].group_ids, + [ + "sglang-hicache:tag_page0", + "sglang-hicache:tag_page2", + "sglang-hicache:tag_page2", + ], + ) + def test_model_names_isolate_the_same_logical_key(self): store_a, fake_store_a = _make_store( enable_group_semantics=False, model_name="org/model-a"