diff --git a/python/sglang/srt/mem_cache/storage/hf3fs/storage_hf3fs.py b/python/sglang/srt/mem_cache/storage/hf3fs/storage_hf3fs.py index 7370f403e8..0fa7bda509 100644 --- a/python/sglang/srt/mem_cache/storage/hf3fs/storage_hf3fs.py +++ b/python/sglang/srt/mem_cache/storage/hf3fs/storage_hf3fs.py @@ -201,8 +201,11 @@ class HiCacheHF3FS(HiCacheStorage): is_page_first_layout: bool = False, use_mock_client: bool = False, enable_storage_metrics: bool = False, + tp_size: int = 1, ): self.rank = rank + self.tp_rank = rank + self.tp_size = tp_size self.file_path = file_path self.file_size = file_size self.numjobs = numjobs @@ -374,6 +377,7 @@ class HiCacheHF3FS(HiCacheStorage): is_page_first_layout=is_page_first_layout, use_mock_client=use_mock_client, enable_storage_metrics=storage_config.enable_storage_metrics, + tp_size=storage_config.tp_size, ) def _batch_get( @@ -694,7 +698,9 @@ class HiCacheHF3FS(HiCacheStorage): final_pages = 0 break - component_keys = [f"{key}_{pool_name}" for key in keys[:kv_pages]] + component_keys = [ + self._pool_component_key(key, pool_name) for key in keys[:kv_pages] + ] exists_results = self.metadata_client.exists( self.rank, component_keys, namespace=ctx.namespace ) @@ -730,7 +736,7 @@ class HiCacheHF3FS(HiCacheStorage): page_size = getattr(host_pool, "page_size", 1) or 1 page_num = len(keys) - component_keys = [f"{key}_{pool_name}" for key in keys] + component_keys = [self._pool_component_key(key, pool_name) for key in keys] page_indices = self.metadata_client.get_page_indices( self.rank, component_keys, namespace=ctx.namespace ) @@ -779,6 +785,12 @@ class HiCacheHF3FS(HiCacheStorage): return results + def _pool_component_key(self, key: str, pool_name: PoolName) -> str: + # Mamba state is TP-sharded even when the model's MLA KV is replicated. + if self.is_mla_model and pool_name == PoolName.MAMBA: + return f"{key}_{pool_name}_{self.tp_rank}_{self.tp_size}" + return f"{key}_{pool_name}" + def _pool_batch_set(self, transfer: PoolTransfer) -> List[bool]: pool_name = transfer.name ctx = self._pool_storage_ctx[pool_name] @@ -788,7 +800,7 @@ class HiCacheHF3FS(HiCacheStorage): page_size = getattr(host_pool, "page_size", 1) or 1 page_num = len(keys) - component_keys = [f"{key}_{pool_name}" for key in keys] + component_keys = [self._pool_component_key(key, pool_name) for key in keys] key_with_prefix = [(k, "") for k in component_keys] indices = self.metadata_client.reserve_and_allocate_page_indices( self.rank, key_with_prefix, namespace=ctx.namespace diff --git a/test/registered/hicache/test_hicache_storage_3fs_backend.py b/test/registered/hicache/test_hicache_storage_3fs_backend.py index 89afd74979..33fee81c62 100644 --- a/test/registered/hicache/test_hicache_storage_3fs_backend.py +++ b/test/registered/hicache/test_hicache_storage_3fs_backend.py @@ -6,10 +6,17 @@ Usage: import json import os +import tempfile import unittest +import torch from test_hicache_storage_file_backend import HiCacheStorageBaseMixin +from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer +from sglang.srt.mem_cache.storage.hf3fs.mini_3fs_metadata_server import ( + Hf3fsLocalMetadataClient, +) +from sglang.srt.mem_cache.storage.hf3fs.storage_hf3fs import HiCacheHF3FS from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.test_utils import CustomTestCase @@ -17,6 +24,9 @@ register_cuda_ci(est_time=210, stage="base-b", runner_config="2-gpu-large") register_amd_ci(est_time=300, suite="base-b-test-2-gpu-large") +PAGE_BYTES = 16 + + class HiCacheStorage3FSBackendBaseMixin(HiCacheStorageBaseMixin): """Base mixin class with common setup and utilities""" @@ -88,5 +98,60 @@ class TestHf3fsBackendAccuracy(HiCacheStorage3FSBackendBaseMixin, CustomTestCase run_eval_accuracy_test(self) +class _StatePool: + page_size = 1 + + def __init__(self, value): + self.buffer = torch.full((1, PAGE_BYTES), value, dtype=torch.uint8) + + def get_ksize_per_token(self): + return PAGE_BYTES + + def get_dummy_flat_data_page(self): + return torch.zeros(PAGE_BYTES, dtype=torch.uint8) + + def get_data_page(self, index, flat=True): + return self.buffer[index].clone() + + def set_from_flat_data_page(self, index, data_page): + self.buffer[index].copy_(data_page) + + +class TestHf3fsMambaKeys(CustomTestCase): + def test_mla_mamba_round_trip_keeps_each_tp_rank(self): + """MLA shares KV keys, but Mamba checkpoints must not overwrite another rank.""" + tmp = tempfile.mkdtemp() + metadata = Hf3fsLocalMetadataClient() + transfer = PoolTransfer( + name=PoolName.MAMBA, keys=["page"], host_indices=torch.tensor([0]) + ) + ranks = [] + for rank in range(2): + store = HiCacheHF3FS( + rank=rank, + file_path=os.path.join(tmp, "hicache.bin"), + file_size=PAGE_BYTES * 16, + numjobs=1, + bytes_per_page=PAGE_BYTES, + entries=8, + client_timeout=5, + dtype=torch.uint8, + metadata_client=metadata, + is_mla_model=True, + use_mock_client=True, + tp_size=2, + ) + self.addCleanup(store.close) + pool = _StatePool(rank + 1) + store.register_mem_host_pool_v2(pool, PoolName.MAMBA) + self.assertEqual(store.batch_set_v2([transfer])[PoolName.MAMBA], [True]) + ranks.append((store, pool)) + + for rank, (store, pool) in enumerate(ranks): + pool.buffer.zero_() + self.assertEqual(store.batch_get_v2([transfer])[PoolName.MAMBA], [True]) + self.assertTrue(torch.all(pool.buffer == rank + 1)) + + if __name__ == "__main__": unittest.main(verbosity=2)