[Fix] hicache: Key MLA Mamba state per TP rank in the HF3FS backend (#43817)
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
@@ -201,8 +201,11 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
is_page_first_layout: bool = False,
|
||||
use_mock_client: bool = False,
|
||||
enable_storage_metrics: bool = False,
|
||||
tp_size: int = 1,
|
||||
):
|
||||
self.rank = rank
|
||||
self.tp_rank = rank
|
||||
self.tp_size = tp_size
|
||||
self.file_path = file_path
|
||||
self.file_size = file_size
|
||||
self.numjobs = numjobs
|
||||
@@ -374,6 +377,7 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
is_page_first_layout=is_page_first_layout,
|
||||
use_mock_client=use_mock_client,
|
||||
enable_storage_metrics=storage_config.enable_storage_metrics,
|
||||
tp_size=storage_config.tp_size,
|
||||
)
|
||||
|
||||
def _batch_get(
|
||||
@@ -694,7 +698,9 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
final_pages = 0
|
||||
break
|
||||
|
||||
component_keys = [f"{key}_{pool_name}" for key in keys[:kv_pages]]
|
||||
component_keys = [
|
||||
self._pool_component_key(key, pool_name) for key in keys[:kv_pages]
|
||||
]
|
||||
exists_results = self.metadata_client.exists(
|
||||
self.rank, component_keys, namespace=ctx.namespace
|
||||
)
|
||||
@@ -730,7 +736,7 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
page_size = getattr(host_pool, "page_size", 1) or 1
|
||||
page_num = len(keys)
|
||||
|
||||
component_keys = [f"{key}_{pool_name}" for key in keys]
|
||||
component_keys = [self._pool_component_key(key, pool_name) for key in keys]
|
||||
page_indices = self.metadata_client.get_page_indices(
|
||||
self.rank, component_keys, namespace=ctx.namespace
|
||||
)
|
||||
@@ -779,6 +785,12 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
|
||||
return results
|
||||
|
||||
def _pool_component_key(self, key: str, pool_name: PoolName) -> str:
|
||||
# Mamba state is TP-sharded even when the model's MLA KV is replicated.
|
||||
if self.is_mla_model and pool_name == PoolName.MAMBA:
|
||||
return f"{key}_{pool_name}_{self.tp_rank}_{self.tp_size}"
|
||||
return f"{key}_{pool_name}"
|
||||
|
||||
def _pool_batch_set(self, transfer: PoolTransfer) -> List[bool]:
|
||||
pool_name = transfer.name
|
||||
ctx = self._pool_storage_ctx[pool_name]
|
||||
@@ -788,7 +800,7 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
page_size = getattr(host_pool, "page_size", 1) or 1
|
||||
page_num = len(keys)
|
||||
|
||||
component_keys = [f"{key}_{pool_name}" for key in keys]
|
||||
component_keys = [self._pool_component_key(key, pool_name) for key in keys]
|
||||
key_with_prefix = [(k, "") for k in component_keys]
|
||||
indices = self.metadata_client.reserve_and_allocate_page_indices(
|
||||
self.rank, key_with_prefix, namespace=ctx.namespace
|
||||
|
||||
Reference in New Issue
Block a user