[Fix] hicache: Key MLA Mamba state per TP rank in the HF3FS backend (#43817)

Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
Shuwen Wang
2026-10-17 11:08:17 +00:00
committed by GitHub
co-authored by Zhangheng
parent 465ddf9758
commit 5fcb07da34
2 changed files with 80 additions and 3 deletions
@@ -201,8 +201,11 @@ class HiCacheHF3FS(HiCacheStorage):
is_page_first_layout: bool = False,
use_mock_client: bool = False,
enable_storage_metrics: bool = False,
tp_size: int = 1,
):
self.rank = rank
self.tp_rank = rank
self.tp_size = tp_size
self.file_path = file_path
self.file_size = file_size
self.numjobs = numjobs
@@ -374,6 +377,7 @@ class HiCacheHF3FS(HiCacheStorage):
is_page_first_layout=is_page_first_layout,
use_mock_client=use_mock_client,
enable_storage_metrics=storage_config.enable_storage_metrics,
tp_size=storage_config.tp_size,
)
def _batch_get(
@@ -694,7 +698,9 @@ class HiCacheHF3FS(HiCacheStorage):
final_pages = 0
break
component_keys = [f"{key}_{pool_name}" for key in keys[:kv_pages]]
component_keys = [
self._pool_component_key(key, pool_name) for key in keys[:kv_pages]
]
exists_results = self.metadata_client.exists(
self.rank, component_keys, namespace=ctx.namespace
)
@@ -730,7 +736,7 @@ class HiCacheHF3FS(HiCacheStorage):
page_size = getattr(host_pool, "page_size", 1) or 1
page_num = len(keys)
component_keys = [f"{key}_{pool_name}" for key in keys]
component_keys = [self._pool_component_key(key, pool_name) for key in keys]
page_indices = self.metadata_client.get_page_indices(
self.rank, component_keys, namespace=ctx.namespace
)
@@ -779,6 +785,12 @@ class HiCacheHF3FS(HiCacheStorage):
return results
def _pool_component_key(self, key: str, pool_name: PoolName) -> str:
# Mamba state is TP-sharded even when the model's MLA KV is replicated.
if self.is_mla_model and pool_name == PoolName.MAMBA:
return f"{key}_{pool_name}_{self.tp_rank}_{self.tp_size}"
return f"{key}_{pool_name}"
def _pool_batch_set(self, transfer: PoolTransfer) -> List[bool]:
pool_name = transfer.name
ctx = self._pool_storage_ctx[pool_name]
@@ -788,7 +800,7 @@ class HiCacheHF3FS(HiCacheStorage):
page_size = getattr(host_pool, "page_size", 1) or 1
page_num = len(keys)
component_keys = [f"{key}_{pool_name}" for key in keys]
component_keys = [self._pool_component_key(key, pool_name) for key in keys]
key_with_prefix = [(k, "") for k in component_keys]
indices = self.metadata_client.reserve_and_allocate_page_indices(
self.rank, key_with_prefix, namespace=ctx.namespace