[Fix] hicache: Key MLA Mamba state per TP rank in the SeaweedFS backend (#43816)

This commit is contained in:
Shuwen Wang
2026-10-18 15:09:58 +08:00
committed by GitHub
parent e6c2cd47a9
commit ffc4573c98
2 changed files with 30 additions and 3 deletions
@@ -111,6 +111,11 @@ class SeaweedFSStore(HiCacheStorage):
self.config = SeaweedFSConfig.from_extra_config(storage_config.extra_config)
scope = [self.config.prefix.strip("/")] if self.config.prefix else []
self.key_prefix = "/".join(scope + key_scope(storage_config))
self._mamba_rank_tag = (
f".tp{storage_config.tp_rank}of{storage_config.tp_size}"
if storage_config.is_mla_model
else ""
)
self._s3 = _make_s3_client(self.config)
self._executor = ThreadPoolExecutor(max_workers=self.config.max_workers)
self._ensure_bucket()
@@ -301,7 +306,12 @@ class SeaweedFSStore(HiCacheStorage):
def _component_key(self, key: str, pool_name) -> str:
name = getattr(pool_name, "value", pool_name)
return key if name == PoolName.KV.value else f"{key}.{name}"
if name == PoolName.KV.value:
return key
# Mamba state is TP-sharded even when the model's MLA KV is replicated.
if name == PoolName.MAMBA.value:
return f"{key}.{name}{self._mamba_rank_tag}"
return f"{key}.{name}"
def batch_exists_v2(
self,
@@ -291,8 +291,8 @@ class TestSeaweedFSStore(CustomTestCase):
# the backend only touches it through the host-pool page API, so what this
# proves is the per-pool object naming, routing and hit policies.
def _v2_store(self, layout="page_first"):
store = self._store()
def _v2_store(self, layout="page_first", **store_kwargs):
store = self._store(**store_kwargs)
kv, side = _host_pool(layout), _host_pool(layout)
store.register_mem_host_pool_v2(kv, PoolName.KV)
store.register_mem_host_pool_v2(side, PoolName.MAMBA)
@@ -325,6 +325,23 @@ class TestSeaweedFSStore(CustomTestCase):
for i in range(NUM_PAGES):
self.assertTrue(torch.equal(_page(pool, i), expected[p][i]))
def test_v2_mla_mamba_round_trip_keeps_each_tp_rank(self):
"""MLA shares KV keys, but Mamba checkpoints must not overwrite another rank."""
ranks = [self._v2_store(tp_rank=r, tp_size=2, is_mla=True) for r in (0, 1)]
for store, _, _, _, transfers in ranks:
transfers[1].keys = ranks[0][3]
self.assertEqual(
store.batch_set_v2(transfers[1:])[PoolName.MAMBA], [True] * NUM_PAGES
)
for store, _, side, _, transfers in ranks:
expected = [_page(side, i) for i in range(NUM_PAGES)]
side.kv_buffer.zero_()
self.assertEqual(
store.batch_get_v2(transfers[1:])[PoolName.MAMBA], [True] * NUM_PAGES
)
for i in range(NUM_PAGES):
self.assertTrue(torch.equal(_page(side, i), expected[i]))
def test_v2_kv_pages_are_the_v1_objects(self):
# KV pages keep their v1 object names, so v1 and v2 share one cache;
# side-pool pages are separate objects and never overwrite them.