[Fix] hicache: Key MLA Mamba state per TP rank in the HF3FS backend (#43817)

Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
Shuwen Wang
2026-10-17 11:08:17 +00:00
committed by GitHub
co-authored by Zhangheng
parent 465ddf9758
commit 5fcb07da34
2 changed files with 80 additions and 3 deletions
@@ -201,8 +201,11 @@ class HiCacheHF3FS(HiCacheStorage):
is_page_first_layout: bool = False,
use_mock_client: bool = False,
enable_storage_metrics: bool = False,
tp_size: int = 1,
):
self.rank = rank
self.tp_rank = rank
self.tp_size = tp_size
self.file_path = file_path
self.file_size = file_size
self.numjobs = numjobs
@@ -374,6 +377,7 @@ class HiCacheHF3FS(HiCacheStorage):
is_page_first_layout=is_page_first_layout,
use_mock_client=use_mock_client,
enable_storage_metrics=storage_config.enable_storage_metrics,
tp_size=storage_config.tp_size,
)
def _batch_get(
@@ -694,7 +698,9 @@ class HiCacheHF3FS(HiCacheStorage):
final_pages = 0
break
component_keys = [f"{key}_{pool_name}" for key in keys[:kv_pages]]
component_keys = [
self._pool_component_key(key, pool_name) for key in keys[:kv_pages]
]
exists_results = self.metadata_client.exists(
self.rank, component_keys, namespace=ctx.namespace
)
@@ -730,7 +736,7 @@ class HiCacheHF3FS(HiCacheStorage):
page_size = getattr(host_pool, "page_size", 1) or 1
page_num = len(keys)
component_keys = [f"{key}_{pool_name}" for key in keys]
component_keys = [self._pool_component_key(key, pool_name) for key in keys]
page_indices = self.metadata_client.get_page_indices(
self.rank, component_keys, namespace=ctx.namespace
)
@@ -779,6 +785,12 @@ class HiCacheHF3FS(HiCacheStorage):
return results
def _pool_component_key(self, key: str, pool_name: PoolName) -> str:
# Mamba state is TP-sharded even when the model's MLA KV is replicated.
if self.is_mla_model and pool_name == PoolName.MAMBA:
return f"{key}_{pool_name}_{self.tp_rank}_{self.tp_size}"
return f"{key}_{pool_name}"
def _pool_batch_set(self, transfer: PoolTransfer) -> List[bool]:
pool_name = transfer.name
ctx = self._pool_storage_ctx[pool_name]
@@ -788,7 +800,7 @@ class HiCacheHF3FS(HiCacheStorage):
page_size = getattr(host_pool, "page_size", 1) or 1
page_num = len(keys)
component_keys = [f"{key}_{pool_name}" for key in keys]
component_keys = [self._pool_component_key(key, pool_name) for key in keys]
key_with_prefix = [(k, "") for k in component_keys]
indices = self.metadata_client.reserve_and_allocate_page_indices(
self.rank, key_with_prefix, namespace=ctx.namespace
@@ -6,10 +6,17 @@ Usage:
import json
import os
import tempfile
import unittest
import torch
from test_hicache_storage_file_backend import HiCacheStorageBaseMixin
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
from sglang.srt.mem_cache.storage.hf3fs.mini_3fs_metadata_server import (
Hf3fsLocalMetadataClient,
)
from sglang.srt.mem_cache.storage.hf3fs.storage_hf3fs import HiCacheHF3FS
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
from sglang.test.test_utils import CustomTestCase
@@ -17,6 +24,9 @@ register_cuda_ci(est_time=210, stage="base-b", runner_config="2-gpu-large")
register_amd_ci(est_time=300, suite="base-b-test-2-gpu-large")
PAGE_BYTES = 16
class HiCacheStorage3FSBackendBaseMixin(HiCacheStorageBaseMixin):
"""Base mixin class with common setup and utilities"""
@@ -88,5 +98,60 @@ class TestHf3fsBackendAccuracy(HiCacheStorage3FSBackendBaseMixin, CustomTestCase
run_eval_accuracy_test(self)
class _StatePool:
page_size = 1
def __init__(self, value):
self.buffer = torch.full((1, PAGE_BYTES), value, dtype=torch.uint8)
def get_ksize_per_token(self):
return PAGE_BYTES
def get_dummy_flat_data_page(self):
return torch.zeros(PAGE_BYTES, dtype=torch.uint8)
def get_data_page(self, index, flat=True):
return self.buffer[index].clone()
def set_from_flat_data_page(self, index, data_page):
self.buffer[index].copy_(data_page)
class TestHf3fsMambaKeys(CustomTestCase):
def test_mla_mamba_round_trip_keeps_each_tp_rank(self):
"""MLA shares KV keys, but Mamba checkpoints must not overwrite another rank."""
tmp = tempfile.mkdtemp()
metadata = Hf3fsLocalMetadataClient()
transfer = PoolTransfer(
name=PoolName.MAMBA, keys=["page"], host_indices=torch.tensor([0])
)
ranks = []
for rank in range(2):
store = HiCacheHF3FS(
rank=rank,
file_path=os.path.join(tmp, "hicache.bin"),
file_size=PAGE_BYTES * 16,
numjobs=1,
bytes_per_page=PAGE_BYTES,
entries=8,
client_timeout=5,
dtype=torch.uint8,
metadata_client=metadata,
is_mla_model=True,
use_mock_client=True,
tp_size=2,
)
self.addCleanup(store.close)
pool = _StatePool(rank + 1)
store.register_mem_host_pool_v2(pool, PoolName.MAMBA)
self.assertEqual(store.batch_set_v2([transfer])[PoolName.MAMBA], [True])
ranks.append((store, pool))
for rank, (store, pool) in enumerate(ranks):
pool.buffer.zero_()
self.assertEqual(store.batch_get_v2([transfer])[PoolName.MAMBA], [True])
self.assertTrue(torch.all(pool.buffer == rank + 1))
if __name__ == "__main__":
unittest.main(verbosity=2)