[Fix] hicache: Key MLA Mamba state per TP rank in the HF3FS backend (#43817)
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
@@ -201,8 +201,11 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
is_page_first_layout: bool = False,
|
||||
use_mock_client: bool = False,
|
||||
enable_storage_metrics: bool = False,
|
||||
tp_size: int = 1,
|
||||
):
|
||||
self.rank = rank
|
||||
self.tp_rank = rank
|
||||
self.tp_size = tp_size
|
||||
self.file_path = file_path
|
||||
self.file_size = file_size
|
||||
self.numjobs = numjobs
|
||||
@@ -374,6 +377,7 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
is_page_first_layout=is_page_first_layout,
|
||||
use_mock_client=use_mock_client,
|
||||
enable_storage_metrics=storage_config.enable_storage_metrics,
|
||||
tp_size=storage_config.tp_size,
|
||||
)
|
||||
|
||||
def _batch_get(
|
||||
@@ -694,7 +698,9 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
final_pages = 0
|
||||
break
|
||||
|
||||
component_keys = [f"{key}_{pool_name}" for key in keys[:kv_pages]]
|
||||
component_keys = [
|
||||
self._pool_component_key(key, pool_name) for key in keys[:kv_pages]
|
||||
]
|
||||
exists_results = self.metadata_client.exists(
|
||||
self.rank, component_keys, namespace=ctx.namespace
|
||||
)
|
||||
@@ -730,7 +736,7 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
page_size = getattr(host_pool, "page_size", 1) or 1
|
||||
page_num = len(keys)
|
||||
|
||||
component_keys = [f"{key}_{pool_name}" for key in keys]
|
||||
component_keys = [self._pool_component_key(key, pool_name) for key in keys]
|
||||
page_indices = self.metadata_client.get_page_indices(
|
||||
self.rank, component_keys, namespace=ctx.namespace
|
||||
)
|
||||
@@ -779,6 +785,12 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
|
||||
return results
|
||||
|
||||
def _pool_component_key(self, key: str, pool_name: PoolName) -> str:
|
||||
# Mamba state is TP-sharded even when the model's MLA KV is replicated.
|
||||
if self.is_mla_model and pool_name == PoolName.MAMBA:
|
||||
return f"{key}_{pool_name}_{self.tp_rank}_{self.tp_size}"
|
||||
return f"{key}_{pool_name}"
|
||||
|
||||
def _pool_batch_set(self, transfer: PoolTransfer) -> List[bool]:
|
||||
pool_name = transfer.name
|
||||
ctx = self._pool_storage_ctx[pool_name]
|
||||
@@ -788,7 +800,7 @@ class HiCacheHF3FS(HiCacheStorage):
|
||||
page_size = getattr(host_pool, "page_size", 1) or 1
|
||||
page_num = len(keys)
|
||||
|
||||
component_keys = [f"{key}_{pool_name}" for key in keys]
|
||||
component_keys = [self._pool_component_key(key, pool_name) for key in keys]
|
||||
key_with_prefix = [(k, "") for k in component_keys]
|
||||
indices = self.metadata_client.reserve_and_allocate_page_indices(
|
||||
self.rank, key_with_prefix, namespace=ctx.namespace
|
||||
|
||||
@@ -6,10 +6,17 @@ Usage:
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
from test_hicache_storage_file_backend import HiCacheStorageBaseMixin
|
||||
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
|
||||
from sglang.srt.mem_cache.storage.hf3fs.mini_3fs_metadata_server import (
|
||||
Hf3fsLocalMetadataClient,
|
||||
)
|
||||
from sglang.srt.mem_cache.storage.hf3fs.storage_hf3fs import HiCacheHF3FS
|
||||
from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -17,6 +24,9 @@ register_cuda_ci(est_time=210, stage="base-b", runner_config="2-gpu-large")
|
||||
register_amd_ci(est_time=300, suite="base-b-test-2-gpu-large")
|
||||
|
||||
|
||||
PAGE_BYTES = 16
|
||||
|
||||
|
||||
class HiCacheStorage3FSBackendBaseMixin(HiCacheStorageBaseMixin):
|
||||
"""Base mixin class with common setup and utilities"""
|
||||
|
||||
@@ -88,5 +98,60 @@ class TestHf3fsBackendAccuracy(HiCacheStorage3FSBackendBaseMixin, CustomTestCase
|
||||
run_eval_accuracy_test(self)
|
||||
|
||||
|
||||
class _StatePool:
|
||||
page_size = 1
|
||||
|
||||
def __init__(self, value):
|
||||
self.buffer = torch.full((1, PAGE_BYTES), value, dtype=torch.uint8)
|
||||
|
||||
def get_ksize_per_token(self):
|
||||
return PAGE_BYTES
|
||||
|
||||
def get_dummy_flat_data_page(self):
|
||||
return torch.zeros(PAGE_BYTES, dtype=torch.uint8)
|
||||
|
||||
def get_data_page(self, index, flat=True):
|
||||
return self.buffer[index].clone()
|
||||
|
||||
def set_from_flat_data_page(self, index, data_page):
|
||||
self.buffer[index].copy_(data_page)
|
||||
|
||||
|
||||
class TestHf3fsMambaKeys(CustomTestCase):
|
||||
def test_mla_mamba_round_trip_keeps_each_tp_rank(self):
|
||||
"""MLA shares KV keys, but Mamba checkpoints must not overwrite another rank."""
|
||||
tmp = tempfile.mkdtemp()
|
||||
metadata = Hf3fsLocalMetadataClient()
|
||||
transfer = PoolTransfer(
|
||||
name=PoolName.MAMBA, keys=["page"], host_indices=torch.tensor([0])
|
||||
)
|
||||
ranks = []
|
||||
for rank in range(2):
|
||||
store = HiCacheHF3FS(
|
||||
rank=rank,
|
||||
file_path=os.path.join(tmp, "hicache.bin"),
|
||||
file_size=PAGE_BYTES * 16,
|
||||
numjobs=1,
|
||||
bytes_per_page=PAGE_BYTES,
|
||||
entries=8,
|
||||
client_timeout=5,
|
||||
dtype=torch.uint8,
|
||||
metadata_client=metadata,
|
||||
is_mla_model=True,
|
||||
use_mock_client=True,
|
||||
tp_size=2,
|
||||
)
|
||||
self.addCleanup(store.close)
|
||||
pool = _StatePool(rank + 1)
|
||||
store.register_mem_host_pool_v2(pool, PoolName.MAMBA)
|
||||
self.assertEqual(store.batch_set_v2([transfer])[PoolName.MAMBA], [True])
|
||||
ranks.append((store, pool))
|
||||
|
||||
for rank, (store, pool) in enumerate(ranks):
|
||||
pool.buffer.zero_()
|
||||
self.assertEqual(store.batch_get_v2([transfer])[PoolName.MAMBA], [True])
|
||||
self.assertTrue(torch.all(pool.buffer == rank + 1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
|
||||
Reference in New Issue
Block a user