diff --git a/python/sglang/srt/mem_cache/storage/nixl/hicache_nixl.py b/python/sglang/srt/mem_cache/storage/nixl/hicache_nixl.py index 2420503980..d972739a95 100644 --- a/python/sglang/srt/mem_cache/storage/nixl/hicache_nixl.py +++ b/python/sglang/srt/mem_cache/storage/nixl/hicache_nixl.py @@ -106,11 +106,12 @@ class HiCacheNixl(HiCacheStorage): self.backup_skip = self.is_mla_model and storage_config.tp_rank != 0 model_name = "-".join(model_name.split("/")) if model_name else "" + self.tp_config_suffix = f"_{model_name}_{tp_rank}_{tp_size}" if self.is_mla_model: self.config_suffix = f"_{model_name}" else: - self.config_suffix = f"_{model_name}_{tp_rank}_{tp_size}" + self.config_suffix = self.tp_config_suffix sync_mode = getattr( nixlBind, "NIXL_THREAD_SYNC_RW", nixlBind.NIXL_THREAD_SYNC_STRICT @@ -182,7 +183,10 @@ class HiCacheNixl(HiCacheStorage): if self._l3_cleaner is not None: self._l3_cleaner.start() - def _get_suffixed_key(self, key: str) -> str: + def _get_suffixed_key(self, key: str, pool_name: Optional[PoolName] = None) -> str: + # Mamba state is TP-sharded even when the model's MLA KV is replicated. + if pool_name == PoolName.MAMBA: + return key + self.tp_config_suffix return key + self.config_suffix def _get_component_key( @@ -190,7 +194,7 @@ class HiCacheNixl(HiCacheStorage): ) -> str: if component_name in (None, PoolName.KV): return self._get_suffixed_key(key) - return f"{self._get_suffixed_key(key)}_{component_name}" + return f"{self._get_suffixed_key(key, component_name)}_{component_name}" def _get_component_keys( self, keys: List[str], pool_name: Optional[PoolName] = None @@ -213,7 +217,7 @@ class HiCacheNixl(HiCacheStorage): suffixes = [f"_{pool_name}_{i}" for i in range(key_multiplier)] return [ - f"{self._get_suffixed_key(key)}{suffix}" + f"{self._get_suffixed_key(key, pool_name)}{suffix}" for key in keys for suffix in suffixes ] diff --git a/test/registered/unit/mem_cache/test_hicache_nixl_storage.py b/test/registered/unit/mem_cache/test_hicache_nixl_storage.py index 72422b8c52..bfc6eb207e 100644 --- a/test/registered/unit/mem_cache/test_hicache_nixl_storage.py +++ b/test/registered/unit/mem_cache/test_hicache_nixl_storage.py @@ -11,6 +11,7 @@ import subprocess import tempfile import time import unittest +from dataclasses import replace from types import SimpleNamespace from unittest import mock @@ -633,6 +634,41 @@ class TestNixlUnified(CustomTestCase): self.assertTrue(ctx.is_zero_copy) self.assertIs(ctx.host_pool, pool) + def test_mla_mamba_round_trip_keeps_each_tp_rank(self): + """MLA shares KV keys, but Mamba checkpoints must not overwrite another rank.""" + for zero_copy in (False, True): + with self.subTest(zero_copy=zero_copy): + caches, pools, expected = [], [], [] + transfer = PoolTransfer( + name=PoolName.MAMBA, + keys=[f"same_prefix_{zero_copy}"], + host_indices=torch.tensor([0]), + ) + for rank in range(2): + config = replace( + self.storage_config, tp_rank=rank, is_mla_model=True + ) + cache = HiCacheNixl(storage_config=config, file_path=self.test_dir) + self.addCleanup(cache.close) + pool = MockHybridPool(expose_zero_copy=zero_copy) + cache.register_mem_host_pool_v2(pool, PoolName.MAMBA) + pool.temporal_buffer[0].fill_(rank + 11) + pool.conv_buffer[0][0].fill_(rank + 21) + expected.append(pool.get_data_page(0).clone()) + caches.append(cache) + pools.append(pool) + self.assertEqual( + cache.batch_set_v2([transfer])[PoolName.MAMBA], [True] + ) + + for cache, pool, page in zip(caches, pools, expected): + pool.temporal_buffer.zero_() + pool.conv_buffer[0].zero_() + self.assertEqual( + cache.batch_get_v2([transfer])[PoolName.MAMBA], [True] + ) + self.assertTrue(torch.equal(pool.get_data_page(0), page)) + def test_register_mem_host_pool_v2_uses_persistent_bounce_otherwise(self): pool = MockHybridPool(expose_zero_copy=False) self.hicache.register_mem_host_pool_v2(pool, PoolName.MAMBA)