diff --git a/python/sglang/srt/mem_cache/hicache_storage.py b/python/sglang/srt/mem_cache/hicache_storage.py index 2aa43ed307..f93600cfe1 100644 --- a/python/sglang/srt/mem_cache/hicache_storage.py +++ b/python/sglang/srt/mem_cache/hicache_storage.py @@ -406,15 +406,17 @@ class HiCacheFile(HiCacheStorage): attn_cp_size = storage_config.attn_cp_size model_name = "-".join(model_name.split("/")) if model_name else "" enable_pp = pp_size > 1 - self.config_suffix = f"_{model_name}" - if not is_mla_model: - self.config_suffix += f"_{tp_rank}_{tp_size}" - if enable_pp: - self.config_suffix += f"_{pp_size}_{pp_rank}" + shard_suffix = f"_{pp_size}_{pp_rank}" if enable_pp else "" # Under NSA context parallel each CP rank holds a disjoint slice of every # page, so give each rank its own file key to avoid a cross-rank write race. if attn_cp_size > 1: - self.config_suffix += f"_cp{attn_cp_rank}_{attn_cp_size}" + shard_suffix += f"_cp{attn_cp_rank}_{attn_cp_size}" + # MLA KV is replicated across TP ranks and shares one file per page; + # TP-sharded state is keyed by tp_config_suffix instead. + self.tp_config_suffix = f"_{model_name}_{tp_rank}_{tp_size}{shard_suffix}" + self.config_suffix = ( + f"_{model_name}{shard_suffix}" if is_mla_model else self.tp_config_suffix + ) if not os.path.exists(self.file_path) and tp_rank == 0 and attn_cp_rank == 0: os.makedirs(self.file_path) @@ -450,11 +452,15 @@ class HiCacheFile(HiCacheStorage): # module, so a top-level import here would be circular. from sglang.srt.mem_cache.storage.file.lru_file_evictor import LRUFileEvictor + # Each rank creates and evicts its TP-sharded files; rank 0 also owns + # the shared MLA files. + owned_suffixes = (self.tp_config_suffix,) + if is_mla_model and tp_rank == 0: + owned_suffixes += (self.config_suffix,) self._evictor = LRUFileEvictor( self.file_path, - self.config_suffix, + owned_suffixes=owned_suffixes, tp_rank=tp_rank, - is_mla_model=is_mla_model, extra_config=storage_config.extra_config, on_evict=( self.metadata_cache.remove if self.metadata_cache is not None else None @@ -462,6 +468,9 @@ class HiCacheFile(HiCacheStorage): ) def _get_suffixed_key(self, key: str) -> str: + # Mamba state is TP-sharded even when the model's MLA KV is replicated. + if key.endswith(f".{PoolName.MAMBA.value}"): + return key + self.tp_config_suffix return key + self.config_suffix def _get_component_key(self, key: str, component_name: Optional[str] = None) -> str: @@ -479,7 +488,7 @@ class HiCacheFile(HiCacheStorage): continue stem = fn[:-4] # Only files belonging to this rank/model. - if stem.endswith(self.config_suffix): + if stem.endswith((self.config_suffix, self.tp_config_suffix)): self.metadata_cache.add(stem) def get( diff --git a/python/sglang/srt/mem_cache/storage/file/lru_file_evictor.py b/python/sglang/srt/mem_cache/storage/file/lru_file_evictor.py index 6b7997ec16..60847bfea0 100644 --- a/python/sglang/srt/mem_cache/storage/file/lru_file_evictor.py +++ b/python/sglang/srt/mem_cache/storage/file/lru_file_evictor.py @@ -66,22 +66,19 @@ class LRUFileEvictor: def __init__( self, file_path: str, - config_suffix: str, *, + owned_suffixes: Tuple[str, ...], tp_rank: int, - is_mla_model: bool, extra_config: Optional[dict] = None, on_evict: Optional[Callable[[str], None]] = None, ) -> None: self.file_path = file_path - self.config_suffix = config_suffix + # Files ending in one of these suffixes are created and evicted by this + # rank; other files in the shared directory are only read. + self._owned_suffixes = owned_suffixes self._tp_rank = tp_rank self._on_evict = on_evict - # MLA ranks share the same physical files, so centralize LRU bookkeeping - # on rank 0; non-MLA ranks each own their own files via the suffix. - self._is_storage_owner = (not is_mla_model) or (tp_rank == 0) - # suffixed_key -> file size in bytes; oldest at front. self._lru: OrderedDict[str, int] = OrderedDict() self._pending_writes: Set[str] = set() @@ -90,14 +87,7 @@ class LRUFileEvictor: self._load_config(extra_config or {}) - self._eviction_configured = self.max_size_bytes > 0 or self.min_free_bytes > 0 - self._eviction_enabled = self._eviction_configured and self._is_storage_owner - if self._eviction_configured and not self._is_storage_owner: - logger.info( - f"HiCacheFile rank {self._tp_rank} (MLA): eviction handled by rank 0; " - f"this rank skips LRU bookkeeping and will not create new files." - ) - + self._eviction_enabled = self.max_size_bytes > 0 or self.min_free_bytes > 0 if not self._eviction_enabled: return @@ -149,18 +139,11 @@ class LRUFileEvictor: @property def enabled(self) -> bool: - """True when this rank actively evicts (configured AND storage owner).""" + """True when a cap or free-space watermark is set.""" return self._eviction_enabled - @property - def configured(self) -> bool: - """True when a cap or free-space watermark is set (on any rank).""" - return self._eviction_configured - - @property - def is_storage_owner(self) -> bool: - """True when this rank owns (and may create/evict) the on-disk files.""" - return self._is_storage_owner + def _owns(self, suffixed_key: str) -> bool: + return suffixed_key.endswith(self._owned_suffixes) def reserve(self, suffixed_key: str, value_bytes: int, key: str = "") -> bool: """Admit a new write of ``value_bytes``, evicting LRU victims as needed. @@ -173,12 +156,12 @@ class LRUFileEvictor: the cap, there is no evictable space, or the free-space watermark cannot be met. When eviction is not configured the write is always admitted. """ - if not self._eviction_configured: + if not self._eviction_enabled: return True # unbounded storage: nothing to enforce - if not self._is_storage_owner: + if not self._owns(suffixed_key): logger.warning( - f"HiCacheFile rank {self._tp_rank} is not the MLA storage owner; " - f"not caching new key {key} because file eviction is enabled." + f"HiCacheFile rank {self._tp_rank} does not own key {key}; " + f"not caching it because file eviction is enabled." ) return False if self.max_size_bytes > 0 and value_bytes > self.max_size_bytes: @@ -240,7 +223,7 @@ class LRUFileEvictor: def touch(self, suffixed_key: str, tensor_path: str) -> None: """Mark key as MRU, adopting an untracked on-disk file if needed.""" - if not self._eviction_enabled: + if not self._eviction_enabled or not self._owns(suffixed_key): return with self._lock: if suffixed_key in self._lru: @@ -307,7 +290,7 @@ class LRUFileEvictor: continue stem = fn[:-4] # Only files belonging to this rank/model. - if not stem.endswith(self.config_suffix): + if not self._owns(stem): continue fp = os.path.join(self.file_path, fn) try: diff --git a/test/registered/unit/mem_cache/test_hicache_file_lru_unit.py b/test/registered/unit/mem_cache/test_hicache_file_lru_unit.py index 4bb491797a..300929fa09 100644 --- a/test/registered/unit/mem_cache/test_hicache_file_lru_unit.py +++ b/test/registered/unit/mem_cache/test_hicache_file_lru_unit.py @@ -284,18 +284,26 @@ class TestCPSuffix(HiCacheFileLRUTestBase): class TestMLAOwnerGating(HiCacheFileLRUTestBase): def test_mla_rank0_owns_eviction(self): b = self.make_backend(max_size="200", is_mla=True, tp_rank=0, tp_size=2) - self.assertTrue(b._evictor.is_storage_owner) self.assertTrue(b._evictor.enabled) - def test_mla_rank1_skips_eviction(self): + def test_mla_rank1_only_creates_its_mamba_files(self): b = self.make_backend(max_size="200", is_mla=True, tp_rank=1, tp_size=2) - self.assertFalse(b._evictor.is_storage_owner) - self.assertFalse(b._evictor.enabled) - # Non-owner MLA ranks must not create new files when eviction is on. + # Replicated MLA files are created by rank 0 only. self.assertFalse(b.set("a", _t(50))) self.assertFalse(b.exists("a")) - self.assertEqual(len(b._evictor._lru), 0) - self.assertEqual(b._evictor._total_bytes, 0) + self.assertTrue(b.set("a.mamba", _t(50))) + self.assertEqual(b._evictor._total_bytes, 50) + + def test_mla_mamba_round_trip_keeps_each_tp_rank(self): + """MLA shares KV files, but Mamba pages must not overwrite another rank's.""" + ranks = [ + self.make_backend(is_mla=True, tp_rank=rank, tp_size=2, subdir="shared") + for rank in range(2) + ] + for rank, b in enumerate(ranks): + self.assertTrue(b.set("a.mamba", _t(16, rank + 1))) + for rank, b in enumerate(ranks): + self.assertTrue(torch.equal(b.get("a.mamba", _t(16)), _t(16, rank + 1))) def test_mla_rank1_can_touch_existing_file(self): # Non-owner ranks may still touch existing files, just not create new ones. @@ -310,7 +318,6 @@ class TestMLAOwnerGating(HiCacheFileLRUTestBase): def test_non_mla_each_rank_owns_its_files(self): # Non-MLA: even rank > 0 is its own owner because suffix isolates files. b = self.make_backend(max_size="200", is_mla=False, tp_rank=3, tp_size=4) - self.assertTrue(b._evictor.is_storage_owner) self.assertTrue(b._evictor.enabled)