[Fix] hicache: Key MLA Mamba state per TP rank in the file backend (#41907)

This commit is contained in:
Shuwen Wang
2026-10-18 15:10:50 +08:00
committed by GitHub
parent ffc4573c98
commit a11fa06be7
3 changed files with 47 additions and 48 deletions
+18 -9
View File
@@ -406,15 +406,17 @@ class HiCacheFile(HiCacheStorage):
attn_cp_size = storage_config.attn_cp_size
model_name = "-".join(model_name.split("/")) if model_name else ""
enable_pp = pp_size > 1
self.config_suffix = f"_{model_name}"
if not is_mla_model:
self.config_suffix += f"_{tp_rank}_{tp_size}"
if enable_pp:
self.config_suffix += f"_{pp_size}_{pp_rank}"
shard_suffix = f"_{pp_size}_{pp_rank}" if enable_pp else ""
# Under NSA context parallel each CP rank holds a disjoint slice of every
# page, so give each rank its own file key to avoid a cross-rank write race.
if attn_cp_size > 1:
self.config_suffix += f"_cp{attn_cp_rank}_{attn_cp_size}"
shard_suffix += f"_cp{attn_cp_rank}_{attn_cp_size}"
# MLA KV is replicated across TP ranks and shares one file per page;
# TP-sharded state is keyed by tp_config_suffix instead.
self.tp_config_suffix = f"_{model_name}_{tp_rank}_{tp_size}{shard_suffix}"
self.config_suffix = (
f"_{model_name}{shard_suffix}" if is_mla_model else self.tp_config_suffix
)
if not os.path.exists(self.file_path) and tp_rank == 0 and attn_cp_rank == 0:
os.makedirs(self.file_path)
@@ -450,11 +452,15 @@ class HiCacheFile(HiCacheStorage):
# module, so a top-level import here would be circular.
from sglang.srt.mem_cache.storage.file.lru_file_evictor import LRUFileEvictor
# Each rank creates and evicts its TP-sharded files; rank 0 also owns
# the shared MLA files.
owned_suffixes = (self.tp_config_suffix,)
if is_mla_model and tp_rank == 0:
owned_suffixes += (self.config_suffix,)
self._evictor = LRUFileEvictor(
self.file_path,
self.config_suffix,
owned_suffixes=owned_suffixes,
tp_rank=tp_rank,
is_mla_model=is_mla_model,
extra_config=storage_config.extra_config,
on_evict=(
self.metadata_cache.remove if self.metadata_cache is not None else None
@@ -462,6 +468,9 @@ class HiCacheFile(HiCacheStorage):
)
def _get_suffixed_key(self, key: str) -> str:
# Mamba state is TP-sharded even when the model's MLA KV is replicated.
if key.endswith(f".{PoolName.MAMBA.value}"):
return key + self.tp_config_suffix
return key + self.config_suffix
def _get_component_key(self, key: str, component_name: Optional[str] = None) -> str:
@@ -479,7 +488,7 @@ class HiCacheFile(HiCacheStorage):
continue
stem = fn[:-4]
# Only files belonging to this rank/model.
if stem.endswith(self.config_suffix):
if stem.endswith((self.config_suffix, self.tp_config_suffix)):
self.metadata_cache.add(stem)
def get(
@@ -66,22 +66,19 @@ class LRUFileEvictor:
def __init__(
self,
file_path: str,
config_suffix: str,
*,
owned_suffixes: Tuple[str, ...],
tp_rank: int,
is_mla_model: bool,
extra_config: Optional[dict] = None,
on_evict: Optional[Callable[[str], None]] = None,
) -> None:
self.file_path = file_path
self.config_suffix = config_suffix
# Files ending in one of these suffixes are created and evicted by this
# rank; other files in the shared directory are only read.
self._owned_suffixes = owned_suffixes
self._tp_rank = tp_rank
self._on_evict = on_evict
# MLA ranks share the same physical files, so centralize LRU bookkeeping
# on rank 0; non-MLA ranks each own their own files via the suffix.
self._is_storage_owner = (not is_mla_model) or (tp_rank == 0)
# suffixed_key -> file size in bytes; oldest at front.
self._lru: OrderedDict[str, int] = OrderedDict()
self._pending_writes: Set[str] = set()
@@ -90,14 +87,7 @@ class LRUFileEvictor:
self._load_config(extra_config or {})
self._eviction_configured = self.max_size_bytes > 0 or self.min_free_bytes > 0
self._eviction_enabled = self._eviction_configured and self._is_storage_owner
if self._eviction_configured and not self._is_storage_owner:
logger.info(
f"HiCacheFile rank {self._tp_rank} (MLA): eviction handled by rank 0; "
f"this rank skips LRU bookkeeping and will not create new files."
)
self._eviction_enabled = self.max_size_bytes > 0 or self.min_free_bytes > 0
if not self._eviction_enabled:
return
@@ -149,18 +139,11 @@ class LRUFileEvictor:
@property
def enabled(self) -> bool:
"""True when this rank actively evicts (configured AND storage owner)."""
"""True when a cap or free-space watermark is set."""
return self._eviction_enabled
@property
def configured(self) -> bool:
"""True when a cap or free-space watermark is set (on any rank)."""
return self._eviction_configured
@property
def is_storage_owner(self) -> bool:
"""True when this rank owns (and may create/evict) the on-disk files."""
return self._is_storage_owner
def _owns(self, suffixed_key: str) -> bool:
return suffixed_key.endswith(self._owned_suffixes)
def reserve(self, suffixed_key: str, value_bytes: int, key: str = "") -> bool:
"""Admit a new write of ``value_bytes``, evicting LRU victims as needed.
@@ -173,12 +156,12 @@ class LRUFileEvictor:
the cap, there is no evictable space, or the free-space watermark cannot
be met. When eviction is not configured the write is always admitted.
"""
if not self._eviction_configured:
if not self._eviction_enabled:
return True # unbounded storage: nothing to enforce
if not self._is_storage_owner:
if not self._owns(suffixed_key):
logger.warning(
f"HiCacheFile rank {self._tp_rank} is not the MLA storage owner; "
f"not caching new key {key} because file eviction is enabled."
f"HiCacheFile rank {self._tp_rank} does not own key {key}; "
f"not caching it because file eviction is enabled."
)
return False
if self.max_size_bytes > 0 and value_bytes > self.max_size_bytes:
@@ -240,7 +223,7 @@ class LRUFileEvictor:
def touch(self, suffixed_key: str, tensor_path: str) -> None:
"""Mark key as MRU, adopting an untracked on-disk file if needed."""
if not self._eviction_enabled:
if not self._eviction_enabled or not self._owns(suffixed_key):
return
with self._lock:
if suffixed_key in self._lru:
@@ -307,7 +290,7 @@ class LRUFileEvictor:
continue
stem = fn[:-4]
# Only files belonging to this rank/model.
if not stem.endswith(self.config_suffix):
if not self._owns(stem):
continue
fp = os.path.join(self.file_path, fn)
try: