[Fix] hicache: Key MLA Mamba state per TP rank in the file backend (#41907)
This commit is contained in:
@@ -406,15 +406,17 @@ class HiCacheFile(HiCacheStorage):
|
||||
attn_cp_size = storage_config.attn_cp_size
|
||||
model_name = "-".join(model_name.split("/")) if model_name else ""
|
||||
enable_pp = pp_size > 1
|
||||
self.config_suffix = f"_{model_name}"
|
||||
if not is_mla_model:
|
||||
self.config_suffix += f"_{tp_rank}_{tp_size}"
|
||||
if enable_pp:
|
||||
self.config_suffix += f"_{pp_size}_{pp_rank}"
|
||||
shard_suffix = f"_{pp_size}_{pp_rank}" if enable_pp else ""
|
||||
# Under NSA context parallel each CP rank holds a disjoint slice of every
|
||||
# page, so give each rank its own file key to avoid a cross-rank write race.
|
||||
if attn_cp_size > 1:
|
||||
self.config_suffix += f"_cp{attn_cp_rank}_{attn_cp_size}"
|
||||
shard_suffix += f"_cp{attn_cp_rank}_{attn_cp_size}"
|
||||
# MLA KV is replicated across TP ranks and shares one file per page;
|
||||
# TP-sharded state is keyed by tp_config_suffix instead.
|
||||
self.tp_config_suffix = f"_{model_name}_{tp_rank}_{tp_size}{shard_suffix}"
|
||||
self.config_suffix = (
|
||||
f"_{model_name}{shard_suffix}" if is_mla_model else self.tp_config_suffix
|
||||
)
|
||||
|
||||
if not os.path.exists(self.file_path) and tp_rank == 0 and attn_cp_rank == 0:
|
||||
os.makedirs(self.file_path)
|
||||
@@ -450,11 +452,15 @@ class HiCacheFile(HiCacheStorage):
|
||||
# module, so a top-level import here would be circular.
|
||||
from sglang.srt.mem_cache.storage.file.lru_file_evictor import LRUFileEvictor
|
||||
|
||||
# Each rank creates and evicts its TP-sharded files; rank 0 also owns
|
||||
# the shared MLA files.
|
||||
owned_suffixes = (self.tp_config_suffix,)
|
||||
if is_mla_model and tp_rank == 0:
|
||||
owned_suffixes += (self.config_suffix,)
|
||||
self._evictor = LRUFileEvictor(
|
||||
self.file_path,
|
||||
self.config_suffix,
|
||||
owned_suffixes=owned_suffixes,
|
||||
tp_rank=tp_rank,
|
||||
is_mla_model=is_mla_model,
|
||||
extra_config=storage_config.extra_config,
|
||||
on_evict=(
|
||||
self.metadata_cache.remove if self.metadata_cache is not None else None
|
||||
@@ -462,6 +468,9 @@ class HiCacheFile(HiCacheStorage):
|
||||
)
|
||||
|
||||
def _get_suffixed_key(self, key: str) -> str:
|
||||
# Mamba state is TP-sharded even when the model's MLA KV is replicated.
|
||||
if key.endswith(f".{PoolName.MAMBA.value}"):
|
||||
return key + self.tp_config_suffix
|
||||
return key + self.config_suffix
|
||||
|
||||
def _get_component_key(self, key: str, component_name: Optional[str] = None) -> str:
|
||||
@@ -479,7 +488,7 @@ class HiCacheFile(HiCacheStorage):
|
||||
continue
|
||||
stem = fn[:-4]
|
||||
# Only files belonging to this rank/model.
|
||||
if stem.endswith(self.config_suffix):
|
||||
if stem.endswith((self.config_suffix, self.tp_config_suffix)):
|
||||
self.metadata_cache.add(stem)
|
||||
|
||||
def get(
|
||||
|
||||
@@ -66,22 +66,19 @@ class LRUFileEvictor:
|
||||
def __init__(
|
||||
self,
|
||||
file_path: str,
|
||||
config_suffix: str,
|
||||
*,
|
||||
owned_suffixes: Tuple[str, ...],
|
||||
tp_rank: int,
|
||||
is_mla_model: bool,
|
||||
extra_config: Optional[dict] = None,
|
||||
on_evict: Optional[Callable[[str], None]] = None,
|
||||
) -> None:
|
||||
self.file_path = file_path
|
||||
self.config_suffix = config_suffix
|
||||
# Files ending in one of these suffixes are created and evicted by this
|
||||
# rank; other files in the shared directory are only read.
|
||||
self._owned_suffixes = owned_suffixes
|
||||
self._tp_rank = tp_rank
|
||||
self._on_evict = on_evict
|
||||
|
||||
# MLA ranks share the same physical files, so centralize LRU bookkeeping
|
||||
# on rank 0; non-MLA ranks each own their own files via the suffix.
|
||||
self._is_storage_owner = (not is_mla_model) or (tp_rank == 0)
|
||||
|
||||
# suffixed_key -> file size in bytes; oldest at front.
|
||||
self._lru: OrderedDict[str, int] = OrderedDict()
|
||||
self._pending_writes: Set[str] = set()
|
||||
@@ -90,14 +87,7 @@ class LRUFileEvictor:
|
||||
|
||||
self._load_config(extra_config or {})
|
||||
|
||||
self._eviction_configured = self.max_size_bytes > 0 or self.min_free_bytes > 0
|
||||
self._eviction_enabled = self._eviction_configured and self._is_storage_owner
|
||||
if self._eviction_configured and not self._is_storage_owner:
|
||||
logger.info(
|
||||
f"HiCacheFile rank {self._tp_rank} (MLA): eviction handled by rank 0; "
|
||||
f"this rank skips LRU bookkeeping and will not create new files."
|
||||
)
|
||||
|
||||
self._eviction_enabled = self.max_size_bytes > 0 or self.min_free_bytes > 0
|
||||
if not self._eviction_enabled:
|
||||
return
|
||||
|
||||
@@ -149,18 +139,11 @@ class LRUFileEvictor:
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
"""True when this rank actively evicts (configured AND storage owner)."""
|
||||
"""True when a cap or free-space watermark is set."""
|
||||
return self._eviction_enabled
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
"""True when a cap or free-space watermark is set (on any rank)."""
|
||||
return self._eviction_configured
|
||||
|
||||
@property
|
||||
def is_storage_owner(self) -> bool:
|
||||
"""True when this rank owns (and may create/evict) the on-disk files."""
|
||||
return self._is_storage_owner
|
||||
def _owns(self, suffixed_key: str) -> bool:
|
||||
return suffixed_key.endswith(self._owned_suffixes)
|
||||
|
||||
def reserve(self, suffixed_key: str, value_bytes: int, key: str = "") -> bool:
|
||||
"""Admit a new write of ``value_bytes``, evicting LRU victims as needed.
|
||||
@@ -173,12 +156,12 @@ class LRUFileEvictor:
|
||||
the cap, there is no evictable space, or the free-space watermark cannot
|
||||
be met. When eviction is not configured the write is always admitted.
|
||||
"""
|
||||
if not self._eviction_configured:
|
||||
if not self._eviction_enabled:
|
||||
return True # unbounded storage: nothing to enforce
|
||||
if not self._is_storage_owner:
|
||||
if not self._owns(suffixed_key):
|
||||
logger.warning(
|
||||
f"HiCacheFile rank {self._tp_rank} is not the MLA storage owner; "
|
||||
f"not caching new key {key} because file eviction is enabled."
|
||||
f"HiCacheFile rank {self._tp_rank} does not own key {key}; "
|
||||
f"not caching it because file eviction is enabled."
|
||||
)
|
||||
return False
|
||||
if self.max_size_bytes > 0 and value_bytes > self.max_size_bytes:
|
||||
@@ -240,7 +223,7 @@ class LRUFileEvictor:
|
||||
|
||||
def touch(self, suffixed_key: str, tensor_path: str) -> None:
|
||||
"""Mark key as MRU, adopting an untracked on-disk file if needed."""
|
||||
if not self._eviction_enabled:
|
||||
if not self._eviction_enabled or not self._owns(suffixed_key):
|
||||
return
|
||||
with self._lock:
|
||||
if suffixed_key in self._lru:
|
||||
@@ -307,7 +290,7 @@ class LRUFileEvictor:
|
||||
continue
|
||||
stem = fn[:-4]
|
||||
# Only files belonging to this rank/model.
|
||||
if not stem.endswith(self.config_suffix):
|
||||
if not self._owns(stem):
|
||||
continue
|
||||
fp = os.path.join(self.file_path, fn)
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user