[MemCache] Fix LMCache component cursors and per-cache backend selection (#41328)
Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
This commit is contained in:
@@ -55,5 +55,6 @@ class CacheInitParams:
|
||||
component_registry_override: Optional[dict[ComponentType, type[TreeComponent]]] = (
|
||||
None
|
||||
)
|
||||
tree_core_backend: Optional[str] = dataclasses.field(default=None, kw_only=True)
|
||||
|
||||
mtp_draft_device_pools: tuple[object, ...] = ()
|
||||
|
||||
@@ -769,7 +769,10 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
external_tokens - self.lmcache_connector.aligned_swa_window_size(),
|
||||
0,
|
||||
)
|
||||
req.kv.swa_evicted_seqlen = max(req.kv.swa_evicted_seqlen, swa_missing_end)
|
||||
req.kv.set_evicted_seqlen(
|
||||
ComponentType.SWA,
|
||||
max(req.kv.get_evicted_seqlen(ComponentType.SWA), swa_missing_end),
|
||||
)
|
||||
|
||||
def _publish_external_loaded_prefix(self, req: Req, *, token_ids_len: int) -> None:
|
||||
"""Publish retrieved KV and immutable Mamba state into the device tree."""
|
||||
@@ -807,7 +810,7 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
value=kv_indices[:total_hit].to(dtype=torch.int64, copy=True),
|
||||
mamba_value=checkpoint,
|
||||
prev_prefix_len=prev_prefix_len,
|
||||
swa_evicted_seqlen=req.kv.swa_evicted_seqlen,
|
||||
component_evicted_seqlens=req.kv.component_evicted_seqlens.copy(),
|
||||
chunked=True,
|
||||
priority=req.priority or 0,
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ class ComponentType(int, Enum):
|
||||
SWA = 1
|
||||
MAMBA = 2
|
||||
C128 = 3
|
||||
AUXILIARY_SWA = 4
|
||||
|
||||
def __str__(self) -> str: # keep human-readable logging
|
||||
return self.name.lower()
|
||||
|
||||
@@ -204,7 +204,11 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
)
|
||||
# The TreeCore owns the tree member-var state (structure, LRUs, sizes,
|
||||
# evictable leaves) and drives the components' tree-level hooks.
|
||||
self._tree_core_backend = envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
|
||||
self._tree_core_backend = (
|
||||
params.tree_core_backend
|
||||
if params.tree_core_backend is not None
|
||||
else envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.get()
|
||||
)
|
||||
self.tree_core = create_tree_core(
|
||||
name=self._tree_core_backend,
|
||||
params=params,
|
||||
|
||||
Reference in New Issue
Block a user