[KVCache] Support lmcache unified radix cache (#38652)
Signed-off-by: chunxiaozheng <1179548172@qq.com> Co-authored-by: Yuwei An <ayw.sirius19@gmail.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Yuwei An
Claude Opus 5.5
parent
d3e3945e41
commit
cf5df82680
@@ -430,6 +430,49 @@ def handle_cache_compatibility(server_args: Any) -> None:
|
||||
if prefix_tails is not None and prefix_tails < 0:
|
||||
raise ValueError("--swa-prefix-tails should be a non-negative integer.")
|
||||
|
||||
if cfg.enable_lmcache:
|
||||
if cfg.enable_hierarchical_cache:
|
||||
raise ValueError(
|
||||
"--enable-lmcache and --enable-hierarchical-cache are "
|
||||
"mutually exclusive"
|
||||
)
|
||||
if cfg.enable_unified_cache_external_linker:
|
||||
raise ValueError(
|
||||
"--enable-lmcache and --enable-unified-cache-external-linker "
|
||||
"are mutually exclusive"
|
||||
)
|
||||
if cfg.disable_radix_cache:
|
||||
raise ValueError("--enable-lmcache requires radix cache to be enabled")
|
||||
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
if SpeculativeAlgorithm.from_string(cfg.speculative_algorithm).is_speculative():
|
||||
raise NotImplementedError(
|
||||
"LMCacheUnifiedRadixCache does not yet support speculative decoding"
|
||||
)
|
||||
if cfg.enable_dp_attention:
|
||||
raise NotImplementedError(
|
||||
"LMCacheUnifiedRadixCache does not yet support DP attention"
|
||||
)
|
||||
if cfg.dcp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"--enable-lmcache with --dcp-size > 1 is not supported: "
|
||||
"LMCache has no DCP-aware index translation"
|
||||
)
|
||||
if cfg.enable_streaming_session:
|
||||
raise NotImplementedError(
|
||||
"LMCacheUnifiedRadixCache does not yet support streaming sessions"
|
||||
)
|
||||
if cfg.hicache_host_memory_mode == "buffer_only":
|
||||
raise ValueError(
|
||||
"--hicache-host-memory-mode=buffer_only is a HiCache-only mode"
|
||||
)
|
||||
if cfg.disaggregation_mode != "null":
|
||||
raise NotImplementedError(
|
||||
"LMCacheUnifiedRadixCache currently supports colocated "
|
||||
"prefill/decode scheduling only"
|
||||
)
|
||||
|
||||
|
||||
def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
|
||||
|
||||
@@ -91,6 +91,11 @@ def handle_int8_mamba_checkpoint(server_args: Any):
|
||||
f"radix cache; --radix-cache-backend={cfg.radix_cache_backend!r} "
|
||||
"is not int8-aware. Omit --radix-cache-backend."
|
||||
)
|
||||
if cfg.enable_lmcache:
|
||||
raise ValueError(
|
||||
"--enable-int8-mamba-checkpoint is not supported together with "
|
||||
"--enable-lmcache: LMCache is not int8-aware. Disable one of them."
|
||||
)
|
||||
|
||||
|
||||
def validate_mamba_extra_buffer(view, model_arch: str, *, mamba_cache_chunk_size_of):
|
||||
|
||||
@@ -496,6 +496,7 @@ class Scheduler(
|
||||
)
|
||||
self.page_size = get_schedule().page_size
|
||||
self.enable_hierarchical_cache = get_memory().enable_hierarchical_cache
|
||||
self.enable_lmcache = get_memory().enable_lmcache
|
||||
self.enable_session_radix_cache = get_memory().enable_session_radix_cache
|
||||
self.enable_hicache_storage = get_memory().hicache_storage_backend is not None
|
||||
self.enable_unified_cache_external_linker = (
|
||||
@@ -2456,7 +2457,9 @@ class Scheduler(
|
||||
is_generation=self.is_generation,
|
||||
spec_algorithm=self.spec_algorithm,
|
||||
disaggregation_mode=self.disaggregation_mode,
|
||||
enable_hicache_storage=lambda: self.enable_hicache_storage,
|
||||
enable_hicache_storage=lambda: (
|
||||
self.enable_hicache_storage or self.enable_lmcache
|
||||
),
|
||||
rust_server=self.rust_server,
|
||||
)
|
||||
|
||||
@@ -3108,7 +3111,7 @@ class Scheduler(
|
||||
self.handle_generate_request(tokenized_req)
|
||||
|
||||
def _prefetch_kvcache(self, req: Req, storage_hit_end: Optional[int] = None):
|
||||
if self.enable_hicache_storage:
|
||||
if self.enable_hicache_storage or self.enable_lmcache:
|
||||
req.init_next_round_input(self.tree_cache, cow_mamba=False)
|
||||
tree_cache = self.tree_cache
|
||||
buffer_mode = get_memory().hicache_host_memory_mode == "buffer_only"
|
||||
@@ -3612,6 +3615,7 @@ class Scheduler(
|
||||
self.enable_hierarchical_cache
|
||||
or get_memory().enable_flexkv
|
||||
or self.enable_unified_cache_external_linker
|
||||
or self.enable_lmcache
|
||||
):
|
||||
self.tree_cache.check_hicache_events()
|
||||
if self.enable_hicache_storage:
|
||||
@@ -3966,7 +3970,7 @@ class Scheduler(
|
||||
):
|
||||
break
|
||||
|
||||
if self.enable_hicache_storage:
|
||||
if self.enable_hicache_storage or self.enable_lmcache:
|
||||
prefetch_done = self.tree_cache.check_prefetch_progress(
|
||||
req.cache_request_handle
|
||||
)
|
||||
@@ -4009,6 +4013,7 @@ class Scheduler(
|
||||
if res == AddReqResult.NO_TOKEN:
|
||||
if (
|
||||
self.enable_hierarchical_cache
|
||||
or self.enable_lmcache
|
||||
or self.enable_unified_cache_external_linker
|
||||
):
|
||||
# Set batch_is_full after making sure there are requests that can be served
|
||||
@@ -4879,12 +4884,12 @@ class Scheduler(
|
||||
return self.external_corpus_manager.list(recv_req)
|
||||
|
||||
def clear_hicache_storage_wrapped(self, recv_req: ClearHiCacheReqInput):
|
||||
if self.enable_hierarchical_cache:
|
||||
if self.enable_hierarchical_cache or self.enable_lmcache:
|
||||
self.tree_cache.clear_storage_backend()
|
||||
logger.info("Hierarchical cache cleared successfully!")
|
||||
logger.info("Hierarchical cache or LMCache cleared successfully!")
|
||||
if_success = True
|
||||
else:
|
||||
logging.warning("Hierarchical cache is not enabled.")
|
||||
logging.warning("Hierarchical cache or LMCache is not enabled.")
|
||||
if_success = False
|
||||
return ClearHiCacheReqOutput(success=if_success)
|
||||
|
||||
@@ -4914,6 +4919,7 @@ class Scheduler(
|
||||
if (
|
||||
self.enable_hicache_storage
|
||||
or self.disaggregation_mode != DisaggregationMode.NULL
|
||||
or self.enable_lmcache
|
||||
):
|
||||
# Storage and transfer workers need the GIL between I/O calls.
|
||||
# Singleton PD polls no longer yield through a collective.
|
||||
@@ -5044,6 +5050,8 @@ class Scheduler(
|
||||
# storage writes still hold host staging
|
||||
# (buffer-mode unified tree only).
|
||||
idle &= tc.buffer_pipeline.is_idle()
|
||||
elif self.enable_lmcache:
|
||||
idle &= not self.tree_cache.has_pending_cache_operations()
|
||||
|
||||
return idle
|
||||
|
||||
|
||||
@@ -79,6 +79,8 @@ class SchedulerOutputStreamer:
|
||||
storage_backend = cache_controller.storage_backend
|
||||
if storage_backend is not None:
|
||||
storage_backend_type = type(storage_backend).__name__
|
||||
elif self.server_args.enable_lmcache:
|
||||
storage_backend_type = "LMCache"
|
||||
return storage_backend_type
|
||||
|
||||
def get_cached_tokens_details(self, req: Req) -> Optional[CachedTokensDetails]:
|
||||
|
||||
@@ -100,6 +100,29 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
|
||||
return SWAChunkCache(params)
|
||||
|
||||
if get_memory().enable_lmcache:
|
||||
from sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache import (
|
||||
LMCacheUnifiedRadixCache,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components import ComponentType
|
||||
|
||||
tree_components = []
|
||||
if not (ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0):
|
||||
tree_components.append(ComponentType.FULL)
|
||||
if ctx.is_hybrid_swa:
|
||||
tree_components.append(ComponentType.SWA)
|
||||
if ctx.is_hybrid_ssm:
|
||||
tree_components.append(ComponentType.MAMBA)
|
||||
params.tree_components = tuple(tree_components)
|
||||
return LMCacheUnifiedRadixCache(
|
||||
params,
|
||||
model_config=ctx.model_config,
|
||||
tp_size=ctx.tp_size,
|
||||
tp_rank=ctx.tp_rank,
|
||||
lmcache_config_file=get_memory().lmcache_config_file,
|
||||
forward_stream=ctx.tp_worker.model_runner.forward_stream,
|
||||
)
|
||||
|
||||
if get_memory().enable_unified_cache_external_linker:
|
||||
return _create_unified_radix_cache(ctx, server_args, params)
|
||||
|
||||
@@ -108,19 +131,6 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
|
||||
return PureSWARadixCache(params=params)
|
||||
|
||||
if get_memory().enable_lmcache:
|
||||
from sglang.srt.mem_cache.storage.lmcache.lmc_radix_cache import (
|
||||
LMCRadixCache,
|
||||
)
|
||||
|
||||
return LMCRadixCache(
|
||||
params=params,
|
||||
model_config=ctx.model_config,
|
||||
tp_size=ctx.tp_size,
|
||||
rank=ctx.tp_rank,
|
||||
tp_group=ctx.tp_group,
|
||||
)
|
||||
|
||||
if get_memory().enable_flexkv:
|
||||
# Importing the package side-effect registers the explicit
|
||||
# ``--radix-cache-backend=flexkv`` factory; we then call the
|
||||
|
||||
@@ -2,9 +2,8 @@
|
||||
|
||||
A `RadixCache` subclass that routes sglang's host-tier KV cache through a
|
||||
FlexKV [`KVManager`](https://github.com/taco-project/FlexKV) (CPU / SSD /
|
||||
Remote offload). Same integration pattern as
|
||||
[`LMCRadixCache`](../lmcache/README.md): `FlexKVRadixCache` overrides
|
||||
`match_prefix` / `init_load_back` / `cache_finished_req` / `evict`; a
|
||||
Remote offload). `FlexKVRadixCache` overrides `match_prefix` /
|
||||
`init_load_back` / `cache_finished_req` / `evict`; a
|
||||
`FlexKVConnector` façade talks to `KVManager`, `KVTPClient`, and a
|
||||
3-axis (PP × CP × TP) sync context.
|
||||
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
This module exposes :class:`FlexKVRadixCache`, a subclass of
|
||||
:class:`sglang.srt.mem_cache.radix_cache.RadixCache` that delegates
|
||||
host-side prefix storage to a FlexKV ``KVManager``. The design mirrors
|
||||
``LMCRadixCache`` (the LMCache integration) so the scheduler-side
|
||||
contract is identical:
|
||||
the two-phase external-cache integration pattern, so the scheduler-side
|
||||
contract is:
|
||||
|
||||
* MP (synchronous) mode — the default.
|
||||
``match_prefix`` fires only a FlexKV LOOKUP and returns ``host_hit_length``;
|
||||
@@ -118,7 +118,7 @@ class FlexKVRadixCache(RadixCache):
|
||||
# forward layer blocks on its own eventfd.
|
||||
self.flexkv_connector.register_layer_transfer_counter(kvcache)
|
||||
|
||||
# CUDA streams (mirroring LMCRadixCache).
|
||||
# CUDA streams.
|
||||
self.load_stream = torch.cuda.Stream()
|
||||
self.store_stream = torch.cuda.Stream()
|
||||
|
||||
@@ -395,7 +395,7 @@ class FlexKVRadixCache(RadixCache):
|
||||
"""Base cache_finished_req then fire an async FlexKV store."""
|
||||
super().cache_finished_req(req, owned_kv_len=owned_kv_len)
|
||||
|
||||
# Compute the committed prefix mirroring LMCRadixCache's logic.
|
||||
# Compute the committed prefix.
|
||||
topk = get_spec().speculative_eagle_topk
|
||||
enable_kv_committed_len = topk is None or topk == 1
|
||||
if enable_kv_committed_len:
|
||||
|
||||
@@ -29,11 +29,9 @@ pip install -e . --no-build-isolation
|
||||
|
||||
## Use LMCache
|
||||
|
||||
LMCache supports two transport modes. **MP (multi-process, default)** issues a single blocking retrieve over ZMQ to a standalone daemon that owns the KV store and survives SGLang restarts. **IP (in-process)** uses an embedded layerwise connector — the cache lives and dies with the SGLang process. Mode selection is currently a code-level setting in `LMCRadixCache.__init__` (`self._mode`); only MP is reachable by default.
|
||||
|
||||
### MP mode (default): multi-process daemon
|
||||
|
||||
Uses `LMCacheMPConnector`. Daemon host/port come from the LMCache YAML config (`mp_host`, `mp_port`).
|
||||
SGLang uses `LMCacheUnifiedRadixCache` with LMCache's multiprocess connector.
|
||||
The standalone LMCache daemon owns the external cache and can survive SGLang
|
||||
process restarts. Daemon host and port come from the LMCache YAML config.
|
||||
|
||||
Terminal 1 — start the LMCache daemon:
|
||||
|
||||
@@ -56,16 +54,3 @@ python -m sglang.launch_server \
|
||||
```
|
||||
|
||||
For full LMCache config options see https://docs.lmcache.ai/api_reference/configurations.html.
|
||||
|
||||
### IP mode: in-process
|
||||
|
||||
Uses `LMCacheLayerwiseConnector`. KV transfer happens per layer inside the SGLang process; the cache lives and dies with the server. To enable, edit `LMCRadixCache.__init__` and set `self._mode = LMCacheMode.IP`.
|
||||
|
||||
The LMCache config still controls chunk_size and storage; `mp_host` / `mp_port` are ignored on this path. Use the bundled `example_config_ip.yaml`:
|
||||
|
||||
```bash
|
||||
python -m sglang.launch_server \
|
||||
--model-path MODEL \
|
||||
--enable-lmcache \
|
||||
--lmcache-config-file example_config_ip.yaml
|
||||
```
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""LMCache-backed KV cache integrations."""
|
||||
@@ -1,7 +0,0 @@
|
||||
# Basic configurations
|
||||
chunk_size: 256
|
||||
|
||||
# CPU offloading configurations
|
||||
local_cpu: true
|
||||
use_layerwise: true
|
||||
max_local_cpu_size: 10 # number of CPU backend GB
|
||||
@@ -1,519 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import logging
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
EvictParams,
|
||||
EvictResult,
|
||||
InitLoadBackParams,
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
|
||||
from sglang.srt.runtime_context import get_memory, get_spec
|
||||
from sglang.srt.utils import create_device_stream, device_stream_context
|
||||
|
||||
try:
|
||||
from lmcache.integration.sglang.multi_process_adapter import LMCacheMPConnector
|
||||
from lmcache.integration.sglang.sglang_adapter import (
|
||||
LMCacheLayerwiseConnector,
|
||||
LoadMetadata,
|
||||
StoreMetadata,
|
||||
)
|
||||
from lmcache.integration.sglang.utils import lmcache_get_config
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"LMCache is not installed. Please install it by running `pip install lmcache`"
|
||||
) from e
|
||||
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _LMCacheLoadBackMarker:
|
||||
"""Carries the data ``init_load_back`` needs from the
|
||||
``match_prefix`` call in MP mode.
|
||||
"""
|
||||
|
||||
key: RadixKey # detached snapshot of the matched key (the live query key
|
||||
# aliases the req's growing fill_ids and must not be retained)
|
||||
value_numel: int # number of tokens already in radix at match time
|
||||
|
||||
|
||||
class LMCacheMode(enum.Enum):
|
||||
MP = enum.auto() # multi-process mode
|
||||
IP = enum.auto() # in-process mode
|
||||
|
||||
|
||||
class LayerTransferCounter:
|
||||
"""Minimal adapter that lets the memory pool notify LMCache per-layer.
|
||||
|
||||
The KV pool calls `wait_until(layer_id)` after finishing a layer, which we
|
||||
translate into a `load_kv_layerwise(layer_id)` call on the LMCache connector
|
||||
within the provided device stream.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_layers: int,
|
||||
load_stream: torch.Stream,
|
||||
lmc_connector: LMCacheLayerwiseConnector,
|
||||
printable: bool = False,
|
||||
):
|
||||
self.num_layers = num_layers
|
||||
self.load_stream = load_stream
|
||||
self.lmc_connector = lmc_connector
|
||||
|
||||
def wait_until(self, layer_id: int):
|
||||
# Ensure ordering of the async loads wrt compute stream(s).
|
||||
self.load_stream.synchronize()
|
||||
with device_stream_context(self.load_stream):
|
||||
self.lmc_connector.load_kv_layerwise(layer_id)
|
||||
|
||||
|
||||
class LMCRadixCache(RadixCache):
|
||||
"""RadixCache + LMCache IO.
|
||||
|
||||
IP mode keeps the existing layerwise connector and
|
||||
its per-layer transfer hook: ``match_prefix`` kicks off the load via
|
||||
``start_load_kv`` and SGLang's per-layer KV-pool hook drives subsequent
|
||||
layers during forward.
|
||||
|
||||
MP mode uses ``LMCacheMPConnector`` with a two-phase
|
||||
load: ``match_prefix`` fires LOOKUP only (``connector.lookup_kv``) and
|
||||
returns ``host_hit_length`` on the ``MatchResult``; the SGLang
|
||||
scheduler then calls `init_load_back` at dispatch time,
|
||||
which fires the actual RETRIEVE (``connector.retrieve_kv``) into
|
||||
pre-allocated GPU slots.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
params: CacheInitParams,
|
||||
model_config: Optional[ModelConfig] = None,
|
||||
tp_size: int = 1,
|
||||
rank: int = 0,
|
||||
tp_group: Optional[torch.distributed.ProcessGroup] = None,
|
||||
):
|
||||
super().__init__(params)
|
||||
|
||||
cli_lmc_cfg = get_memory().lmcache_config_file or ""
|
||||
|
||||
kvcache = self.token_to_kv_pool_allocator.get_kvcache()
|
||||
connector_kwargs = dict(
|
||||
sgl_config=model_config,
|
||||
tp_size=tp_size,
|
||||
rank=rank,
|
||||
# NOTE: The original implementation accessed private buffers via
|
||||
# `_kvcache.k_buffer` / `.v_buffer`. We prefer public accessors when
|
||||
# available; fall back to private fields if needed.
|
||||
k_pool=getattr(
|
||||
kvcache,
|
||||
"k_buffer",
|
||||
getattr(self.token_to_kv_pool_allocator._kvcache, "k_buffer"),
|
||||
),
|
||||
v_pool=getattr(
|
||||
kvcache,
|
||||
"v_buffer",
|
||||
getattr(self.token_to_kv_pool_allocator._kvcache, "v_buffer"),
|
||||
),
|
||||
tp_group=tp_group.device_group if tp_group is not None else None,
|
||||
)
|
||||
|
||||
self.load_stream = create_device_stream(self.device)
|
||||
self.store_stream = create_device_stream(self.device)
|
||||
|
||||
# MP (multi-process) is the default. XPU defaults to IP (in-process
|
||||
# layerwise) because the MP connector shares the KV cache via CUDA IPC
|
||||
# (``Tensor._share_cuda_``), which is unavailable on XPU.
|
||||
self._mode = LMCacheMode.IP if self.device.type == "xpu" else LMCacheMode.MP
|
||||
if self._mode is LMCacheMode.MP:
|
||||
if not cli_lmc_cfg:
|
||||
raise ValueError(
|
||||
"MP mode requires --lmcache-config-file (the YAML "
|
||||
"supplies mp_host / mp_port)."
|
||||
)
|
||||
lm_cfg = lmcache_get_config(cli_lmc_cfg)
|
||||
self.lmcache_connector = LMCacheMPConnector(
|
||||
page_size=params.page_size,
|
||||
host=lm_cfg.mp_host,
|
||||
port=lm_cfg.mp_port,
|
||||
**connector_kwargs,
|
||||
)
|
||||
elif self._mode is LMCacheMode.IP:
|
||||
self.lmcache_connector = LMCacheLayerwiseConnector(
|
||||
config_file=cli_lmc_cfg, **connector_kwargs
|
||||
)
|
||||
# Per-layer hook
|
||||
self.layer_done_executor = LayerTransferCounter(
|
||||
num_layers=(
|
||||
model_config.num_hidden_layers if model_config is not None else 0
|
||||
),
|
||||
load_stream=self.load_stream,
|
||||
lmc_connector=self.lmcache_connector,
|
||||
)
|
||||
kvcache.register_layer_transfer_counter(self.layer_done_executor)
|
||||
|
||||
self._in_flight_nodes: list[TreeNode] = []
|
||||
self._node_lock = threading.Lock()
|
||||
self._mp_load_back_markers: dict[str, _LMCacheLoadBackMarker] = {}
|
||||
|
||||
def reset(self):
|
||||
super().reset()
|
||||
if hasattr(self, "_in_flight_nodes"):
|
||||
with self._node_lock:
|
||||
self._in_flight_nodes.clear()
|
||||
if hasattr(self, "_mp_load_back_markers"):
|
||||
self._mp_load_back_markers.clear()
|
||||
|
||||
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
|
||||
"""Dispatch to the mode-specific match_prefix.
|
||||
|
||||
MP mode → ``_mp_match_prefix`` (fires LOOKUP only).
|
||||
IP mode → ``_ip_match_prefix`` (single-shot ``start_load_kv``
|
||||
plus per-layer hook).
|
||||
"""
|
||||
key = params.key
|
||||
if self.disable or not key:
|
||||
return super().match_prefix(params)
|
||||
|
||||
if self.page_size != 1:
|
||||
aligned_len = len(key) // self.page_size * self.page_size
|
||||
key = key[:aligned_len]
|
||||
|
||||
base_res = super().match_prefix(params)
|
||||
value: torch.Tensor = base_res.device_indices
|
||||
last_node: TreeNode = base_res.last_device_node
|
||||
|
||||
if self._mode is LMCacheMode.MP:
|
||||
if params.req is None:
|
||||
return base_res
|
||||
return self._mp_match_prefix(key, base_res, value, last_node, params.req)
|
||||
elif self._mode is LMCacheMode.IP:
|
||||
return self._ip_match_prefix(key, base_res, value, last_node)
|
||||
return base_res
|
||||
|
||||
def _mp_match_prefix(
|
||||
self,
|
||||
key: RadixKey,
|
||||
base_res: MatchResult,
|
||||
value: torch.Tensor,
|
||||
last_node: TreeNode,
|
||||
req: Req,
|
||||
) -> MatchResult:
|
||||
"""MP LOOKUP
|
||||
|
||||
Returns a ``MatchResult`` with ``host_hit_length`` set when
|
||||
LMCache has tokens beyond radix. Otherwise releases
|
||||
the held read locks and returns the radix-only result.
|
||||
"""
|
||||
token_ids = key.raw_token_ids()
|
||||
matched = self.lmcache_connector.lookup_kv(token_ids, req.rid)
|
||||
if matched <= value.numel():
|
||||
# Release the read locks; keep the pending session for end_session.
|
||||
self.lmcache_connector.release_pending(req.rid)
|
||||
return base_res
|
||||
|
||||
if token_ids is key.token_ids:
|
||||
token_ids = token_ids[:]
|
||||
self._mp_load_back_markers[req.rid] = _LMCacheLoadBackMarker(
|
||||
key=RadixKey(
|
||||
token_ids,
|
||||
key.extra_key,
|
||||
key.is_bigram,
|
||||
cache_salt=key.cache_salt,
|
||||
),
|
||||
value_numel=int(value.numel()),
|
||||
)
|
||||
return MatchResult(
|
||||
device_indices=value,
|
||||
last_device_node=last_node,
|
||||
last_host_node=last_node,
|
||||
best_match_node=last_node,
|
||||
host_hit_length=matched - int(value.numel()),
|
||||
)
|
||||
|
||||
def _ip_match_prefix(
|
||||
self,
|
||||
key: RadixKey,
|
||||
base_res: MatchResult,
|
||||
value: torch.Tensor,
|
||||
last_node: TreeNode,
|
||||
) -> MatchResult:
|
||||
"""IP mode: ``start_load_kv`` + per-layer hook.
|
||||
|
||||
Allocates slots for the page-aligned uncached tail and kicks off
|
||||
the layerwise load. Returns ``base_res`` if there's nothing to
|
||||
fetch or alloc/load fails.
|
||||
"""
|
||||
if value.numel() == len(key):
|
||||
return base_res
|
||||
|
||||
uncached_len = len(key) - value.numel()
|
||||
if uncached_len == 0:
|
||||
return base_res
|
||||
|
||||
token_ids = key.raw_token_ids()
|
||||
result = self._load_back(
|
||||
key=key,
|
||||
value_numel=int(value.numel()),
|
||||
uncached_len=uncached_len,
|
||||
last_node=last_node,
|
||||
load_fn=lambda sm, pp: self._ip_load_back(
|
||||
token_ids=token_ids,
|
||||
value_numel=int(value.numel()),
|
||||
slot_mapping=sm,
|
||||
prefix_pad=pp,
|
||||
),
|
||||
)
|
||||
if result is None:
|
||||
return base_res
|
||||
new_slots, new_node = result
|
||||
return MatchResult(
|
||||
device_indices=torch.cat([value, new_slots]),
|
||||
last_device_node=new_node,
|
||||
last_host_node=new_node,
|
||||
best_match_node=new_node,
|
||||
)
|
||||
|
||||
def init_load_back(
|
||||
self, params: InitLoadBackParams
|
||||
) -> Tuple[torch.Tensor, Optional[TreeNode]]:
|
||||
"""MP RETRIEVE.
|
||||
|
||||
Called by the scheduler when ``match_prefix`` returned
|
||||
``host_hit_length > 0``. Uses the cached LOOKUP result to
|
||||
allocate slots and fire RETRIEVE, inserts the resulting
|
||||
TreeNode into the radix tree, and returns
|
||||
``(new_indices, new_last_node)``.
|
||||
"""
|
||||
req = params.req
|
||||
marker = self._mp_load_back_markers.pop(req.rid)
|
||||
last_node: TreeNode = params.best_match_node
|
||||
|
||||
result = self._load_back(
|
||||
key=marker.key,
|
||||
value_numel=marker.value_numel,
|
||||
uncached_len=params.host_hit_length,
|
||||
last_node=last_node,
|
||||
load_fn=lambda sm, pp: self._mp_load_back(
|
||||
marker=marker,
|
||||
request_id=req.rid,
|
||||
slot_mapping=sm,
|
||||
prefix_pad=pp,
|
||||
),
|
||||
)
|
||||
if result is None:
|
||||
# Either alloc failed (locks still held by lookup_kv) or
|
||||
# retrieve returned nothing (locks already released by
|
||||
# retrieve_kv). release_pending is idempotent on locks_held.
|
||||
self.lmcache_connector.release_pending(req.rid)
|
||||
return (
|
||||
torch.empty((0,), dtype=torch.int64, device=self.device),
|
||||
last_node,
|
||||
)
|
||||
return result
|
||||
|
||||
def _load_back(
|
||||
self,
|
||||
*,
|
||||
key: RadixKey,
|
||||
value_numel: int,
|
||||
uncached_len: int,
|
||||
last_node: TreeNode,
|
||||
load_fn, # Callable[[torch.Tensor, int], int] — (slot_mapping, prefix_pad) -> num_retrieved
|
||||
) -> Optional[Tuple[torch.Tensor, TreeNode]]:
|
||||
"""Alloc slots, run ``load_fn``, attach a TreeNode for what was loaded.
|
||||
|
||||
Returns ``(slots, new_node)`` on success, ``None`` if alloc fails
|
||||
or the load returned zero (slots are freed in either case).
|
||||
"""
|
||||
chunk_size = self.lmcache_connector.chunk_size()
|
||||
prefix_pad = value_numel % chunk_size
|
||||
|
||||
if self.token_to_kv_pool_allocator.available_size() < uncached_len:
|
||||
self.evict(EvictParams(num_tokens=uncached_len))
|
||||
|
||||
token_slots = self.token_to_kv_pool_allocator.alloc(uncached_len)
|
||||
if token_slots is None:
|
||||
return None
|
||||
|
||||
slot_mapping = torch.empty(
|
||||
value_numel + token_slots.numel(),
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
slot_mapping[:value_numel].fill_(-1)
|
||||
slot_mapping[value_numel:].copy_(token_slots)
|
||||
|
||||
# Dispatch to the mode-specific loader (IP: start_load_kv, MP:
|
||||
# retrieve_kv). Each loader manages its own load_stream context.
|
||||
num_retrieved = load_fn(slot_mapping, prefix_pad)
|
||||
logger.debug("num_retrieved_tokens: %s", num_retrieved)
|
||||
|
||||
if num_retrieved > 0:
|
||||
self.token_to_kv_pool_allocator.free(
|
||||
token_slots[(num_retrieved - prefix_pad) :]
|
||||
)
|
||||
else:
|
||||
self.token_to_kv_pool_allocator.free(token_slots)
|
||||
|
||||
if num_retrieved > 0:
|
||||
fetched = num_retrieved - prefix_pad
|
||||
new_node = TreeNode(priority=last_node.priority)
|
||||
start = value_numel
|
||||
end = start + fetched
|
||||
new_node.key = key[start:end]
|
||||
new_node.value = token_slots[:fetched]
|
||||
new_node.parent = last_node
|
||||
last_node.children[new_node.key.child_key(self.page_size)] = new_node
|
||||
self.evictable_size_ += fetched
|
||||
self._update_leaf_status(last_node)
|
||||
self._update_leaf_status(new_node)
|
||||
|
||||
self.kv_events.record_store(new_node.parent)
|
||||
self.kv_events.record_store(new_node)
|
||||
|
||||
return token_slots[:fetched], new_node
|
||||
|
||||
return None
|
||||
|
||||
def _mp_load_back(
|
||||
self,
|
||||
*,
|
||||
marker: _LMCacheLoadBackMarker,
|
||||
request_id: str,
|
||||
slot_mapping: torch.Tensor,
|
||||
prefix_pad: int,
|
||||
) -> int:
|
||||
"""MP non-layerwise loader: fire ``retrieve_kv`` and wait for the
|
||||
load_stream so the compute stream observes the writes.
|
||||
"""
|
||||
current_stream = torch.get_device_module(self.device).current_stream()
|
||||
self.load_stream.wait_stream(current_stream)
|
||||
with device_stream_context(self.load_stream):
|
||||
n = self.lmcache_connector.retrieve_kv(
|
||||
LoadMetadata(
|
||||
token_ids=marker.key.token_ids,
|
||||
slot_mapping=slot_mapping,
|
||||
offset=marker.value_numel - prefix_pad,
|
||||
prefix_pad=prefix_pad,
|
||||
request_id=request_id,
|
||||
)
|
||||
)
|
||||
current_stream.wait_stream(self.load_stream)
|
||||
return n
|
||||
|
||||
def _ip_load_back(
|
||||
self,
|
||||
*,
|
||||
token_ids: list[int],
|
||||
value_numel: int,
|
||||
slot_mapping: torch.Tensor,
|
||||
prefix_pad: int,
|
||||
) -> int:
|
||||
"""IP layerwise loader: kick off ``start_load_kv`` on ``self.load_stream``.
|
||||
|
||||
``start_load_kv`` enqueues the first layer's transfer; the
|
||||
``LayerTransferCounter`` hook drives the rest during forward.
|
||||
"""
|
||||
with device_stream_context(self.load_stream):
|
||||
return self.lmcache_connector.start_load_kv(
|
||||
LoadMetadata(
|
||||
token_ids=token_ids,
|
||||
slot_mapping=slot_mapping,
|
||||
offset=value_numel - prefix_pad,
|
||||
)
|
||||
)
|
||||
|
||||
def on_release(self, req: Req, *, inserted: bool) -> None:
|
||||
if not inserted and self._mode is LMCacheMode.MP:
|
||||
self._mp_load_back_markers.pop(req.rid, None)
|
||||
self.lmcache_connector.end_session(req.rid)
|
||||
|
||||
def cache_finished_req(self, req: Req, *, owned_kv_len: int) -> None:
|
||||
"""On request completion, insert device KV into radix and store to LMCache."""
|
||||
|
||||
super().cache_finished_req(req, owned_kv_len=owned_kv_len)
|
||||
|
||||
topk = get_spec().speculative_eagle_topk
|
||||
enable_kv_committed_len = topk is None or topk == 1
|
||||
if enable_kv_committed_len:
|
||||
kv_committed_len = req.kv.kv_committed_len
|
||||
else:
|
||||
kv_committed_len = len(req.origin_input_ids) + max(
|
||||
len(req.output_ids) - 1, 0
|
||||
)
|
||||
|
||||
token_ids = (req.origin_input_ids + req.output_ids)[:kv_committed_len]
|
||||
kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, :kv_committed_len
|
||||
]
|
||||
|
||||
# Use super() to avoid a redundant LOOKUP — we only need new_last_node from radix.
|
||||
match_result = super().match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids,
|
||||
req.extra_key,
|
||||
cache_salt=req.cache_salt,
|
||||
)
|
||||
)
|
||||
)
|
||||
new_last_node = match_result.last_device_node
|
||||
assert new_last_node is not None
|
||||
|
||||
self.inc_lock_ref(new_last_node)
|
||||
store_md = StoreMetadata(
|
||||
last_node=new_last_node,
|
||||
token_ids=token_ids,
|
||||
kv_indices=kv_indices,
|
||||
offset=0,
|
||||
request_id=req.rid,
|
||||
)
|
||||
if self._mode is LMCacheMode.MP:
|
||||
self.lmcache_connector.store_kv(store_md)
|
||||
# MP store_kv blocks until the daemon's signal event fires, so the slots are safe to evict immediately.
|
||||
self._mp_load_back_markers.pop(req.rid, None)
|
||||
self.dec_lock_ref(new_last_node)
|
||||
self.lmcache_connector.end_session(req.rid)
|
||||
elif self._mode is LMCacheMode.IP:
|
||||
with device_stream_context(self.store_stream):
|
||||
self.lmcache_connector.store_kv(store_md)
|
||||
# Layerwise store is async on store_stream; defer the unlock to evict()'s store_stream.synchronize().
|
||||
with self._node_lock:
|
||||
self._in_flight_nodes.append(new_last_node)
|
||||
|
||||
def evict(self, params: EvictParams) -> EvictResult:
|
||||
"""Before base eviction, wait for any outstanding stores and release locks."""
|
||||
if self.disable:
|
||||
return EvictResult()
|
||||
|
||||
self.store_stream.synchronize()
|
||||
with self._node_lock:
|
||||
for node in self._in_flight_nodes:
|
||||
self.dec_lock_ref(node)
|
||||
self._in_flight_nodes.clear()
|
||||
|
||||
return super().evict(params)
|
||||
|
||||
def pretty_print(self):
|
||||
super().pretty_print()
|
||||
try:
|
||||
logger.debug(
|
||||
"evictable=%d protected=%d", self.evictable_size_, self.protected_size_
|
||||
)
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
@@ -0,0 +1,982 @@
|
||||
"""Unified radix tree backed by the LMCache multiprocess service.
|
||||
|
||||
LMCache owns external storage; retrieved KV is published after reaching GPU.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
from array import array
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
import torch
|
||||
from lmcache.integration.sglang.lmcache_mp_metadata import (
|
||||
LMCacheExternalFlow,
|
||||
LMCachePendingStore,
|
||||
)
|
||||
from lmcache.integration.sglang.unified_lmcache_mp_connector import (
|
||||
UnifiedLMCacheMPConnector,
|
||||
)
|
||||
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWAAllocatorBase,
|
||||
)
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
CacheRequestHandle,
|
||||
EvictParams,
|
||||
InitLoadBackParams,
|
||||
InsertParams,
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.components import ComponentType
|
||||
from sglang.srt.mem_cache.unified_radix_cache import NodeId, UnifiedRadixCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
"""Unified radix tree with device-direct LMCache MP KV/state I/O."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
params: CacheInitParams,
|
||||
*,
|
||||
model_config: ModelConfig,
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
lmcache_config_file: Optional[str],
|
||||
forward_stream: Any,
|
||||
) -> None:
|
||||
# The parent constructor calls reset() before the connector exists.
|
||||
self.lmcache_connector: Optional[UnifiedLMCacheMPConnector] = None
|
||||
super().__init__(params)
|
||||
self._mamba_component = self._find_mamba_component()
|
||||
self.lmcache_connector = UnifiedLMCacheMPConnector(
|
||||
config_file=lmcache_config_file,
|
||||
model_config=model_config,
|
||||
tp_size=tp_size,
|
||||
tp_rank=tp_rank,
|
||||
tp_group=params.tp_cache_group,
|
||||
pp_size=params.pp_size,
|
||||
pp_rank=params.pp_rank,
|
||||
pp_group=params.pp_cache_group,
|
||||
page_size=self.page_size,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
tree_components=self.tree_components,
|
||||
mamba_component=self._mamba_component,
|
||||
sliding_window_size=self._sliding_window_size,
|
||||
)
|
||||
self._external_flows: dict[str, LMCacheExternalFlow] = {}
|
||||
self._forward_stream = forward_stream
|
||||
self._pending_stores: list[LMCachePendingStore] = []
|
||||
self._pending_store_counts: dict[str, int] = {}
|
||||
self._session_finish_requested: set[str] = set()
|
||||
self._lmcache_closed = False
|
||||
atexit.register(self.shutdown)
|
||||
|
||||
def is_backuped(self, node_id: NodeId) -> bool:
|
||||
# LMCache rebuilds lookup keys from tokens, so any L1 node is valid.
|
||||
return True
|
||||
|
||||
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
|
||||
"""Report LMCache hits through host fields until GPU slots are ready."""
|
||||
requested_key_len = len(params.key)
|
||||
req = params.req
|
||||
flow = self._external_flows.get(req.rid) if req is not None else None
|
||||
if (
|
||||
flow is not None
|
||||
and flow.total_hit is not None
|
||||
and flow.local_hit_tokens is not None
|
||||
and flow.load is None
|
||||
):
|
||||
# Use the shortest cross-rank L1 prefix for consistent admission.
|
||||
params = MatchPrefixParams(
|
||||
key=params.key[: flow.local_hit_tokens],
|
||||
cow_mamba=params.cow_mamba,
|
||||
req=req,
|
||||
)
|
||||
elif flow is not None and flow.load is not None and params.cow_mamba:
|
||||
# Re-arm Mamba COW from the retrieved checkpoint on retry.
|
||||
self._arm_external_mamba_cow(flow, req)
|
||||
params = MatchPrefixParams(key=params.key, cow_mamba=False, req=req)
|
||||
result = super().match_prefix(params)
|
||||
if req is None:
|
||||
return result
|
||||
if flow is None or flow.total_hit is None or flow.cancelled:
|
||||
return result
|
||||
|
||||
total_hit = min(flow.total_hit, requested_key_len, len(flow.key))
|
||||
local_hit = len(result.device_indices)
|
||||
skip = 0
|
||||
if flow.load is not None and not flow.prefix_published:
|
||||
# Track private slots shadowed by a concurrent tree insertion.
|
||||
skip = max(
|
||||
min(local_hit, total_hit) - flow.load.local_hit_tokens,
|
||||
0,
|
||||
)
|
||||
flow.loaded_skip_tokens = max(flow.loaded_skip_tokens, skip)
|
||||
if flow.load.result:
|
||||
self._release_unused_loaded_slots(flow)
|
||||
if total_hit <= local_hit:
|
||||
return result
|
||||
|
||||
if flow.load is None:
|
||||
external_hit = total_hit - local_hit
|
||||
return result._replace(
|
||||
last_host_node=result.last_device_node,
|
||||
best_match_node=result.last_device_node,
|
||||
host_hit_length=external_hit,
|
||||
swa_host_hit_length=(
|
||||
min(
|
||||
external_hit,
|
||||
self.lmcache_connector.aligned_swa_window_size(),
|
||||
)
|
||||
if self.is_swa_enabled
|
||||
else 0
|
||||
),
|
||||
mamba_host_hit_length=(1 if self._mamba_component is not None else 0),
|
||||
mamba_branching_seqlen=None,
|
||||
full_kv_hit_length=max(result.full_kv_hit_length, total_hit),
|
||||
)
|
||||
|
||||
# Reuse an already submitted H2D load on the next admission attempt.
|
||||
suffix = flow.load.device_indices[skip : skip + max(total_hit - local_hit, 0)]
|
||||
return result._replace(
|
||||
device_indices=torch.cat([result.device_indices, suffix]),
|
||||
last_host_node=result.last_device_node,
|
||||
best_match_node=result.last_device_node,
|
||||
host_hit_length=0,
|
||||
swa_host_hit_length=0,
|
||||
mamba_host_hit_length=0,
|
||||
mamba_branching_seqlen=None,
|
||||
full_kv_hit_length=max(result.full_kv_hit_length, total_hit),
|
||||
)
|
||||
|
||||
def prefetch_from_storage(
|
||||
self,
|
||||
request: CacheRequestHandle,
|
||||
last_host_node_id: NodeId,
|
||||
new_input_tokens: list[int],
|
||||
last_hash: Optional[str] = None,
|
||||
prefix_keys: Optional[list[str]] = None,
|
||||
matched_prefix_tokens: Optional[list[int]] = None,
|
||||
extra_key: Optional[str] = None,
|
||||
cache_salt: Optional[str] = None,
|
||||
storage_hit_end: Optional[int] = None,
|
||||
) -> None:
|
||||
del last_hash, prefix_keys, storage_hit_end
|
||||
req_id = request.rid
|
||||
if req_id in self._external_flows:
|
||||
return
|
||||
local_tokens = list(matched_prefix_tokens or [])
|
||||
token_ids = local_tokens + list(new_input_tokens)
|
||||
anchor_extra_key, anchor_cache_salt = self.tree_core.prefetch_anchor_info(
|
||||
last_host_node_id
|
||||
)
|
||||
extra_key = extra_key or anchor_extra_key
|
||||
cache_salt = cache_salt or anchor_cache_salt
|
||||
key = RadixKey(
|
||||
array("q", token_ids),
|
||||
extra_key=extra_key,
|
||||
is_bigram=self.tree_core.is_eagle,
|
||||
cache_salt=cache_salt,
|
||||
).page_aligned(self.page_size)
|
||||
if len(key) == 0:
|
||||
return
|
||||
token_ids = key.raw_token_ids()[: len(key)]
|
||||
lookup = self.lmcache_connector.submit_lookup(
|
||||
req_id,
|
||||
token_ids,
|
||||
local_hit_tokens=min(len(local_tokens), len(key)),
|
||||
cache_salt=self.lmcache_connector.build_cache_salt(cache_salt, extra_key),
|
||||
)
|
||||
self._external_flows[req_id] = LMCacheExternalFlow(
|
||||
key=key,
|
||||
lookup=lookup,
|
||||
)
|
||||
|
||||
def check_prefetch_progress(self, handle: CacheRequestHandle) -> bool:
|
||||
req_id = handle.rid
|
||||
flow = self._external_flows.get(req_id)
|
||||
if flow is None:
|
||||
return True
|
||||
if flow.cancelled:
|
||||
return False
|
||||
|
||||
if flow.total_hit is None:
|
||||
total_hit = self.lmcache_connector.poll_lookup(flow.lookup)
|
||||
if total_hit is None:
|
||||
return False
|
||||
latest = super().match_prefix(MatchPrefixParams(key=flow.key))
|
||||
local_hit = torch.tensor(
|
||||
[len(latest.device_indices)], dtype=torch.int64, device="cpu"
|
||||
)
|
||||
self.lmcache_connector.parallel_all_reduce(
|
||||
local_hit, torch.distributed.ReduceOp.MIN
|
||||
)
|
||||
total_hit = min(total_hit, len(flow.key))
|
||||
local_hit_tokens = int(local_hit.item())
|
||||
release_end = min(
|
||||
total_hit,
|
||||
local_hit_tokens
|
||||
// self.lmcache_connector.chunk_size
|
||||
* self.lmcache_connector.chunk_size,
|
||||
)
|
||||
if release_end > 0:
|
||||
# All ranks advance state; only the leader releases remote locks.
|
||||
self.lmcache_connector.free_lookup_locks(
|
||||
req_id, start=0, end=release_end
|
||||
)
|
||||
if total_hit <= local_hit_tokens:
|
||||
self._external_flows.pop(req_id, None)
|
||||
self.prefetch_loaded_tokens_by_reqid[handle] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(handle, None)
|
||||
return True
|
||||
flow.total_hit = total_hit
|
||||
flow.local_hit_tokens = local_hit_tokens
|
||||
self.prefetch_loaded_tokens_by_reqid[handle] = total_hit - local_hit_tokens
|
||||
self.prefetch_loaded_storage_start_by_reqid[handle] = local_hit_tokens
|
||||
|
||||
return True
|
||||
|
||||
def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None:
|
||||
self._publish_external_loaded_prefix(req, token_ids_len=len(req.get_fill_ids()))
|
||||
super().cache_unfinished_req(req, chunked=chunked, **kwargs)
|
||||
self._retire_loaded_flow(req.rid)
|
||||
self._submit_store(req, req.get_fill_ids())
|
||||
|
||||
def on_release(self, req: Req, *, inserted: bool) -> None:
|
||||
super().on_release(req, inserted=inserted)
|
||||
if not inserted:
|
||||
self.release_aborted_request(req.cache_request_handle)
|
||||
|
||||
def cache_finished_req(self, req: Req, *, owned_kv_len: int, **kwargs) -> None:
|
||||
self._publish_external_loaded_prefix(req, token_ids_len=owned_kv_len)
|
||||
super().cache_finished_req(req, owned_kv_len=owned_kv_len, **kwargs)
|
||||
self._retire_loaded_flow(req.rid)
|
||||
token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len]
|
||||
self._submit_store(req, token_ids)
|
||||
self._request_session_finish(req.rid)
|
||||
|
||||
def check_hicache_events(self) -> None:
|
||||
"""Poll LMCache retrieve/store futures at the scheduler safe point."""
|
||||
load_flows = [
|
||||
flow
|
||||
for flow in self._external_flows.values()
|
||||
if flow.load is not None and not flow.load_completed
|
||||
]
|
||||
ready_loads = self.lmcache_connector.ready_prefix_count(
|
||||
[f.load for f in load_flows]
|
||||
)
|
||||
for flow in load_flows[:ready_loads]:
|
||||
assert flow.load is not None
|
||||
success = self.lmcache_connector.complete_load(flow.load)
|
||||
flow.load_completed = True
|
||||
cancelled = flow.cancelled
|
||||
if cancelled or not success:
|
||||
self._finish_failed_load(flow)
|
||||
if not cancelled:
|
||||
logger.warning(
|
||||
"LMCache retrieve failed after admission for request %s",
|
||||
flow.lookup.request_id,
|
||||
)
|
||||
elif flow.retire_requested:
|
||||
self._finalize_retired_flow(flow)
|
||||
else:
|
||||
self._finish_successful_load(flow)
|
||||
|
||||
ready_stores = self.lmcache_connector.ready_prefix_count(
|
||||
[pending.operation for pending in self._pending_stores]
|
||||
)
|
||||
for _ in range(ready_stores):
|
||||
pending = self._pending_stores.pop(0)
|
||||
rid = pending.operation.request_id
|
||||
try:
|
||||
self.lmcache_connector.complete_store(pending.operation)
|
||||
finally:
|
||||
# Always unlock the source node after store completion.
|
||||
self.dec_lock_ref(pending.node_id, pending.lock_params)
|
||||
remaining = self._pending_store_counts[rid] - 1
|
||||
if remaining > 0:
|
||||
self._pending_store_counts[rid] = remaining
|
||||
else:
|
||||
self._pending_store_counts.pop(rid)
|
||||
self._finish_session_if_store_idle(rid)
|
||||
|
||||
def has_pending_cache_operations(self) -> bool:
|
||||
return bool(self._external_flows or self._pending_stores)
|
||||
|
||||
def release_aborted_request(self, handle: CacheRequestHandle) -> None:
|
||||
rid = handle.rid
|
||||
self.prefetch_loaded_tokens_by_reqid.pop(handle, None)
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(handle, None)
|
||||
flow = self._external_flows.get(rid)
|
||||
if flow is None:
|
||||
self._request_session_finish(rid)
|
||||
return
|
||||
flow.cancelled = True
|
||||
# Keep the checkpoint alive until H2D completes.
|
||||
if flow.load is not None and flow.mamba_value is not None:
|
||||
flow.free_mamba_after_load = True
|
||||
if (
|
||||
flow.load_req is not None
|
||||
and flow.load_req.kv.mamba_cow_src_index is flow.mamba_value
|
||||
):
|
||||
flow.load_req.kv.mamba_cow_src_index = None
|
||||
if flow.load is None:
|
||||
if flow.total_hit is not None:
|
||||
# No retrieve will release locks from this completed lookup.
|
||||
self._retire_loaded_flow(rid)
|
||||
else:
|
||||
# END_SESSION follows the in-flight lookup and releases its locks.
|
||||
self._external_flows.pop(rid, None)
|
||||
elif flow.load_completed:
|
||||
self._finish_failed_load(flow)
|
||||
else:
|
||||
# Complete the cross-rank load only in check_hicache_events().
|
||||
flow.retire_requested = True
|
||||
self._request_session_finish(rid)
|
||||
|
||||
def init_load_back(self, params: InitLoadBackParams) -> tuple[torch.Tensor, NodeId]:
|
||||
req = params.req
|
||||
if req is None:
|
||||
return (
|
||||
self.tree_core.empty_match_result.device_indices,
|
||||
params.best_match_node,
|
||||
)
|
||||
flow = self._external_flows.get(req.rid)
|
||||
if flow is None or flow.total_hit is None or flow.load is not None:
|
||||
return (
|
||||
self.tree_core.empty_match_result.device_indices,
|
||||
params.best_match_node,
|
||||
)
|
||||
|
||||
device_indices = self._start_external_load(flow, req)
|
||||
if device_indices is None:
|
||||
req.storage_hit_length = 0
|
||||
req.host_hit_length = 0
|
||||
req.swa_host_hit_length = 0
|
||||
req.mamba_host_hit_length = 0
|
||||
self.prefetch_loaded_tokens_by_reqid.pop(req.cache_request_handle, None)
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(
|
||||
req.cache_request_handle, None
|
||||
)
|
||||
# Retire the unloaded flow to release its lookup locks.
|
||||
self._retire_loaded_flow(req.rid)
|
||||
return (
|
||||
self.tree_core.empty_match_result.device_indices,
|
||||
params.best_match_node,
|
||||
)
|
||||
return device_indices, params.best_match_node
|
||||
|
||||
def ready_to_load_host_cache(self) -> int:
|
||||
# H2D is submitted in init_load_back to preserve prefill fallback.
|
||||
return -1
|
||||
|
||||
def supports_retraction_backup(self) -> bool:
|
||||
# TODO(chunxiaozheng): implement retraction backup
|
||||
return False
|
||||
|
||||
def clear_storage_backend(self) -> bool:
|
||||
return self.lmcache_connector.clear()
|
||||
|
||||
def reset(self) -> None:
|
||||
connector = self.lmcache_connector
|
||||
if connector is not None:
|
||||
for flow in list(self._external_flows.values()):
|
||||
if flow.load is not None:
|
||||
try:
|
||||
flow.load.future.result(timeout=connector.operation_timeout)
|
||||
except Exception:
|
||||
pass
|
||||
connector.complete_load(flow.load, synchronize=False)
|
||||
flow.load_completed = True
|
||||
if flow.prefix_published:
|
||||
# Free only private slots; the tree owns the published suffix.
|
||||
self._release_unused_loaded_slots(flow)
|
||||
else:
|
||||
self.token_to_kv_pool_allocator.free(
|
||||
flow.load.device_indices[flow.released_skip_tokens :]
|
||||
)
|
||||
if flow.mamba_value is not None:
|
||||
self.req_to_token_pool.mamba_allocator.free(flow.mamba_value)
|
||||
flow.mamba_value = None
|
||||
self._release_flow_anchor(flow)
|
||||
for pending in list(self._pending_stores):
|
||||
try:
|
||||
pending.operation.future.result(timeout=connector.operation_timeout)
|
||||
except Exception:
|
||||
pass
|
||||
connector.complete_store(pending.operation, synchronize=False)
|
||||
self.dec_lock_ref(pending.node_id, pending.lock_params)
|
||||
self._external_flows.clear()
|
||||
self._pending_stores.clear()
|
||||
self._pending_store_counts.clear()
|
||||
self._session_finish_requested.clear()
|
||||
self.prefetch_loaded_tokens_by_reqid.clear()
|
||||
connector.end_all_sessions()
|
||||
super().reset()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self._lmcache_closed:
|
||||
return
|
||||
self.reset()
|
||||
self.lmcache_connector.close()
|
||||
self._lmcache_closed = True
|
||||
|
||||
def release_host_resources(self) -> None:
|
||||
self.shutdown()
|
||||
|
||||
def _find_mamba_component(self):
|
||||
return next(
|
||||
(
|
||||
component
|
||||
for component in self._components_tuple
|
||||
if component.component_type is ComponentType.MAMBA
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
def _allocate_external_mamba_slot(self) -> Optional[torch.Tensor]:
|
||||
if self._mamba_component is None:
|
||||
return None
|
||||
allocator = self.req_to_token_pool.mamba_allocator
|
||||
slot = allocator.alloc(1)
|
||||
if slot is None:
|
||||
self.evict(EvictParams(num_tokens=0, mamba_num=1))
|
||||
slot = allocator.alloc(1)
|
||||
if slot is not None:
|
||||
physical = self.req_to_token_pool.translate_mamba_indices(slot)
|
||||
self.lmcache_connector.reset_mamba_checkpoint_metadata(physical)
|
||||
return slot
|
||||
|
||||
def _arm_external_mamba_cow(self, flow: LMCacheExternalFlow, req: Req) -> None:
|
||||
"""Give a request a mutable state slot and COW from the loaded checkpoint."""
|
||||
checkpoint = flow.mamba_value
|
||||
if checkpoint is None:
|
||||
return
|
||||
if req.kv.mamba_pool_idx is None:
|
||||
active = self._allocate_external_mamba_slot()
|
||||
assert active is not None, "Cannot allocate Mamba request state for LMCache"
|
||||
req.kv.mamba_pool_idx = active[0]
|
||||
flow.request_mamba_value = active
|
||||
flow.allocated_request_mamba_for_load = True
|
||||
elif flow.request_mamba_value is None:
|
||||
flow.request_mamba_value = req.kv.mamba_pool_idx.reshape(1)
|
||||
flow.allocated_request_mamba_for_load = False
|
||||
flow.load_req = req
|
||||
req.kv.mamba_cow_src_index = checkpoint
|
||||
req.kv.mamba_needs_clear = False
|
||||
|
||||
def _allocate_external_slots(self, num_tokens: int) -> Optional[torch.Tensor]:
|
||||
allocator = self.token_to_kv_pool_allocator
|
||||
if not self.is_swa_enabled:
|
||||
if allocator.available_size() < num_tokens:
|
||||
self.evict(EvictParams(num_tokens=num_tokens))
|
||||
return allocator.alloc(num_tokens)
|
||||
|
||||
# Allocate SWA only for the trailing window; older pages use dummy page 0.
|
||||
assert self._sliding_window_size is not None
|
||||
swa_tail_tokens = min(
|
||||
num_tokens,
|
||||
self.lmcache_connector.aligned_swa_window_size(),
|
||||
)
|
||||
full_allocator = allocator.full_attn_allocator
|
||||
swa_allocator = allocator.swa_attn_allocator
|
||||
if full_allocator.available_size() < num_tokens:
|
||||
self.evict(EvictParams(num_tokens=num_tokens))
|
||||
if swa_allocator.available_size() < swa_tail_tokens:
|
||||
self.evict(EvictParams(swa_num_tokens=swa_tail_tokens))
|
||||
|
||||
full_indices = full_allocator.alloc(num_tokens)
|
||||
if full_indices is None:
|
||||
return None
|
||||
if swa_tail_tokens == 0:
|
||||
return full_indices
|
||||
|
||||
tail_full_indices = full_indices[-swa_tail_tokens:]
|
||||
if isinstance(allocator, UnifiedSWAAllocatorBase):
|
||||
# Unified-memory FULL and SWA share virtual page IDs.
|
||||
virtual_pages = torch.unique(tail_full_indices // self.page_size)
|
||||
try:
|
||||
allocator.swa_attn_allocator.alloc_with_virtual(virtual_pages)
|
||||
except Exception:
|
||||
full_allocator.free(full_indices)
|
||||
logger.exception("Failed to allocate unified SWA slots for LMCache")
|
||||
return None
|
||||
else:
|
||||
swa_indices = swa_allocator.alloc(swa_tail_tokens)
|
||||
if swa_indices is None:
|
||||
full_allocator.free(full_indices)
|
||||
return None
|
||||
allocator.set_full_to_swa_mapping(tail_full_indices, swa_indices)
|
||||
return full_indices
|
||||
|
||||
def _start_external_load(
|
||||
self, flow: LMCacheExternalFlow, req: Req
|
||||
) -> Optional[torch.Tensor]:
|
||||
assert flow.total_hit is not None
|
||||
assert flow.local_hit_tokens is not None
|
||||
local_hit = flow.local_hit_tokens
|
||||
latest = super().match_prefix(MatchPrefixParams(key=flow.key[:local_hit]))
|
||||
total_hit = min(flow.total_hit, len(flow.key))
|
||||
if total_hit <= local_hit:
|
||||
return self.tree_core.empty_match_result.device_indices
|
||||
|
||||
# Pin the common L1 boundary before allocation can trigger eviction.
|
||||
flow.anchor_node = latest.last_device_node
|
||||
flow.anchor_lock = self.inc_lock_ref(flow.anchor_node).to_dec_params()
|
||||
num_tokens = total_hit - local_hit
|
||||
device_indices = self._allocate_external_slots(num_tokens)
|
||||
mamba_value = None
|
||||
allocated_mamba_for_load = False
|
||||
request_mamba_value = None
|
||||
allocated_request_mamba_for_load = False
|
||||
if device_indices is not None and self._mamba_component is not None:
|
||||
# Keep the immutable checkpoint separate from mutable request state.
|
||||
mamba_value = self._allocate_external_mamba_slot()
|
||||
allocated_mamba_for_load = mamba_value is not None
|
||||
if req.kv.mamba_pool_idx is None:
|
||||
request_mamba_value = self._allocate_external_mamba_slot()
|
||||
allocated_request_mamba_for_load = request_mamba_value is not None
|
||||
else:
|
||||
request_mamba_value = req.kv.mamba_pool_idx.reshape(1)
|
||||
allocation_ok = torch.tensor(
|
||||
[
|
||||
int(
|
||||
device_indices is not None
|
||||
and (
|
||||
self._mamba_component is None
|
||||
or (mamba_value is not None and request_mamba_value is not None)
|
||||
)
|
||||
)
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
)
|
||||
self.lmcache_connector.parallel_all_reduce(
|
||||
allocation_ok, torch.distributed.ReduceOp.MIN
|
||||
)
|
||||
if not allocation_ok.item():
|
||||
if device_indices is not None:
|
||||
self.token_to_kv_pool_allocator.free(device_indices)
|
||||
if allocated_mamba_for_load and mamba_value is not None:
|
||||
self.req_to_token_pool.mamba_allocator.free(mamba_value)
|
||||
if allocated_request_mamba_for_load and request_mamba_value is not None:
|
||||
self.req_to_token_pool.mamba_allocator.free(request_mamba_value)
|
||||
self._release_flow_anchor(flow)
|
||||
logger.debug(
|
||||
"LMCache retrieve declined for %s: a parallel rank cannot allocate "
|
||||
"%d GPU slots",
|
||||
flow.lookup.request_id,
|
||||
num_tokens,
|
||||
)
|
||||
return None
|
||||
assert device_indices is not None
|
||||
flow.mamba_value = mamba_value
|
||||
flow.request_mamba_value = request_mamba_value
|
||||
flow.allocated_request_mamba_for_load = allocated_request_mamba_for_load
|
||||
flow.load_req = req
|
||||
flow.free_mamba_after_load = allocated_mamba_for_load
|
||||
flow.loaded_skip_tokens = 0
|
||||
if allocated_request_mamba_for_load:
|
||||
assert request_mamba_value is not None
|
||||
req.kv.mamba_pool_idx = request_mamba_value[0]
|
||||
req.kv.mamba_needs_clear = False
|
||||
|
||||
try:
|
||||
load_start = local_hit // self.lmcache_connector.chunk_size
|
||||
load_start *= self.lmcache_connector.chunk_size
|
||||
# Submit asynchronously while admission can still fall back to prefill.
|
||||
flow.load = self.lmcache_connector.submit_load(
|
||||
flow.lookup,
|
||||
self.lmcache_connector.device_indices_by_group(
|
||||
device_indices,
|
||||
mamba_value=mamba_value,
|
||||
mamba_transfer_tokens=total_hit - load_start,
|
||||
),
|
||||
local_hit_tokens=local_hit,
|
||||
owned_device_indices=device_indices,
|
||||
producer_stream=self._forward_stream,
|
||||
)
|
||||
if not self.lmcache_connector.prepare_load_on_stream(
|
||||
flow.load, self._forward_stream
|
||||
):
|
||||
raise RuntimeError("LMCache server rejected the retrieve request")
|
||||
if mamba_value is not None:
|
||||
# Forward waits for H2D before deferred Mamba COW runs.
|
||||
self._arm_external_mamba_cow(flow, req)
|
||||
except Exception:
|
||||
self.token_to_kv_pool_allocator.free(device_indices)
|
||||
if allocated_mamba_for_load and mamba_value is not None:
|
||||
self.req_to_token_pool.mamba_allocator.free(mamba_value)
|
||||
if allocated_request_mamba_for_load and request_mamba_value is not None:
|
||||
self.req_to_token_pool.mamba_allocator.free(request_mamba_value)
|
||||
req.kv.mamba_pool_idx = None
|
||||
if req.kv.mamba_cow_src_index is mamba_value:
|
||||
req.kv.mamba_cow_src_index = None
|
||||
flow.mamba_value = None
|
||||
flow.request_mamba_value = None
|
||||
flow.allocated_request_mamba_for_load = False
|
||||
flow.load_req = None
|
||||
flow.free_mamba_after_load = False
|
||||
self._release_flow_anchor(flow)
|
||||
logger.exception(
|
||||
"LMCache retrieve submission failed for %s", flow.lookup.request_id
|
||||
)
|
||||
flow.load = None
|
||||
return None
|
||||
return device_indices[flow.loaded_skip_tokens :]
|
||||
|
||||
def _release_flow_anchor(self, flow: LMCacheExternalFlow) -> None:
|
||||
assert (flow.anchor_node is None) == (flow.anchor_lock is None), (
|
||||
"LMCache flow anchor node and lock must be set together"
|
||||
)
|
||||
if flow.anchor_node is not None:
|
||||
assert flow.anchor_lock is not None
|
||||
self.dec_lock_ref(flow.anchor_node, flow.anchor_lock)
|
||||
flow.anchor_node = None
|
||||
flow.anchor_lock = None
|
||||
|
||||
def _release_unused_loaded_slots(self, flow: LMCacheExternalFlow) -> None:
|
||||
"""Free retrieved slots shadowed by a longer rank-local L1 prefix."""
|
||||
assert flow.load is not None
|
||||
release_end = min(flow.loaded_skip_tokens, len(flow.load.device_indices))
|
||||
if release_end <= flow.released_skip_tokens:
|
||||
return
|
||||
self.token_to_kv_pool_allocator.free(
|
||||
flow.load.device_indices[flow.released_skip_tokens : release_end]
|
||||
)
|
||||
flow.released_skip_tokens = release_end
|
||||
|
||||
def _finish_failed_load(self, flow: LMCacheExternalFlow) -> None:
|
||||
assert flow.load is not None
|
||||
handle = (
|
||||
flow.load_req.cache_request_handle if flow.load_req is not None else None
|
||||
)
|
||||
if not flow.prefix_published:
|
||||
self.token_to_kv_pool_allocator.free(
|
||||
flow.load.device_indices[flow.released_skip_tokens :]
|
||||
)
|
||||
if flow.free_mamba_after_load and flow.mamba_value is not None:
|
||||
self.req_to_token_pool.mamba_allocator.free(flow.mamba_value)
|
||||
if (
|
||||
flow.load_req is not None
|
||||
and flow.load_req.kv.mamba_cow_src_index is flow.mamba_value
|
||||
):
|
||||
flow.load_req.kv.mamba_cow_src_index = None
|
||||
flow.mamba_value = None
|
||||
if (
|
||||
not flow.cancelled
|
||||
and flow.allocated_request_mamba_for_load
|
||||
and flow.request_mamba_value is not None
|
||||
and flow.load_req is not None
|
||||
and flow.load_req.kv.mamba_pool_idx is not None
|
||||
and int(flow.load_req.kv.mamba_pool_idx.item())
|
||||
== int(flow.request_mamba_value[0].item())
|
||||
):
|
||||
self.req_to_token_pool.mamba_allocator.free(flow.request_mamba_value)
|
||||
flow.load_req.kv.mamba_pool_idx = None
|
||||
flow.request_mamba_value = None
|
||||
flow.allocated_request_mamba_for_load = False
|
||||
flow.load_req = None
|
||||
flow.free_mamba_after_load = False
|
||||
self._release_flow_anchor(flow)
|
||||
rid = flow.lookup.request_id
|
||||
self._external_flows.pop(rid, None)
|
||||
if handle is not None:
|
||||
if flow.cancelled:
|
||||
self.prefetch_loaded_tokens_by_reqid.pop(handle, None)
|
||||
else:
|
||||
self.prefetch_loaded_tokens_by_reqid[handle] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(handle, None)
|
||||
|
||||
def _finish_successful_load(self, flow: LMCacheExternalFlow) -> None:
|
||||
"""Finish local bookkeeping after LMCache has completed the retrieve."""
|
||||
assert flow.load is not None and flow.load_completed and flow.load.result
|
||||
self._release_unused_loaded_slots(flow)
|
||||
# Keep Mamba state until the cache callback publishes it.
|
||||
|
||||
def _finalize_retired_flow(self, flow: LMCacheExternalFlow) -> None:
|
||||
"""Release a loaded flow after its cache callback retires it."""
|
||||
assert flow.load is not None and flow.load_completed and flow.load.result
|
||||
self._finish_successful_load(flow)
|
||||
if flow.mamba_value is not None:
|
||||
# Free checkpoints from request paths that retire without insertion.
|
||||
if (
|
||||
flow.load_req is not None
|
||||
and flow.load_req.kv.mamba_cow_src_index is flow.mamba_value
|
||||
):
|
||||
flow.load_req.kv.mamba_cow_src_index = None
|
||||
self.req_to_token_pool.mamba_allocator.free(flow.mamba_value)
|
||||
flow.mamba_value = None
|
||||
flow.free_mamba_after_load = False
|
||||
flow.request_mamba_value = None
|
||||
flow.allocated_request_mamba_for_load = False
|
||||
flow.load_req = None
|
||||
self._release_flow_anchor(flow)
|
||||
rid = flow.lookup.request_id
|
||||
self._external_flows.pop(rid, None)
|
||||
|
||||
def _retire_loaded_flow(self, rid: str) -> None:
|
||||
"""Mark a load for retirement without entering a collective."""
|
||||
flow = self._external_flows.get(rid)
|
||||
if flow is None:
|
||||
return
|
||||
if flow.load is None:
|
||||
if flow.total_hit is None:
|
||||
return
|
||||
self.lmcache_connector.free_lookup_locks(
|
||||
rid,
|
||||
start=flow.lookup.lock_start,
|
||||
end=flow.total_hit,
|
||||
)
|
||||
self._external_flows.pop(rid, None)
|
||||
return
|
||||
|
||||
flow.retire_requested = True
|
||||
if not flow.load_completed:
|
||||
return
|
||||
if flow.load.result:
|
||||
self._finalize_retired_flow(flow)
|
||||
else:
|
||||
cancelled = flow.cancelled
|
||||
self._finish_failed_load(flow)
|
||||
if not cancelled:
|
||||
logger.warning(
|
||||
"LMCache retrieve failed after admission for request %s", rid
|
||||
)
|
||||
|
||||
def _prepare_external_slots_for_insert(self, req: Req) -> None:
|
||||
"""Restore tree ownership and mark the missing SWA prefix as evicted."""
|
||||
flow = self._external_flows.get(req.rid)
|
||||
if flow is None or flow.load is None:
|
||||
return
|
||||
tree_owned_len = flow.load.local_hit_tokens + flow.loaded_skip_tokens
|
||||
req.kv.cache_protected_len = min(req.kv.cache_protected_len, tree_owned_len)
|
||||
if self.is_swa_enabled:
|
||||
external_tokens = len(flow.load.device_indices)
|
||||
swa_missing_end = flow.load.local_hit_tokens + max(
|
||||
external_tokens - self.lmcache_connector.aligned_swa_window_size(),
|
||||
0,
|
||||
)
|
||||
req.kv.swa_evicted_seqlen = max(req.kv.swa_evicted_seqlen, swa_missing_end)
|
||||
|
||||
def _publish_external_loaded_prefix(self, req: Req, *, token_ids_len: int) -> None:
|
||||
"""Publish retrieved KV and immutable Mamba state into the device tree."""
|
||||
flow = self._external_flows.get(req.rid)
|
||||
if (
|
||||
flow is None
|
||||
or flow.load is None
|
||||
or flow.total_hit is None
|
||||
or flow.prefix_published
|
||||
):
|
||||
return
|
||||
# Restore the ownership boundary once before the parent cache callback.
|
||||
self._prepare_external_slots_for_insert(req)
|
||||
if flow.mamba_value is None:
|
||||
# The parent callback will adopt the retrieved Full/SWA slots.
|
||||
flow.prefix_published = True
|
||||
return
|
||||
|
||||
total_hit = min(flow.total_hit, len(flow.key))
|
||||
assert total_hit > 0, "LMCache loaded prefix must contain at least one token"
|
||||
if total_hit > token_ids_len:
|
||||
raise RuntimeError(
|
||||
"LMCache Mamba checkpoint lies beyond the request KV boundary: "
|
||||
f"{total_hit}>{token_ids_len} for request {req.rid}"
|
||||
)
|
||||
prev_prefix_len = min(req.kv.cache_protected_len, total_hit)
|
||||
kv_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, :token_ids_len
|
||||
]
|
||||
key = flow.key[:total_hit]
|
||||
checkpoint = flow.mamba_value
|
||||
result = self.insert(
|
||||
InsertParams(
|
||||
key=key,
|
||||
value=kv_indices[:total_hit].to(dtype=torch.int64, copy=True),
|
||||
mamba_value=checkpoint,
|
||||
prev_prefix_len=prev_prefix_len,
|
||||
swa_evicted_seqlen=req.kv.swa_evicted_seqlen,
|
||||
chunked=True,
|
||||
priority=req.priority or 0,
|
||||
)
|
||||
)
|
||||
|
||||
# Free our checkpoint if a concurrent insertion already owns this boundary.
|
||||
if result.mamba_exist:
|
||||
self.req_to_token_pool.mamba_allocator.free(checkpoint)
|
||||
flow.mamba_value = None
|
||||
flow.free_mamba_after_load = False
|
||||
|
||||
matched = super().match_prefix(
|
||||
MatchPrefixParams(key=key, req=req, cow_mamba=False)
|
||||
)
|
||||
if len(matched.device_indices) < total_hit:
|
||||
raise RuntimeError(
|
||||
"LMCache loaded prefix was inserted but is not reusable across "
|
||||
f"all UnifiedRadixCache components: {len(matched.device_indices)}"
|
||||
f"/{total_hit} tokens for request {req.rid}"
|
||||
)
|
||||
canonical = matched.device_indices[:total_hit]
|
||||
self.req_to_token_pool.write(
|
||||
(req.kv.req_pool_idx, slice(prev_prefix_len, total_hit)),
|
||||
canonical[prev_prefix_len:],
|
||||
)
|
||||
|
||||
# Move the request lock to the published boundary.
|
||||
if req.last_node is not None:
|
||||
self._dec_req_lock(req)
|
||||
lock_result = self.inc_lock_ref(matched.last_device_node)
|
||||
if total_hit < token_ids_len:
|
||||
req.prefix_indices = torch.cat(
|
||||
[canonical, kv_indices[total_hit:].to(dtype=torch.int64, copy=True)]
|
||||
)
|
||||
else:
|
||||
req.prefix_indices = canonical
|
||||
req.kv.cache_protected_len = total_hit
|
||||
req.last_node = matched.last_device_node
|
||||
req.lock_receipt = lock_result.to_dec_params()
|
||||
req.swa_prefix_lock_released = False
|
||||
flow.request_mamba_value = None
|
||||
flow.allocated_request_mamba_for_load = False
|
||||
flow.load_req = None
|
||||
flow.prefix_published = True
|
||||
|
||||
def _submit_store(self, req: Req, token_ids: list[int]) -> None:
|
||||
key = RadixKey(
|
||||
token_ids,
|
||||
req.extra_key,
|
||||
is_bigram=self.tree_core.is_eagle,
|
||||
cache_salt=req.cache_salt,
|
||||
).page_aligned(self.page_size)
|
||||
aligned_len = (
|
||||
len(key)
|
||||
// self.lmcache_connector.chunk_size
|
||||
* self.lmcache_connector.chunk_size
|
||||
)
|
||||
key = key[:aligned_len]
|
||||
if len(key) == 0:
|
||||
return
|
||||
matched = super().match_prefix(MatchPrefixParams(key=key))
|
||||
# Store the currently resident SWA prefix; later callbacks may extend it.
|
||||
resident_len = torch.tensor(
|
||||
[min(len(matched.device_indices), len(key))],
|
||||
dtype=torch.int64,
|
||||
device="cpu",
|
||||
)
|
||||
self.lmcache_connector.parallel_all_reduce(
|
||||
resident_len, torch.distributed.ReduceOp.MIN
|
||||
)
|
||||
resident_len = (
|
||||
int(resident_len.item())
|
||||
// self.lmcache_connector.chunk_size
|
||||
* self.lmcache_connector.chunk_size
|
||||
)
|
||||
store_start = self.lmcache_connector.get_store_start(req.rid, len(key))
|
||||
if resident_len <= store_start:
|
||||
logger.debug(
|
||||
"LMCache store skipped for %s: no new resident chunk "
|
||||
"after token %d (%d/%d tokens)",
|
||||
req.rid,
|
||||
store_start,
|
||||
resident_len,
|
||||
len(key),
|
||||
)
|
||||
return
|
||||
if resident_len < len(key):
|
||||
logger.debug(
|
||||
"LMCache store for %s is limited to the common resident prefix: "
|
||||
"%d/%d tokens",
|
||||
req.rid,
|
||||
resident_len,
|
||||
len(key),
|
||||
)
|
||||
key = key[:resident_len]
|
||||
# Re-match so the lock and Mamba state use the shortened boundary.
|
||||
matched = super().match_prefix(MatchPrefixParams(key=key))
|
||||
store_indices = matched.device_indices[store_start : len(key)]
|
||||
if len(store_indices) != len(key) - store_start:
|
||||
logger.debug(
|
||||
"LMCache store skipped for %s: store indices length: %d, "
|
||||
"store start: %d, key length: %d",
|
||||
req.rid,
|
||||
len(store_indices),
|
||||
store_start,
|
||||
len(key),
|
||||
)
|
||||
return
|
||||
lock_params = self.inc_lock_ref(matched.last_device_node).to_dec_params()
|
||||
mamba_value = (
|
||||
self.tree_core.get_component_device_value(
|
||||
matched.best_match_node, ComponentType.MAMBA
|
||||
)
|
||||
if self._mamba_component is not None
|
||||
else None
|
||||
)
|
||||
mamba_is_resident = torch.tensor(
|
||||
[int(self._mamba_component is None or mamba_value is not None)],
|
||||
dtype=torch.int32,
|
||||
device="cpu",
|
||||
)
|
||||
self.lmcache_connector.parallel_all_reduce(
|
||||
mamba_is_resident, torch.distributed.ReduceOp.MIN
|
||||
)
|
||||
if not mamba_is_resident.item():
|
||||
self.dec_lock_ref(matched.last_device_node, lock_params)
|
||||
logger.debug(
|
||||
"LMCache store skipped for %s: no Mamba checkpoint at token %d",
|
||||
req.rid,
|
||||
len(key),
|
||||
)
|
||||
return
|
||||
try:
|
||||
operation = self.lmcache_connector.submit_store(
|
||||
req.rid,
|
||||
key.raw_token_ids()[: len(key)],
|
||||
self.lmcache_connector.device_indices_by_group(
|
||||
store_indices,
|
||||
mamba_value=mamba_value,
|
||||
),
|
||||
device_indices_start=store_start,
|
||||
cache_salt=self.lmcache_connector.build_cache_salt(
|
||||
req.cache_salt, req.extra_key
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
self.dec_lock_ref(matched.last_device_node, lock_params)
|
||||
logger.exception("LMCache store submission failed for %s", req.rid)
|
||||
return
|
||||
if operation is None:
|
||||
self.dec_lock_ref(matched.last_device_node, lock_params)
|
||||
return
|
||||
self._pending_stores.append(
|
||||
LMCachePendingStore(operation, matched.last_device_node, lock_params)
|
||||
)
|
||||
self._pending_store_counts[req.rid] = (
|
||||
self._pending_store_counts.get(req.rid, 0) + 1
|
||||
)
|
||||
|
||||
def _request_session_finish(self, rid: str) -> None:
|
||||
"""End the LMCache session after this request's stores have completed."""
|
||||
self._session_finish_requested.add(rid)
|
||||
self._finish_session_if_store_idle(rid)
|
||||
|
||||
def _finish_session_if_store_idle(self, rid: str) -> None:
|
||||
if (
|
||||
rid not in self._session_finish_requested
|
||||
or self._pending_store_counts.get(rid, 0) > 0
|
||||
):
|
||||
return
|
||||
self._session_finish_requested.remove(rid)
|
||||
self.lmcache_connector.finish_request(rid)
|
||||
@@ -1,118 +0,0 @@
|
||||
try:
|
||||
from lmcache.integration.sglang.sglang_adapter import (
|
||||
LMCacheLayerwiseConnector,
|
||||
LoadMetadata,
|
||||
StoreMetadata,
|
||||
)
|
||||
except ImportError:
|
||||
raise RuntimeError(
|
||||
"LMCache is not installed. Please install it by running `pip install lmcache` in the root directory of LMCache"
|
||||
)
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
|
||||
|
||||
def test_load_store_metadata():
|
||||
model_config = ModelConfig(
|
||||
model_path="Qwen/Qwen3-4B",
|
||||
)
|
||||
|
||||
# Generate Dummy KV Cache
|
||||
head_num = model_config.num_key_value_heads
|
||||
head_dim = model_config.head_dim
|
||||
layer_num = model_config.num_hidden_layers
|
||||
buffer_size = 256
|
||||
input_id_len = 16
|
||||
|
||||
k_buffer = [
|
||||
torch.randn(buffer_size, head_num, head_dim, dtype=torch.bfloat16).cuda()
|
||||
for _ in range(layer_num)
|
||||
]
|
||||
v_buffer = [
|
||||
torch.randn(buffer_size, head_num, head_dim, dtype=torch.bfloat16).cuda()
|
||||
for _ in range(layer_num)
|
||||
]
|
||||
|
||||
connector = LMCacheLayerwiseConnector(
|
||||
model_config, 1, 0, k_buffer, v_buffer, config_file="example_config_ip.yaml"
|
||||
)
|
||||
|
||||
fake_token_ids = torch.randint(0, model_config.vocab_size, (input_id_len,)).tolist()
|
||||
fake_kv_indices = torch.randint(0, buffer_size, (input_id_len,))
|
||||
offset = 0
|
||||
|
||||
store_metadata = StoreMetadata(
|
||||
last_node=None,
|
||||
token_ids=fake_token_ids,
|
||||
kv_indices=fake_kv_indices,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
load_metadata = LoadMetadata(
|
||||
token_ids=fake_token_ids,
|
||||
slot_mapping=fake_kv_indices,
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
current_stream = torch.cuda.current_stream()
|
||||
|
||||
retrieve_token_num = connector.start_load_kv(load_metadata)
|
||||
assert retrieve_token_num == 0
|
||||
|
||||
connector.store_kv(store_metadata)
|
||||
current_stream.synchronize()
|
||||
|
||||
# check retrieve
|
||||
gt_key_buffer = [
|
||||
torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda()
|
||||
for _ in range(layer_num)
|
||||
]
|
||||
gt_value_buffer = [
|
||||
torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda()
|
||||
for _ in range(layer_num)
|
||||
]
|
||||
|
||||
for i in range(layer_num):
|
||||
gt_key_buffer[i] = k_buffer[i][fake_kv_indices]
|
||||
gt_value_buffer[i] = v_buffer[i][fake_kv_indices]
|
||||
|
||||
# clear the k_buffer and v_buffer
|
||||
for _ in range(layer_num):
|
||||
k_buffer[i].zero_()
|
||||
v_buffer[i].zero_()
|
||||
|
||||
retrieve_token_num = connector.start_load_kv(load_metadata)
|
||||
assert retrieve_token_num == input_id_len
|
||||
|
||||
for i in range(layer_num):
|
||||
current_stream.synchronize()
|
||||
connector.load_kv_layerwise(i)
|
||||
|
||||
current_stream.synchronize()
|
||||
test_key_buffer = [
|
||||
torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda()
|
||||
for _ in range(layer_num)
|
||||
]
|
||||
test_value_buffer = [
|
||||
torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda()
|
||||
for _ in range(layer_num)
|
||||
]
|
||||
|
||||
for i in range(layer_num):
|
||||
test_key_buffer[i] = k_buffer[i][fake_kv_indices]
|
||||
test_value_buffer[i] = v_buffer[i][fake_kv_indices]
|
||||
|
||||
for i in range(layer_num):
|
||||
assert torch.allclose(test_key_buffer[i], gt_key_buffer[i])
|
||||
assert torch.allclose(test_value_buffer[i], gt_value_buffer[i])
|
||||
|
||||
print("================================================")
|
||||
print("TEST_LOAD_STORE_METADATA PASSED!")
|
||||
print("================================================")
|
||||
connector.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_load_store_metadata()
|
||||
Reference in New Issue
Block a user