[KVCache] Support lmcache unified radix cache (#38652)

Signed-off-by: chunxiaozheng <1179548172@qq.com>
Co-authored-by: Yuwei An <ayw.sirius19@gmail.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
chunxiaozheng
2026-09-25 14:08:45 -07:00
committed by GitHub
co-authored by Yuwei An Claude Opus 5.5
parent d3e3945e41
commit cf5df82680
20 changed files with 1158 additions and 1301 deletions
@@ -430,6 +430,49 @@ def handle_cache_compatibility(server_args: Any) -> None:
if prefix_tails is not None and prefix_tails < 0:
raise ValueError("--swa-prefix-tails should be a non-negative integer.")
if cfg.enable_lmcache:
if cfg.enable_hierarchical_cache:
raise ValueError(
"--enable-lmcache and --enable-hierarchical-cache are "
"mutually exclusive"
)
if cfg.enable_unified_cache_external_linker:
raise ValueError(
"--enable-lmcache and --enable-unified-cache-external-linker "
"are mutually exclusive"
)
if cfg.disable_radix_cache:
raise ValueError("--enable-lmcache requires radix cache to be enabled")
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
if SpeculativeAlgorithm.from_string(cfg.speculative_algorithm).is_speculative():
raise NotImplementedError(
"LMCacheUnifiedRadixCache does not yet support speculative decoding"
)
if cfg.enable_dp_attention:
raise NotImplementedError(
"LMCacheUnifiedRadixCache does not yet support DP attention"
)
if cfg.dcp_size > 1:
raise NotImplementedError(
"--enable-lmcache with --dcp-size > 1 is not supported: "
"LMCache has no DCP-aware index translation"
)
if cfg.enable_streaming_session:
raise NotImplementedError(
"LMCacheUnifiedRadixCache does not yet support streaming sessions"
)
if cfg.hicache_host_memory_mode == "buffer_only":
raise ValueError(
"--hicache-host-memory-mode=buffer_only is a HiCache-only mode"
)
if cfg.disaggregation_mode != "null":
raise NotImplementedError(
"LMCacheUnifiedRadixCache currently supports colocated "
"prefill/decode scheduling only"
)
def handle_unified_memory_pool(server_args: Any) -> None:
@@ -91,6 +91,11 @@ def handle_int8_mamba_checkpoint(server_args: Any):
f"radix cache; --radix-cache-backend={cfg.radix_cache_backend!r} "
"is not int8-aware. Omit --radix-cache-backend."
)
if cfg.enable_lmcache:
raise ValueError(
"--enable-int8-mamba-checkpoint is not supported together with "
"--enable-lmcache: LMCache is not int8-aware. Disable one of them."
)
def validate_mamba_extra_buffer(view, model_arch: str, *, mamba_cache_chunk_size_of):
+14 -6
View File
@@ -496,6 +496,7 @@ class Scheduler(
)
self.page_size = get_schedule().page_size
self.enable_hierarchical_cache = get_memory().enable_hierarchical_cache
self.enable_lmcache = get_memory().enable_lmcache
self.enable_session_radix_cache = get_memory().enable_session_radix_cache
self.enable_hicache_storage = get_memory().hicache_storage_backend is not None
self.enable_unified_cache_external_linker = (
@@ -2456,7 +2457,9 @@ class Scheduler(
is_generation=self.is_generation,
spec_algorithm=self.spec_algorithm,
disaggregation_mode=self.disaggregation_mode,
enable_hicache_storage=lambda: self.enable_hicache_storage,
enable_hicache_storage=lambda: (
self.enable_hicache_storage or self.enable_lmcache
),
rust_server=self.rust_server,
)
@@ -3108,7 +3111,7 @@ class Scheduler(
self.handle_generate_request(tokenized_req)
def _prefetch_kvcache(self, req: Req, storage_hit_end: Optional[int] = None):
if self.enable_hicache_storage:
if self.enable_hicache_storage or self.enable_lmcache:
req.init_next_round_input(self.tree_cache, cow_mamba=False)
tree_cache = self.tree_cache
buffer_mode = get_memory().hicache_host_memory_mode == "buffer_only"
@@ -3612,6 +3615,7 @@ class Scheduler(
self.enable_hierarchical_cache
or get_memory().enable_flexkv
or self.enable_unified_cache_external_linker
or self.enable_lmcache
):
self.tree_cache.check_hicache_events()
if self.enable_hicache_storage:
@@ -3966,7 +3970,7 @@ class Scheduler(
):
break
if self.enable_hicache_storage:
if self.enable_hicache_storage or self.enable_lmcache:
prefetch_done = self.tree_cache.check_prefetch_progress(
req.cache_request_handle
)
@@ -4009,6 +4013,7 @@ class Scheduler(
if res == AddReqResult.NO_TOKEN:
if (
self.enable_hierarchical_cache
or self.enable_lmcache
or self.enable_unified_cache_external_linker
):
# Set batch_is_full after making sure there are requests that can be served
@@ -4879,12 +4884,12 @@ class Scheduler(
return self.external_corpus_manager.list(recv_req)
def clear_hicache_storage_wrapped(self, recv_req: ClearHiCacheReqInput):
if self.enable_hierarchical_cache:
if self.enable_hierarchical_cache or self.enable_lmcache:
self.tree_cache.clear_storage_backend()
logger.info("Hierarchical cache cleared successfully!")
logger.info("Hierarchical cache or LMCache cleared successfully!")
if_success = True
else:
logging.warning("Hierarchical cache is not enabled.")
logging.warning("Hierarchical cache or LMCache is not enabled.")
if_success = False
return ClearHiCacheReqOutput(success=if_success)
@@ -4914,6 +4919,7 @@ class Scheduler(
if (
self.enable_hicache_storage
or self.disaggregation_mode != DisaggregationMode.NULL
or self.enable_lmcache
):
# Storage and transfer workers need the GIL between I/O calls.
# Singleton PD polls no longer yield through a collective.
@@ -5044,6 +5050,8 @@ class Scheduler(
# storage writes still hold host staging
# (buffer-mode unified tree only).
idle &= tc.buffer_pipeline.is_idle()
elif self.enable_lmcache:
idle &= not self.tree_cache.has_pending_cache_operations()
return idle
@@ -79,6 +79,8 @@ class SchedulerOutputStreamer:
storage_backend = cache_controller.storage_backend
if storage_backend is not None:
storage_backend_type = type(storage_backend).__name__
elif self.server_args.enable_lmcache:
storage_backend_type = "LMCache"
return storage_backend_type
def get_cached_tokens_details(self, req: Req) -> Optional[CachedTokensDetails]:
+23 -13
View File
@@ -100,6 +100,29 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
return SWAChunkCache(params)
if get_memory().enable_lmcache:
from sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache import (
LMCacheUnifiedRadixCache,
)
from sglang.srt.mem_cache.unified_cache.components import ComponentType
tree_components = []
if not (ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0):
tree_components.append(ComponentType.FULL)
if ctx.is_hybrid_swa:
tree_components.append(ComponentType.SWA)
if ctx.is_hybrid_ssm:
tree_components.append(ComponentType.MAMBA)
params.tree_components = tuple(tree_components)
return LMCacheUnifiedRadixCache(
params,
model_config=ctx.model_config,
tp_size=ctx.tp_size,
tp_rank=ctx.tp_rank,
lmcache_config_file=get_memory().lmcache_config_file,
forward_stream=ctx.tp_worker.model_runner.forward_stream,
)
if get_memory().enable_unified_cache_external_linker:
return _create_unified_radix_cache(ctx, server_args, params)
@@ -108,19 +131,6 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
return PureSWARadixCache(params=params)
if get_memory().enable_lmcache:
from sglang.srt.mem_cache.storage.lmcache.lmc_radix_cache import (
LMCRadixCache,
)
return LMCRadixCache(
params=params,
model_config=ctx.model_config,
tp_size=ctx.tp_size,
rank=ctx.tp_rank,
tp_group=ctx.tp_group,
)
if get_memory().enable_flexkv:
# Importing the package side-effect registers the explicit
# ``--radix-cache-backend=flexkv`` factory; we then call the
@@ -2,9 +2,8 @@
A `RadixCache` subclass that routes sglang's host-tier KV cache through a
FlexKV [`KVManager`](https://github.com/taco-project/FlexKV) (CPU / SSD /
Remote offload). Same integration pattern as
[`LMCRadixCache`](../lmcache/README.md): `FlexKVRadixCache` overrides
`match_prefix` / `init_load_back` / `cache_finished_req` / `evict`; a
Remote offload). `FlexKVRadixCache` overrides `match_prefix` /
`init_load_back` / `cache_finished_req` / `evict`; a
`FlexKVConnector` façade talks to `KVManager`, `KVTPClient`, and a
3-axis (PP × CP × TP) sync context.
@@ -3,8 +3,8 @@
This module exposes :class:`FlexKVRadixCache`, a subclass of
:class:`sglang.srt.mem_cache.radix_cache.RadixCache` that delegates
host-side prefix storage to a FlexKV ``KVManager``. The design mirrors
``LMCRadixCache`` (the LMCache integration) so the scheduler-side
contract is identical:
the two-phase external-cache integration pattern, so the scheduler-side
contract is:
* MP (synchronous) mode — the default.
``match_prefix`` fires only a FlexKV LOOKUP and returns ``host_hit_length``;
@@ -118,7 +118,7 @@ class FlexKVRadixCache(RadixCache):
# forward layer blocks on its own eventfd.
self.flexkv_connector.register_layer_transfer_counter(kvcache)
# CUDA streams (mirroring LMCRadixCache).
# CUDA streams.
self.load_stream = torch.cuda.Stream()
self.store_stream = torch.cuda.Stream()
@@ -395,7 +395,7 @@ class FlexKVRadixCache(RadixCache):
"""Base cache_finished_req then fire an async FlexKV store."""
super().cache_finished_req(req, owned_kv_len=owned_kv_len)
# Compute the committed prefix mirroring LMCRadixCache's logic.
# Compute the committed prefix.
topk = get_spec().speculative_eagle_topk
enable_kv_committed_len = topk is None or topk == 1
if enable_kv_committed_len:
@@ -29,11 +29,9 @@ pip install -e . --no-build-isolation
## Use LMCache
LMCache supports two transport modes. **MP (multi-process, default)** issues a single blocking retrieve over ZMQ to a standalone daemon that owns the KV store and survives SGLang restarts. **IP (in-process)** uses an embedded layerwise connector — the cache lives and dies with the SGLang process. Mode selection is currently a code-level setting in `LMCRadixCache.__init__` (`self._mode`); only MP is reachable by default.
### MP mode (default): multi-process daemon
Uses `LMCacheMPConnector`. Daemon host/port come from the LMCache YAML config (`mp_host`, `mp_port`).
SGLang uses `LMCacheUnifiedRadixCache` with LMCache's multiprocess connector.
The standalone LMCache daemon owns the external cache and can survive SGLang
process restarts. Daemon host and port come from the LMCache YAML config.
Terminal 1 — start the LMCache daemon:
@@ -56,16 +54,3 @@ python -m sglang.launch_server \
```
For full LMCache config options see https://docs.lmcache.ai/api_reference/configurations.html.
### IP mode: in-process
Uses `LMCacheLayerwiseConnector`. KV transfer happens per layer inside the SGLang process; the cache lives and dies with the server. To enable, edit `LMCRadixCache.__init__` and set `self._mode = LMCacheMode.IP`.
The LMCache config still controls chunk_size and storage; `mp_host` / `mp_port` are ignored on this path. Use the bundled `example_config_ip.yaml`:
```bash
python -m sglang.launch_server \
--model-path MODEL \
--enable-lmcache \
--lmcache-config-file example_config_ip.yaml
```
@@ -0,0 +1 @@
"""LMCache-backed KV cache integrations."""
@@ -1,7 +0,0 @@
# Basic configurations
chunk_size: 256
# CPU offloading configurations
local_cpu: true
use_layerwise: true
max_local_cpu_size: 10 # number of CPU backend GB
@@ -1,519 +0,0 @@
from __future__ import annotations
import enum
import logging
import threading
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional, Tuple
import torch
from sglang.srt.mem_cache.base_prefix_cache import (
EvictParams,
EvictResult,
InitLoadBackParams,
MatchPrefixParams,
MatchResult,
)
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
from sglang.srt.runtime_context import get_memory, get_spec
from sglang.srt.utils import create_device_stream, device_stream_context
try:
from lmcache.integration.sglang.multi_process_adapter import LMCacheMPConnector
from lmcache.integration.sglang.sglang_adapter import (
LMCacheLayerwiseConnector,
LoadMetadata,
StoreMetadata,
)
from lmcache.integration.sglang.utils import lmcache_get_config
except ImportError as e:
raise RuntimeError(
"LMCache is not installed. Please install it by running `pip install lmcache`"
) from e
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
logger = logging.getLogger(__name__)
@dataclass
class _LMCacheLoadBackMarker:
"""Carries the data ``init_load_back`` needs from the
``match_prefix`` call in MP mode.
"""
key: RadixKey # detached snapshot of the matched key (the live query key
# aliases the req's growing fill_ids and must not be retained)
value_numel: int # number of tokens already in radix at match time
class LMCacheMode(enum.Enum):
MP = enum.auto() # multi-process mode
IP = enum.auto() # in-process mode
class LayerTransferCounter:
"""Minimal adapter that lets the memory pool notify LMCache per-layer.
The KV pool calls `wait_until(layer_id)` after finishing a layer, which we
translate into a `load_kv_layerwise(layer_id)` call on the LMCache connector
within the provided device stream.
"""
def __init__(
self,
num_layers: int,
load_stream: torch.Stream,
lmc_connector: LMCacheLayerwiseConnector,
printable: bool = False,
):
self.num_layers = num_layers
self.load_stream = load_stream
self.lmc_connector = lmc_connector
def wait_until(self, layer_id: int):
# Ensure ordering of the async loads wrt compute stream(s).
self.load_stream.synchronize()
with device_stream_context(self.load_stream):
self.lmc_connector.load_kv_layerwise(layer_id)
class LMCRadixCache(RadixCache):
"""RadixCache + LMCache IO.
IP mode keeps the existing layerwise connector and
its per-layer transfer hook: ``match_prefix`` kicks off the load via
``start_load_kv`` and SGLang's per-layer KV-pool hook drives subsequent
layers during forward.
MP mode uses ``LMCacheMPConnector`` with a two-phase
load: ``match_prefix`` fires LOOKUP only (``connector.lookup_kv``) and
returns ``host_hit_length`` on the ``MatchResult``; the SGLang
scheduler then calls `init_load_back` at dispatch time,
which fires the actual RETRIEVE (``connector.retrieve_kv``) into
pre-allocated GPU slots.
"""
def __init__(
self,
params: CacheInitParams,
model_config: Optional[ModelConfig] = None,
tp_size: int = 1,
rank: int = 0,
tp_group: Optional[torch.distributed.ProcessGroup] = None,
):
super().__init__(params)
cli_lmc_cfg = get_memory().lmcache_config_file or ""
kvcache = self.token_to_kv_pool_allocator.get_kvcache()
connector_kwargs = dict(
sgl_config=model_config,
tp_size=tp_size,
rank=rank,
# NOTE: The original implementation accessed private buffers via
# `_kvcache.k_buffer` / `.v_buffer`. We prefer public accessors when
# available; fall back to private fields if needed.
k_pool=getattr(
kvcache,
"k_buffer",
getattr(self.token_to_kv_pool_allocator._kvcache, "k_buffer"),
),
v_pool=getattr(
kvcache,
"v_buffer",
getattr(self.token_to_kv_pool_allocator._kvcache, "v_buffer"),
),
tp_group=tp_group.device_group if tp_group is not None else None,
)
self.load_stream = create_device_stream(self.device)
self.store_stream = create_device_stream(self.device)
# MP (multi-process) is the default. XPU defaults to IP (in-process
# layerwise) because the MP connector shares the KV cache via CUDA IPC
# (``Tensor._share_cuda_``), which is unavailable on XPU.
self._mode = LMCacheMode.IP if self.device.type == "xpu" else LMCacheMode.MP
if self._mode is LMCacheMode.MP:
if not cli_lmc_cfg:
raise ValueError(
"MP mode requires --lmcache-config-file (the YAML "
"supplies mp_host / mp_port)."
)
lm_cfg = lmcache_get_config(cli_lmc_cfg)
self.lmcache_connector = LMCacheMPConnector(
page_size=params.page_size,
host=lm_cfg.mp_host,
port=lm_cfg.mp_port,
**connector_kwargs,
)
elif self._mode is LMCacheMode.IP:
self.lmcache_connector = LMCacheLayerwiseConnector(
config_file=cli_lmc_cfg, **connector_kwargs
)
# Per-layer hook
self.layer_done_executor = LayerTransferCounter(
num_layers=(
model_config.num_hidden_layers if model_config is not None else 0
),
load_stream=self.load_stream,
lmc_connector=self.lmcache_connector,
)
kvcache.register_layer_transfer_counter(self.layer_done_executor)
self._in_flight_nodes: list[TreeNode] = []
self._node_lock = threading.Lock()
self._mp_load_back_markers: dict[str, _LMCacheLoadBackMarker] = {}
def reset(self):
super().reset()
if hasattr(self, "_in_flight_nodes"):
with self._node_lock:
self._in_flight_nodes.clear()
if hasattr(self, "_mp_load_back_markers"):
self._mp_load_back_markers.clear()
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
"""Dispatch to the mode-specific match_prefix.
MP mode → ``_mp_match_prefix`` (fires LOOKUP only).
IP mode → ``_ip_match_prefix`` (single-shot ``start_load_kv``
plus per-layer hook).
"""
key = params.key
if self.disable or not key:
return super().match_prefix(params)
if self.page_size != 1:
aligned_len = len(key) // self.page_size * self.page_size
key = key[:aligned_len]
base_res = super().match_prefix(params)
value: torch.Tensor = base_res.device_indices
last_node: TreeNode = base_res.last_device_node
if self._mode is LMCacheMode.MP:
if params.req is None:
return base_res
return self._mp_match_prefix(key, base_res, value, last_node, params.req)
elif self._mode is LMCacheMode.IP:
return self._ip_match_prefix(key, base_res, value, last_node)
return base_res
def _mp_match_prefix(
self,
key: RadixKey,
base_res: MatchResult,
value: torch.Tensor,
last_node: TreeNode,
req: Req,
) -> MatchResult:
"""MP LOOKUP
Returns a ``MatchResult`` with ``host_hit_length`` set when
LMCache has tokens beyond radix. Otherwise releases
the held read locks and returns the radix-only result.
"""
token_ids = key.raw_token_ids()
matched = self.lmcache_connector.lookup_kv(token_ids, req.rid)
if matched <= value.numel():
# Release the read locks; keep the pending session for end_session.
self.lmcache_connector.release_pending(req.rid)
return base_res
if token_ids is key.token_ids:
token_ids = token_ids[:]
self._mp_load_back_markers[req.rid] = _LMCacheLoadBackMarker(
key=RadixKey(
token_ids,
key.extra_key,
key.is_bigram,
cache_salt=key.cache_salt,
),
value_numel=int(value.numel()),
)
return MatchResult(
device_indices=value,
last_device_node=last_node,
last_host_node=last_node,
best_match_node=last_node,
host_hit_length=matched - int(value.numel()),
)
def _ip_match_prefix(
self,
key: RadixKey,
base_res: MatchResult,
value: torch.Tensor,
last_node: TreeNode,
) -> MatchResult:
"""IP mode: ``start_load_kv`` + per-layer hook.
Allocates slots for the page-aligned uncached tail and kicks off
the layerwise load. Returns ``base_res`` if there's nothing to
fetch or alloc/load fails.
"""
if value.numel() == len(key):
return base_res
uncached_len = len(key) - value.numel()
if uncached_len == 0:
return base_res
token_ids = key.raw_token_ids()
result = self._load_back(
key=key,
value_numel=int(value.numel()),
uncached_len=uncached_len,
last_node=last_node,
load_fn=lambda sm, pp: self._ip_load_back(
token_ids=token_ids,
value_numel=int(value.numel()),
slot_mapping=sm,
prefix_pad=pp,
),
)
if result is None:
return base_res
new_slots, new_node = result
return MatchResult(
device_indices=torch.cat([value, new_slots]),
last_device_node=new_node,
last_host_node=new_node,
best_match_node=new_node,
)
def init_load_back(
self, params: InitLoadBackParams
) -> Tuple[torch.Tensor, Optional[TreeNode]]:
"""MP RETRIEVE.
Called by the scheduler when ``match_prefix`` returned
``host_hit_length > 0``. Uses the cached LOOKUP result to
allocate slots and fire RETRIEVE, inserts the resulting
TreeNode into the radix tree, and returns
``(new_indices, new_last_node)``.
"""
req = params.req
marker = self._mp_load_back_markers.pop(req.rid)
last_node: TreeNode = params.best_match_node
result = self._load_back(
key=marker.key,
value_numel=marker.value_numel,
uncached_len=params.host_hit_length,
last_node=last_node,
load_fn=lambda sm, pp: self._mp_load_back(
marker=marker,
request_id=req.rid,
slot_mapping=sm,
prefix_pad=pp,
),
)
if result is None:
# Either alloc failed (locks still held by lookup_kv) or
# retrieve returned nothing (locks already released by
# retrieve_kv). release_pending is idempotent on locks_held.
self.lmcache_connector.release_pending(req.rid)
return (
torch.empty((0,), dtype=torch.int64, device=self.device),
last_node,
)
return result
def _load_back(
self,
*,
key: RadixKey,
value_numel: int,
uncached_len: int,
last_node: TreeNode,
load_fn, # Callable[[torch.Tensor, int], int] — (slot_mapping, prefix_pad) -> num_retrieved
) -> Optional[Tuple[torch.Tensor, TreeNode]]:
"""Alloc slots, run ``load_fn``, attach a TreeNode for what was loaded.
Returns ``(slots, new_node)`` on success, ``None`` if alloc fails
or the load returned zero (slots are freed in either case).
"""
chunk_size = self.lmcache_connector.chunk_size()
prefix_pad = value_numel % chunk_size
if self.token_to_kv_pool_allocator.available_size() < uncached_len:
self.evict(EvictParams(num_tokens=uncached_len))
token_slots = self.token_to_kv_pool_allocator.alloc(uncached_len)
if token_slots is None:
return None
slot_mapping = torch.empty(
value_numel + token_slots.numel(),
dtype=torch.int64,
device=self.device,
)
slot_mapping[:value_numel].fill_(-1)
slot_mapping[value_numel:].copy_(token_slots)
# Dispatch to the mode-specific loader (IP: start_load_kv, MP:
# retrieve_kv). Each loader manages its own load_stream context.
num_retrieved = load_fn(slot_mapping, prefix_pad)
logger.debug("num_retrieved_tokens: %s", num_retrieved)
if num_retrieved > 0:
self.token_to_kv_pool_allocator.free(
token_slots[(num_retrieved - prefix_pad) :]
)
else:
self.token_to_kv_pool_allocator.free(token_slots)
if num_retrieved > 0:
fetched = num_retrieved - prefix_pad
new_node = TreeNode(priority=last_node.priority)
start = value_numel
end = start + fetched
new_node.key = key[start:end]
new_node.value = token_slots[:fetched]
new_node.parent = last_node
last_node.children[new_node.key.child_key(self.page_size)] = new_node
self.evictable_size_ += fetched
self._update_leaf_status(last_node)
self._update_leaf_status(new_node)
self.kv_events.record_store(new_node.parent)
self.kv_events.record_store(new_node)
return token_slots[:fetched], new_node
return None
def _mp_load_back(
self,
*,
marker: _LMCacheLoadBackMarker,
request_id: str,
slot_mapping: torch.Tensor,
prefix_pad: int,
) -> int:
"""MP non-layerwise loader: fire ``retrieve_kv`` and wait for the
load_stream so the compute stream observes the writes.
"""
current_stream = torch.get_device_module(self.device).current_stream()
self.load_stream.wait_stream(current_stream)
with device_stream_context(self.load_stream):
n = self.lmcache_connector.retrieve_kv(
LoadMetadata(
token_ids=marker.key.token_ids,
slot_mapping=slot_mapping,
offset=marker.value_numel - prefix_pad,
prefix_pad=prefix_pad,
request_id=request_id,
)
)
current_stream.wait_stream(self.load_stream)
return n
def _ip_load_back(
self,
*,
token_ids: list[int],
value_numel: int,
slot_mapping: torch.Tensor,
prefix_pad: int,
) -> int:
"""IP layerwise loader: kick off ``start_load_kv`` on ``self.load_stream``.
``start_load_kv`` enqueues the first layer's transfer; the
``LayerTransferCounter`` hook drives the rest during forward.
"""
with device_stream_context(self.load_stream):
return self.lmcache_connector.start_load_kv(
LoadMetadata(
token_ids=token_ids,
slot_mapping=slot_mapping,
offset=value_numel - prefix_pad,
)
)
def on_release(self, req: Req, *, inserted: bool) -> None:
if not inserted and self._mode is LMCacheMode.MP:
self._mp_load_back_markers.pop(req.rid, None)
self.lmcache_connector.end_session(req.rid)
def cache_finished_req(self, req: Req, *, owned_kv_len: int) -> None:
"""On request completion, insert device KV into radix and store to LMCache."""
super().cache_finished_req(req, owned_kv_len=owned_kv_len)
topk = get_spec().speculative_eagle_topk
enable_kv_committed_len = topk is None or topk == 1
if enable_kv_committed_len:
kv_committed_len = req.kv.kv_committed_len
else:
kv_committed_len = len(req.origin_input_ids) + max(
len(req.output_ids) - 1, 0
)
token_ids = (req.origin_input_ids + req.output_ids)[:kv_committed_len]
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, :kv_committed_len
]
# Use super() to avoid a redundant LOOKUP — we only need new_last_node from radix.
match_result = super().match_prefix(
MatchPrefixParams(
key=RadixKey(
token_ids,
req.extra_key,
cache_salt=req.cache_salt,
)
)
)
new_last_node = match_result.last_device_node
assert new_last_node is not None
self.inc_lock_ref(new_last_node)
store_md = StoreMetadata(
last_node=new_last_node,
token_ids=token_ids,
kv_indices=kv_indices,
offset=0,
request_id=req.rid,
)
if self._mode is LMCacheMode.MP:
self.lmcache_connector.store_kv(store_md)
# MP store_kv blocks until the daemon's signal event fires, so the slots are safe to evict immediately.
self._mp_load_back_markers.pop(req.rid, None)
self.dec_lock_ref(new_last_node)
self.lmcache_connector.end_session(req.rid)
elif self._mode is LMCacheMode.IP:
with device_stream_context(self.store_stream):
self.lmcache_connector.store_kv(store_md)
# Layerwise store is async on store_stream; defer the unlock to evict()'s store_stream.synchronize().
with self._node_lock:
self._in_flight_nodes.append(new_last_node)
def evict(self, params: EvictParams) -> EvictResult:
"""Before base eviction, wait for any outstanding stores and release locks."""
if self.disable:
return EvictResult()
self.store_stream.synchronize()
with self._node_lock:
for node in self._in_flight_nodes:
self.dec_lock_ref(node)
self._in_flight_nodes.clear()
return super().evict(params)
def pretty_print(self):
super().pretty_print()
try:
logger.debug(
"evictable=%d protected=%d", self.evictable_size_, self.protected_size_
)
except Exception: # pragma: no cover
pass
@@ -0,0 +1,982 @@
"""Unified radix tree backed by the LMCache multiprocess service.
LMCache owns external storage; retrieved KV is published after reaching GPU.
"""
from __future__ import annotations
import atexit
import logging
from array import array
from typing import TYPE_CHECKING, Any, Optional
import torch
from lmcache.integration.sglang.lmcache_mp_metadata import (
LMCacheExternalFlow,
LMCachePendingStore,
)
from lmcache.integration.sglang.unified_lmcache_mp_connector import (
UnifiedLMCacheMPConnector,
)
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWAAllocatorBase,
)
from sglang.srt.mem_cache.base_prefix_cache import (
CacheRequestHandle,
EvictParams,
InitLoadBackParams,
InsertParams,
MatchPrefixParams,
MatchResult,
)
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.components import ComponentType
from sglang.srt.mem_cache.unified_radix_cache import NodeId, UnifiedRadixCache
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
logger = logging.getLogger(__name__)
class LMCacheUnifiedRadixCache(UnifiedRadixCache):
"""Unified radix tree with device-direct LMCache MP KV/state I/O."""
def __init__(
self,
params: CacheInitParams,
*,
model_config: ModelConfig,
tp_size: int,
tp_rank: int,
lmcache_config_file: Optional[str],
forward_stream: Any,
) -> None:
# The parent constructor calls reset() before the connector exists.
self.lmcache_connector: Optional[UnifiedLMCacheMPConnector] = None
super().__init__(params)
self._mamba_component = self._find_mamba_component()
self.lmcache_connector = UnifiedLMCacheMPConnector(
config_file=lmcache_config_file,
model_config=model_config,
tp_size=tp_size,
tp_rank=tp_rank,
tp_group=params.tp_cache_group,
pp_size=params.pp_size,
pp_rank=params.pp_rank,
pp_group=params.pp_cache_group,
page_size=self.page_size,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
req_to_token_pool=self.req_to_token_pool,
tree_components=self.tree_components,
mamba_component=self._mamba_component,
sliding_window_size=self._sliding_window_size,
)
self._external_flows: dict[str, LMCacheExternalFlow] = {}
self._forward_stream = forward_stream
self._pending_stores: list[LMCachePendingStore] = []
self._pending_store_counts: dict[str, int] = {}
self._session_finish_requested: set[str] = set()
self._lmcache_closed = False
atexit.register(self.shutdown)
def is_backuped(self, node_id: NodeId) -> bool:
# LMCache rebuilds lookup keys from tokens, so any L1 node is valid.
return True
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
"""Report LMCache hits through host fields until GPU slots are ready."""
requested_key_len = len(params.key)
req = params.req
flow = self._external_flows.get(req.rid) if req is not None else None
if (
flow is not None
and flow.total_hit is not None
and flow.local_hit_tokens is not None
and flow.load is None
):
# Use the shortest cross-rank L1 prefix for consistent admission.
params = MatchPrefixParams(
key=params.key[: flow.local_hit_tokens],
cow_mamba=params.cow_mamba,
req=req,
)
elif flow is not None and flow.load is not None and params.cow_mamba:
# Re-arm Mamba COW from the retrieved checkpoint on retry.
self._arm_external_mamba_cow(flow, req)
params = MatchPrefixParams(key=params.key, cow_mamba=False, req=req)
result = super().match_prefix(params)
if req is None:
return result
if flow is None or flow.total_hit is None or flow.cancelled:
return result
total_hit = min(flow.total_hit, requested_key_len, len(flow.key))
local_hit = len(result.device_indices)
skip = 0
if flow.load is not None and not flow.prefix_published:
# Track private slots shadowed by a concurrent tree insertion.
skip = max(
min(local_hit, total_hit) - flow.load.local_hit_tokens,
0,
)
flow.loaded_skip_tokens = max(flow.loaded_skip_tokens, skip)
if flow.load.result:
self._release_unused_loaded_slots(flow)
if total_hit <= local_hit:
return result
if flow.load is None:
external_hit = total_hit - local_hit
return result._replace(
last_host_node=result.last_device_node,
best_match_node=result.last_device_node,
host_hit_length=external_hit,
swa_host_hit_length=(
min(
external_hit,
self.lmcache_connector.aligned_swa_window_size(),
)
if self.is_swa_enabled
else 0
),
mamba_host_hit_length=(1 if self._mamba_component is not None else 0),
mamba_branching_seqlen=None,
full_kv_hit_length=max(result.full_kv_hit_length, total_hit),
)
# Reuse an already submitted H2D load on the next admission attempt.
suffix = flow.load.device_indices[skip : skip + max(total_hit - local_hit, 0)]
return result._replace(
device_indices=torch.cat([result.device_indices, suffix]),
last_host_node=result.last_device_node,
best_match_node=result.last_device_node,
host_hit_length=0,
swa_host_hit_length=0,
mamba_host_hit_length=0,
mamba_branching_seqlen=None,
full_kv_hit_length=max(result.full_kv_hit_length, total_hit),
)
def prefetch_from_storage(
self,
request: CacheRequestHandle,
last_host_node_id: NodeId,
new_input_tokens: list[int],
last_hash: Optional[str] = None,
prefix_keys: Optional[list[str]] = None,
matched_prefix_tokens: Optional[list[int]] = None,
extra_key: Optional[str] = None,
cache_salt: Optional[str] = None,
storage_hit_end: Optional[int] = None,
) -> None:
del last_hash, prefix_keys, storage_hit_end
req_id = request.rid
if req_id in self._external_flows:
return
local_tokens = list(matched_prefix_tokens or [])
token_ids = local_tokens + list(new_input_tokens)
anchor_extra_key, anchor_cache_salt = self.tree_core.prefetch_anchor_info(
last_host_node_id
)
extra_key = extra_key or anchor_extra_key
cache_salt = cache_salt or anchor_cache_salt
key = RadixKey(
array("q", token_ids),
extra_key=extra_key,
is_bigram=self.tree_core.is_eagle,
cache_salt=cache_salt,
).page_aligned(self.page_size)
if len(key) == 0:
return
token_ids = key.raw_token_ids()[: len(key)]
lookup = self.lmcache_connector.submit_lookup(
req_id,
token_ids,
local_hit_tokens=min(len(local_tokens), len(key)),
cache_salt=self.lmcache_connector.build_cache_salt(cache_salt, extra_key),
)
self._external_flows[req_id] = LMCacheExternalFlow(
key=key,
lookup=lookup,
)
def check_prefetch_progress(self, handle: CacheRequestHandle) -> bool:
req_id = handle.rid
flow = self._external_flows.get(req_id)
if flow is None:
return True
if flow.cancelled:
return False
if flow.total_hit is None:
total_hit = self.lmcache_connector.poll_lookup(flow.lookup)
if total_hit is None:
return False
latest = super().match_prefix(MatchPrefixParams(key=flow.key))
local_hit = torch.tensor(
[len(latest.device_indices)], dtype=torch.int64, device="cpu"
)
self.lmcache_connector.parallel_all_reduce(
local_hit, torch.distributed.ReduceOp.MIN
)
total_hit = min(total_hit, len(flow.key))
local_hit_tokens = int(local_hit.item())
release_end = min(
total_hit,
local_hit_tokens
// self.lmcache_connector.chunk_size
* self.lmcache_connector.chunk_size,
)
if release_end > 0:
# All ranks advance state; only the leader releases remote locks.
self.lmcache_connector.free_lookup_locks(
req_id, start=0, end=release_end
)
if total_hit <= local_hit_tokens:
self._external_flows.pop(req_id, None)
self.prefetch_loaded_tokens_by_reqid[handle] = 0
self.prefetch_loaded_storage_start_by_reqid.pop(handle, None)
return True
flow.total_hit = total_hit
flow.local_hit_tokens = local_hit_tokens
self.prefetch_loaded_tokens_by_reqid[handle] = total_hit - local_hit_tokens
self.prefetch_loaded_storage_start_by_reqid[handle] = local_hit_tokens
return True
def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None:
self._publish_external_loaded_prefix(req, token_ids_len=len(req.get_fill_ids()))
super().cache_unfinished_req(req, chunked=chunked, **kwargs)
self._retire_loaded_flow(req.rid)
self._submit_store(req, req.get_fill_ids())
def on_release(self, req: Req, *, inserted: bool) -> None:
super().on_release(req, inserted=inserted)
if not inserted:
self.release_aborted_request(req.cache_request_handle)
def cache_finished_req(self, req: Req, *, owned_kv_len: int, **kwargs) -> None:
self._publish_external_loaded_prefix(req, token_ids_len=owned_kv_len)
super().cache_finished_req(req, owned_kv_len=owned_kv_len, **kwargs)
self._retire_loaded_flow(req.rid)
token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len]
self._submit_store(req, token_ids)
self._request_session_finish(req.rid)
def check_hicache_events(self) -> None:
"""Poll LMCache retrieve/store futures at the scheduler safe point."""
load_flows = [
flow
for flow in self._external_flows.values()
if flow.load is not None and not flow.load_completed
]
ready_loads = self.lmcache_connector.ready_prefix_count(
[f.load for f in load_flows]
)
for flow in load_flows[:ready_loads]:
assert flow.load is not None
success = self.lmcache_connector.complete_load(flow.load)
flow.load_completed = True
cancelled = flow.cancelled
if cancelled or not success:
self._finish_failed_load(flow)
if not cancelled:
logger.warning(
"LMCache retrieve failed after admission for request %s",
flow.lookup.request_id,
)
elif flow.retire_requested:
self._finalize_retired_flow(flow)
else:
self._finish_successful_load(flow)
ready_stores = self.lmcache_connector.ready_prefix_count(
[pending.operation for pending in self._pending_stores]
)
for _ in range(ready_stores):
pending = self._pending_stores.pop(0)
rid = pending.operation.request_id
try:
self.lmcache_connector.complete_store(pending.operation)
finally:
# Always unlock the source node after store completion.
self.dec_lock_ref(pending.node_id, pending.lock_params)
remaining = self._pending_store_counts[rid] - 1
if remaining > 0:
self._pending_store_counts[rid] = remaining
else:
self._pending_store_counts.pop(rid)
self._finish_session_if_store_idle(rid)
def has_pending_cache_operations(self) -> bool:
return bool(self._external_flows or self._pending_stores)
def release_aborted_request(self, handle: CacheRequestHandle) -> None:
rid = handle.rid
self.prefetch_loaded_tokens_by_reqid.pop(handle, None)
self.prefetch_loaded_storage_start_by_reqid.pop(handle, None)
flow = self._external_flows.get(rid)
if flow is None:
self._request_session_finish(rid)
return
flow.cancelled = True
# Keep the checkpoint alive until H2D completes.
if flow.load is not None and flow.mamba_value is not None:
flow.free_mamba_after_load = True
if (
flow.load_req is not None
and flow.load_req.kv.mamba_cow_src_index is flow.mamba_value
):
flow.load_req.kv.mamba_cow_src_index = None
if flow.load is None:
if flow.total_hit is not None:
# No retrieve will release locks from this completed lookup.
self._retire_loaded_flow(rid)
else:
# END_SESSION follows the in-flight lookup and releases its locks.
self._external_flows.pop(rid, None)
elif flow.load_completed:
self._finish_failed_load(flow)
else:
# Complete the cross-rank load only in check_hicache_events().
flow.retire_requested = True
self._request_session_finish(rid)
def init_load_back(self, params: InitLoadBackParams) -> tuple[torch.Tensor, NodeId]:
req = params.req
if req is None:
return (
self.tree_core.empty_match_result.device_indices,
params.best_match_node,
)
flow = self._external_flows.get(req.rid)
if flow is None or flow.total_hit is None or flow.load is not None:
return (
self.tree_core.empty_match_result.device_indices,
params.best_match_node,
)
device_indices = self._start_external_load(flow, req)
if device_indices is None:
req.storage_hit_length = 0
req.host_hit_length = 0
req.swa_host_hit_length = 0
req.mamba_host_hit_length = 0
self.prefetch_loaded_tokens_by_reqid.pop(req.cache_request_handle, None)
self.prefetch_loaded_storage_start_by_reqid.pop(
req.cache_request_handle, None
)
# Retire the unloaded flow to release its lookup locks.
self._retire_loaded_flow(req.rid)
return (
self.tree_core.empty_match_result.device_indices,
params.best_match_node,
)
return device_indices, params.best_match_node
def ready_to_load_host_cache(self) -> int:
# H2D is submitted in init_load_back to preserve prefill fallback.
return -1
def supports_retraction_backup(self) -> bool:
# TODO(chunxiaozheng): implement retraction backup
return False
def clear_storage_backend(self) -> bool:
return self.lmcache_connector.clear()
def reset(self) -> None:
connector = self.lmcache_connector
if connector is not None:
for flow in list(self._external_flows.values()):
if flow.load is not None:
try:
flow.load.future.result(timeout=connector.operation_timeout)
except Exception:
pass
connector.complete_load(flow.load, synchronize=False)
flow.load_completed = True
if flow.prefix_published:
# Free only private slots; the tree owns the published suffix.
self._release_unused_loaded_slots(flow)
else:
self.token_to_kv_pool_allocator.free(
flow.load.device_indices[flow.released_skip_tokens :]
)
if flow.mamba_value is not None:
self.req_to_token_pool.mamba_allocator.free(flow.mamba_value)
flow.mamba_value = None
self._release_flow_anchor(flow)
for pending in list(self._pending_stores):
try:
pending.operation.future.result(timeout=connector.operation_timeout)
except Exception:
pass
connector.complete_store(pending.operation, synchronize=False)
self.dec_lock_ref(pending.node_id, pending.lock_params)
self._external_flows.clear()
self._pending_stores.clear()
self._pending_store_counts.clear()
self._session_finish_requested.clear()
self.prefetch_loaded_tokens_by_reqid.clear()
connector.end_all_sessions()
super().reset()
def shutdown(self) -> None:
if self._lmcache_closed:
return
self.reset()
self.lmcache_connector.close()
self._lmcache_closed = True
def release_host_resources(self) -> None:
self.shutdown()
def _find_mamba_component(self):
return next(
(
component
for component in self._components_tuple
if component.component_type is ComponentType.MAMBA
),
None,
)
def _allocate_external_mamba_slot(self) -> Optional[torch.Tensor]:
if self._mamba_component is None:
return None
allocator = self.req_to_token_pool.mamba_allocator
slot = allocator.alloc(1)
if slot is None:
self.evict(EvictParams(num_tokens=0, mamba_num=1))
slot = allocator.alloc(1)
if slot is not None:
physical = self.req_to_token_pool.translate_mamba_indices(slot)
self.lmcache_connector.reset_mamba_checkpoint_metadata(physical)
return slot
def _arm_external_mamba_cow(self, flow: LMCacheExternalFlow, req: Req) -> None:
"""Give a request a mutable state slot and COW from the loaded checkpoint."""
checkpoint = flow.mamba_value
if checkpoint is None:
return
if req.kv.mamba_pool_idx is None:
active = self._allocate_external_mamba_slot()
assert active is not None, "Cannot allocate Mamba request state for LMCache"
req.kv.mamba_pool_idx = active[0]
flow.request_mamba_value = active
flow.allocated_request_mamba_for_load = True
elif flow.request_mamba_value is None:
flow.request_mamba_value = req.kv.mamba_pool_idx.reshape(1)
flow.allocated_request_mamba_for_load = False
flow.load_req = req
req.kv.mamba_cow_src_index = checkpoint
req.kv.mamba_needs_clear = False
def _allocate_external_slots(self, num_tokens: int) -> Optional[torch.Tensor]:
allocator = self.token_to_kv_pool_allocator
if not self.is_swa_enabled:
if allocator.available_size() < num_tokens:
self.evict(EvictParams(num_tokens=num_tokens))
return allocator.alloc(num_tokens)
# Allocate SWA only for the trailing window; older pages use dummy page 0.
assert self._sliding_window_size is not None
swa_tail_tokens = min(
num_tokens,
self.lmcache_connector.aligned_swa_window_size(),
)
full_allocator = allocator.full_attn_allocator
swa_allocator = allocator.swa_attn_allocator
if full_allocator.available_size() < num_tokens:
self.evict(EvictParams(num_tokens=num_tokens))
if swa_allocator.available_size() < swa_tail_tokens:
self.evict(EvictParams(swa_num_tokens=swa_tail_tokens))
full_indices = full_allocator.alloc(num_tokens)
if full_indices is None:
return None
if swa_tail_tokens == 0:
return full_indices
tail_full_indices = full_indices[-swa_tail_tokens:]
if isinstance(allocator, UnifiedSWAAllocatorBase):
# Unified-memory FULL and SWA share virtual page IDs.
virtual_pages = torch.unique(tail_full_indices // self.page_size)
try:
allocator.swa_attn_allocator.alloc_with_virtual(virtual_pages)
except Exception:
full_allocator.free(full_indices)
logger.exception("Failed to allocate unified SWA slots for LMCache")
return None
else:
swa_indices = swa_allocator.alloc(swa_tail_tokens)
if swa_indices is None:
full_allocator.free(full_indices)
return None
allocator.set_full_to_swa_mapping(tail_full_indices, swa_indices)
return full_indices
def _start_external_load(
self, flow: LMCacheExternalFlow, req: Req
) -> Optional[torch.Tensor]:
assert flow.total_hit is not None
assert flow.local_hit_tokens is not None
local_hit = flow.local_hit_tokens
latest = super().match_prefix(MatchPrefixParams(key=flow.key[:local_hit]))
total_hit = min(flow.total_hit, len(flow.key))
if total_hit <= local_hit:
return self.tree_core.empty_match_result.device_indices
# Pin the common L1 boundary before allocation can trigger eviction.
flow.anchor_node = latest.last_device_node
flow.anchor_lock = self.inc_lock_ref(flow.anchor_node).to_dec_params()
num_tokens = total_hit - local_hit
device_indices = self._allocate_external_slots(num_tokens)
mamba_value = None
allocated_mamba_for_load = False
request_mamba_value = None
allocated_request_mamba_for_load = False
if device_indices is not None and self._mamba_component is not None:
# Keep the immutable checkpoint separate from mutable request state.
mamba_value = self._allocate_external_mamba_slot()
allocated_mamba_for_load = mamba_value is not None
if req.kv.mamba_pool_idx is None:
request_mamba_value = self._allocate_external_mamba_slot()
allocated_request_mamba_for_load = request_mamba_value is not None
else:
request_mamba_value = req.kv.mamba_pool_idx.reshape(1)
allocation_ok = torch.tensor(
[
int(
device_indices is not None
and (
self._mamba_component is None
or (mamba_value is not None and request_mamba_value is not None)
)
)
],
dtype=torch.int32,
device="cpu",
)
self.lmcache_connector.parallel_all_reduce(
allocation_ok, torch.distributed.ReduceOp.MIN
)
if not allocation_ok.item():
if device_indices is not None:
self.token_to_kv_pool_allocator.free(device_indices)
if allocated_mamba_for_load and mamba_value is not None:
self.req_to_token_pool.mamba_allocator.free(mamba_value)
if allocated_request_mamba_for_load and request_mamba_value is not None:
self.req_to_token_pool.mamba_allocator.free(request_mamba_value)
self._release_flow_anchor(flow)
logger.debug(
"LMCache retrieve declined for %s: a parallel rank cannot allocate "
"%d GPU slots",
flow.lookup.request_id,
num_tokens,
)
return None
assert device_indices is not None
flow.mamba_value = mamba_value
flow.request_mamba_value = request_mamba_value
flow.allocated_request_mamba_for_load = allocated_request_mamba_for_load
flow.load_req = req
flow.free_mamba_after_load = allocated_mamba_for_load
flow.loaded_skip_tokens = 0
if allocated_request_mamba_for_load:
assert request_mamba_value is not None
req.kv.mamba_pool_idx = request_mamba_value[0]
req.kv.mamba_needs_clear = False
try:
load_start = local_hit // self.lmcache_connector.chunk_size
load_start *= self.lmcache_connector.chunk_size
# Submit asynchronously while admission can still fall back to prefill.
flow.load = self.lmcache_connector.submit_load(
flow.lookup,
self.lmcache_connector.device_indices_by_group(
device_indices,
mamba_value=mamba_value,
mamba_transfer_tokens=total_hit - load_start,
),
local_hit_tokens=local_hit,
owned_device_indices=device_indices,
producer_stream=self._forward_stream,
)
if not self.lmcache_connector.prepare_load_on_stream(
flow.load, self._forward_stream
):
raise RuntimeError("LMCache server rejected the retrieve request")
if mamba_value is not None:
# Forward waits for H2D before deferred Mamba COW runs.
self._arm_external_mamba_cow(flow, req)
except Exception:
self.token_to_kv_pool_allocator.free(device_indices)
if allocated_mamba_for_load and mamba_value is not None:
self.req_to_token_pool.mamba_allocator.free(mamba_value)
if allocated_request_mamba_for_load and request_mamba_value is not None:
self.req_to_token_pool.mamba_allocator.free(request_mamba_value)
req.kv.mamba_pool_idx = None
if req.kv.mamba_cow_src_index is mamba_value:
req.kv.mamba_cow_src_index = None
flow.mamba_value = None
flow.request_mamba_value = None
flow.allocated_request_mamba_for_load = False
flow.load_req = None
flow.free_mamba_after_load = False
self._release_flow_anchor(flow)
logger.exception(
"LMCache retrieve submission failed for %s", flow.lookup.request_id
)
flow.load = None
return None
return device_indices[flow.loaded_skip_tokens :]
def _release_flow_anchor(self, flow: LMCacheExternalFlow) -> None:
assert (flow.anchor_node is None) == (flow.anchor_lock is None), (
"LMCache flow anchor node and lock must be set together"
)
if flow.anchor_node is not None:
assert flow.anchor_lock is not None
self.dec_lock_ref(flow.anchor_node, flow.anchor_lock)
flow.anchor_node = None
flow.anchor_lock = None
def _release_unused_loaded_slots(self, flow: LMCacheExternalFlow) -> None:
"""Free retrieved slots shadowed by a longer rank-local L1 prefix."""
assert flow.load is not None
release_end = min(flow.loaded_skip_tokens, len(flow.load.device_indices))
if release_end <= flow.released_skip_tokens:
return
self.token_to_kv_pool_allocator.free(
flow.load.device_indices[flow.released_skip_tokens : release_end]
)
flow.released_skip_tokens = release_end
def _finish_failed_load(self, flow: LMCacheExternalFlow) -> None:
assert flow.load is not None
handle = (
flow.load_req.cache_request_handle if flow.load_req is not None else None
)
if not flow.prefix_published:
self.token_to_kv_pool_allocator.free(
flow.load.device_indices[flow.released_skip_tokens :]
)
if flow.free_mamba_after_load and flow.mamba_value is not None:
self.req_to_token_pool.mamba_allocator.free(flow.mamba_value)
if (
flow.load_req is not None
and flow.load_req.kv.mamba_cow_src_index is flow.mamba_value
):
flow.load_req.kv.mamba_cow_src_index = None
flow.mamba_value = None
if (
not flow.cancelled
and flow.allocated_request_mamba_for_load
and flow.request_mamba_value is not None
and flow.load_req is not None
and flow.load_req.kv.mamba_pool_idx is not None
and int(flow.load_req.kv.mamba_pool_idx.item())
== int(flow.request_mamba_value[0].item())
):
self.req_to_token_pool.mamba_allocator.free(flow.request_mamba_value)
flow.load_req.kv.mamba_pool_idx = None
flow.request_mamba_value = None
flow.allocated_request_mamba_for_load = False
flow.load_req = None
flow.free_mamba_after_load = False
self._release_flow_anchor(flow)
rid = flow.lookup.request_id
self._external_flows.pop(rid, None)
if handle is not None:
if flow.cancelled:
self.prefetch_loaded_tokens_by_reqid.pop(handle, None)
else:
self.prefetch_loaded_tokens_by_reqid[handle] = 0
self.prefetch_loaded_storage_start_by_reqid.pop(handle, None)
def _finish_successful_load(self, flow: LMCacheExternalFlow) -> None:
"""Finish local bookkeeping after LMCache has completed the retrieve."""
assert flow.load is not None and flow.load_completed and flow.load.result
self._release_unused_loaded_slots(flow)
# Keep Mamba state until the cache callback publishes it.
def _finalize_retired_flow(self, flow: LMCacheExternalFlow) -> None:
"""Release a loaded flow after its cache callback retires it."""
assert flow.load is not None and flow.load_completed and flow.load.result
self._finish_successful_load(flow)
if flow.mamba_value is not None:
# Free checkpoints from request paths that retire without insertion.
if (
flow.load_req is not None
and flow.load_req.kv.mamba_cow_src_index is flow.mamba_value
):
flow.load_req.kv.mamba_cow_src_index = None
self.req_to_token_pool.mamba_allocator.free(flow.mamba_value)
flow.mamba_value = None
flow.free_mamba_after_load = False
flow.request_mamba_value = None
flow.allocated_request_mamba_for_load = False
flow.load_req = None
self._release_flow_anchor(flow)
rid = flow.lookup.request_id
self._external_flows.pop(rid, None)
def _retire_loaded_flow(self, rid: str) -> None:
"""Mark a load for retirement without entering a collective."""
flow = self._external_flows.get(rid)
if flow is None:
return
if flow.load is None:
if flow.total_hit is None:
return
self.lmcache_connector.free_lookup_locks(
rid,
start=flow.lookup.lock_start,
end=flow.total_hit,
)
self._external_flows.pop(rid, None)
return
flow.retire_requested = True
if not flow.load_completed:
return
if flow.load.result:
self._finalize_retired_flow(flow)
else:
cancelled = flow.cancelled
self._finish_failed_load(flow)
if not cancelled:
logger.warning(
"LMCache retrieve failed after admission for request %s", rid
)
def _prepare_external_slots_for_insert(self, req: Req) -> None:
"""Restore tree ownership and mark the missing SWA prefix as evicted."""
flow = self._external_flows.get(req.rid)
if flow is None or flow.load is None:
return
tree_owned_len = flow.load.local_hit_tokens + flow.loaded_skip_tokens
req.kv.cache_protected_len = min(req.kv.cache_protected_len, tree_owned_len)
if self.is_swa_enabled:
external_tokens = len(flow.load.device_indices)
swa_missing_end = flow.load.local_hit_tokens + max(
external_tokens - self.lmcache_connector.aligned_swa_window_size(),
0,
)
req.kv.swa_evicted_seqlen = max(req.kv.swa_evicted_seqlen, swa_missing_end)
def _publish_external_loaded_prefix(self, req: Req, *, token_ids_len: int) -> None:
"""Publish retrieved KV and immutable Mamba state into the device tree."""
flow = self._external_flows.get(req.rid)
if (
flow is None
or flow.load is None
or flow.total_hit is None
or flow.prefix_published
):
return
# Restore the ownership boundary once before the parent cache callback.
self._prepare_external_slots_for_insert(req)
if flow.mamba_value is None:
# The parent callback will adopt the retrieved Full/SWA slots.
flow.prefix_published = True
return
total_hit = min(flow.total_hit, len(flow.key))
assert total_hit > 0, "LMCache loaded prefix must contain at least one token"
if total_hit > token_ids_len:
raise RuntimeError(
"LMCache Mamba checkpoint lies beyond the request KV boundary: "
f"{total_hit}>{token_ids_len} for request {req.rid}"
)
prev_prefix_len = min(req.kv.cache_protected_len, total_hit)
kv_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, :token_ids_len
]
key = flow.key[:total_hit]
checkpoint = flow.mamba_value
result = self.insert(
InsertParams(
key=key,
value=kv_indices[:total_hit].to(dtype=torch.int64, copy=True),
mamba_value=checkpoint,
prev_prefix_len=prev_prefix_len,
swa_evicted_seqlen=req.kv.swa_evicted_seqlen,
chunked=True,
priority=req.priority or 0,
)
)
# Free our checkpoint if a concurrent insertion already owns this boundary.
if result.mamba_exist:
self.req_to_token_pool.mamba_allocator.free(checkpoint)
flow.mamba_value = None
flow.free_mamba_after_load = False
matched = super().match_prefix(
MatchPrefixParams(key=key, req=req, cow_mamba=False)
)
if len(matched.device_indices) < total_hit:
raise RuntimeError(
"LMCache loaded prefix was inserted but is not reusable across "
f"all UnifiedRadixCache components: {len(matched.device_indices)}"
f"/{total_hit} tokens for request {req.rid}"
)
canonical = matched.device_indices[:total_hit]
self.req_to_token_pool.write(
(req.kv.req_pool_idx, slice(prev_prefix_len, total_hit)),
canonical[prev_prefix_len:],
)
# Move the request lock to the published boundary.
if req.last_node is not None:
self._dec_req_lock(req)
lock_result = self.inc_lock_ref(matched.last_device_node)
if total_hit < token_ids_len:
req.prefix_indices = torch.cat(
[canonical, kv_indices[total_hit:].to(dtype=torch.int64, copy=True)]
)
else:
req.prefix_indices = canonical
req.kv.cache_protected_len = total_hit
req.last_node = matched.last_device_node
req.lock_receipt = lock_result.to_dec_params()
req.swa_prefix_lock_released = False
flow.request_mamba_value = None
flow.allocated_request_mamba_for_load = False
flow.load_req = None
flow.prefix_published = True
def _submit_store(self, req: Req, token_ids: list[int]) -> None:
key = RadixKey(
token_ids,
req.extra_key,
is_bigram=self.tree_core.is_eagle,
cache_salt=req.cache_salt,
).page_aligned(self.page_size)
aligned_len = (
len(key)
// self.lmcache_connector.chunk_size
* self.lmcache_connector.chunk_size
)
key = key[:aligned_len]
if len(key) == 0:
return
matched = super().match_prefix(MatchPrefixParams(key=key))
# Store the currently resident SWA prefix; later callbacks may extend it.
resident_len = torch.tensor(
[min(len(matched.device_indices), len(key))],
dtype=torch.int64,
device="cpu",
)
self.lmcache_connector.parallel_all_reduce(
resident_len, torch.distributed.ReduceOp.MIN
)
resident_len = (
int(resident_len.item())
// self.lmcache_connector.chunk_size
* self.lmcache_connector.chunk_size
)
store_start = self.lmcache_connector.get_store_start(req.rid, len(key))
if resident_len <= store_start:
logger.debug(
"LMCache store skipped for %s: no new resident chunk "
"after token %d (%d/%d tokens)",
req.rid,
store_start,
resident_len,
len(key),
)
return
if resident_len < len(key):
logger.debug(
"LMCache store for %s is limited to the common resident prefix: "
"%d/%d tokens",
req.rid,
resident_len,
len(key),
)
key = key[:resident_len]
# Re-match so the lock and Mamba state use the shortened boundary.
matched = super().match_prefix(MatchPrefixParams(key=key))
store_indices = matched.device_indices[store_start : len(key)]
if len(store_indices) != len(key) - store_start:
logger.debug(
"LMCache store skipped for %s: store indices length: %d, "
"store start: %d, key length: %d",
req.rid,
len(store_indices),
store_start,
len(key),
)
return
lock_params = self.inc_lock_ref(matched.last_device_node).to_dec_params()
mamba_value = (
self.tree_core.get_component_device_value(
matched.best_match_node, ComponentType.MAMBA
)
if self._mamba_component is not None
else None
)
mamba_is_resident = torch.tensor(
[int(self._mamba_component is None or mamba_value is not None)],
dtype=torch.int32,
device="cpu",
)
self.lmcache_connector.parallel_all_reduce(
mamba_is_resident, torch.distributed.ReduceOp.MIN
)
if not mamba_is_resident.item():
self.dec_lock_ref(matched.last_device_node, lock_params)
logger.debug(
"LMCache store skipped for %s: no Mamba checkpoint at token %d",
req.rid,
len(key),
)
return
try:
operation = self.lmcache_connector.submit_store(
req.rid,
key.raw_token_ids()[: len(key)],
self.lmcache_connector.device_indices_by_group(
store_indices,
mamba_value=mamba_value,
),
device_indices_start=store_start,
cache_salt=self.lmcache_connector.build_cache_salt(
req.cache_salt, req.extra_key
),
)
except Exception:
self.dec_lock_ref(matched.last_device_node, lock_params)
logger.exception("LMCache store submission failed for %s", req.rid)
return
if operation is None:
self.dec_lock_ref(matched.last_device_node, lock_params)
return
self._pending_stores.append(
LMCachePendingStore(operation, matched.last_device_node, lock_params)
)
self._pending_store_counts[req.rid] = (
self._pending_store_counts.get(req.rid, 0) + 1
)
def _request_session_finish(self, rid: str) -> None:
"""End the LMCache session after this request's stores have completed."""
self._session_finish_requested.add(rid)
self._finish_session_if_store_idle(rid)
def _finish_session_if_store_idle(self, rid: str) -> None:
if (
rid not in self._session_finish_requested
or self._pending_store_counts.get(rid, 0) > 0
):
return
self._session_finish_requested.remove(rid)
self.lmcache_connector.finish_request(rid)
@@ -1,118 +0,0 @@
try:
from lmcache.integration.sglang.sglang_adapter import (
LMCacheLayerwiseConnector,
LoadMetadata,
StoreMetadata,
)
except ImportError:
raise RuntimeError(
"LMCache is not installed. Please install it by running `pip install lmcache` in the root directory of LMCache"
)
import torch
from sglang.srt.configs.model_config import ModelConfig
def test_load_store_metadata():
model_config = ModelConfig(
model_path="Qwen/Qwen3-4B",
)
# Generate Dummy KV Cache
head_num = model_config.num_key_value_heads
head_dim = model_config.head_dim
layer_num = model_config.num_hidden_layers
buffer_size = 256
input_id_len = 16
k_buffer = [
torch.randn(buffer_size, head_num, head_dim, dtype=torch.bfloat16).cuda()
for _ in range(layer_num)
]
v_buffer = [
torch.randn(buffer_size, head_num, head_dim, dtype=torch.bfloat16).cuda()
for _ in range(layer_num)
]
connector = LMCacheLayerwiseConnector(
model_config, 1, 0, k_buffer, v_buffer, config_file="example_config_ip.yaml"
)
fake_token_ids = torch.randint(0, model_config.vocab_size, (input_id_len,)).tolist()
fake_kv_indices = torch.randint(0, buffer_size, (input_id_len,))
offset = 0
store_metadata = StoreMetadata(
last_node=None,
token_ids=fake_token_ids,
kv_indices=fake_kv_indices,
offset=offset,
)
load_metadata = LoadMetadata(
token_ids=fake_token_ids,
slot_mapping=fake_kv_indices,
offset=offset,
)
current_stream = torch.cuda.current_stream()
retrieve_token_num = connector.start_load_kv(load_metadata)
assert retrieve_token_num == 0
connector.store_kv(store_metadata)
current_stream.synchronize()
# check retrieve
gt_key_buffer = [
torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda()
for _ in range(layer_num)
]
gt_value_buffer = [
torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda()
for _ in range(layer_num)
]
for i in range(layer_num):
gt_key_buffer[i] = k_buffer[i][fake_kv_indices]
gt_value_buffer[i] = v_buffer[i][fake_kv_indices]
# clear the k_buffer and v_buffer
for _ in range(layer_num):
k_buffer[i].zero_()
v_buffer[i].zero_()
retrieve_token_num = connector.start_load_kv(load_metadata)
assert retrieve_token_num == input_id_len
for i in range(layer_num):
current_stream.synchronize()
connector.load_kv_layerwise(i)
current_stream.synchronize()
test_key_buffer = [
torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda()
for _ in range(layer_num)
]
test_value_buffer = [
torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda()
for _ in range(layer_num)
]
for i in range(layer_num):
test_key_buffer[i] = k_buffer[i][fake_kv_indices]
test_value_buffer[i] = v_buffer[i][fake_kv_indices]
for i in range(layer_num):
assert torch.allclose(test_key_buffer[i], gt_key_buffer[i])
assert torch.allclose(test_value_buffer[i], gt_value_buffer[i])
print("================================================")
print("TEST_LOAD_STORE_METADATA PASSED!")
print("================================================")
connector.close()
if __name__ == "__main__":
test_load_store_metadata()