diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 8ac8e63130..5db8194b9d 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -430,6 +430,49 @@ def handle_cache_compatibility(server_args: Any) -> None: if prefix_tails is not None and prefix_tails < 0: raise ValueError("--swa-prefix-tails should be a non-negative integer.") + if cfg.enable_lmcache: + if cfg.enable_hierarchical_cache: + raise ValueError( + "--enable-lmcache and --enable-hierarchical-cache are " + "mutually exclusive" + ) + if cfg.enable_unified_cache_external_linker: + raise ValueError( + "--enable-lmcache and --enable-unified-cache-external-linker " + "are mutually exclusive" + ) + if cfg.disable_radix_cache: + raise ValueError("--enable-lmcache requires radix cache to be enabled") + + from sglang.srt.speculative.spec_info import SpeculativeAlgorithm + + if SpeculativeAlgorithm.from_string(cfg.speculative_algorithm).is_speculative(): + raise NotImplementedError( + "LMCacheUnifiedRadixCache does not yet support speculative decoding" + ) + if cfg.enable_dp_attention: + raise NotImplementedError( + "LMCacheUnifiedRadixCache does not yet support DP attention" + ) + if cfg.dcp_size > 1: + raise NotImplementedError( + "--enable-lmcache with --dcp-size > 1 is not supported: " + "LMCache has no DCP-aware index translation" + ) + if cfg.enable_streaming_session: + raise NotImplementedError( + "LMCacheUnifiedRadixCache does not yet support streaming sessions" + ) + if cfg.hicache_host_memory_mode == "buffer_only": + raise ValueError( + "--hicache-host-memory-mode=buffer_only is a HiCache-only mode" + ) + if cfg.disaggregation_mode != "null": + raise NotImplementedError( + "LMCacheUnifiedRadixCache currently supports colocated " + "prefill/decode scheduling only" + ) + def handle_unified_memory_pool(server_args: Any) -> None: diff --git a/python/sglang/srt/arg_groups/mamba_hook.py b/python/sglang/srt/arg_groups/mamba_hook.py index 3310bf039a..425e1a9421 100644 --- a/python/sglang/srt/arg_groups/mamba_hook.py +++ b/python/sglang/srt/arg_groups/mamba_hook.py @@ -91,6 +91,11 @@ def handle_int8_mamba_checkpoint(server_args: Any): f"radix cache; --radix-cache-backend={cfg.radix_cache_backend!r} " "is not int8-aware. Omit --radix-cache-backend." ) + if cfg.enable_lmcache: + raise ValueError( + "--enable-int8-mamba-checkpoint is not supported together with " + "--enable-lmcache: LMCache is not int8-aware. Disable one of them." + ) def validate_mamba_extra_buffer(view, model_arch: str, *, mamba_cache_chunk_size_of): diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index f74ce7307d..497d49c72f 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -496,6 +496,7 @@ class Scheduler( ) self.page_size = get_schedule().page_size self.enable_hierarchical_cache = get_memory().enable_hierarchical_cache + self.enable_lmcache = get_memory().enable_lmcache self.enable_session_radix_cache = get_memory().enable_session_radix_cache self.enable_hicache_storage = get_memory().hicache_storage_backend is not None self.enable_unified_cache_external_linker = ( @@ -2456,7 +2457,9 @@ class Scheduler( is_generation=self.is_generation, spec_algorithm=self.spec_algorithm, disaggregation_mode=self.disaggregation_mode, - enable_hicache_storage=lambda: self.enable_hicache_storage, + enable_hicache_storage=lambda: ( + self.enable_hicache_storage or self.enable_lmcache + ), rust_server=self.rust_server, ) @@ -3108,7 +3111,7 @@ class Scheduler( self.handle_generate_request(tokenized_req) def _prefetch_kvcache(self, req: Req, storage_hit_end: Optional[int] = None): - if self.enable_hicache_storage: + if self.enable_hicache_storage or self.enable_lmcache: req.init_next_round_input(self.tree_cache, cow_mamba=False) tree_cache = self.tree_cache buffer_mode = get_memory().hicache_host_memory_mode == "buffer_only" @@ -3612,6 +3615,7 @@ class Scheduler( self.enable_hierarchical_cache or get_memory().enable_flexkv or self.enable_unified_cache_external_linker + or self.enable_lmcache ): self.tree_cache.check_hicache_events() if self.enable_hicache_storage: @@ -3966,7 +3970,7 @@ class Scheduler( ): break - if self.enable_hicache_storage: + if self.enable_hicache_storage or self.enable_lmcache: prefetch_done = self.tree_cache.check_prefetch_progress( req.cache_request_handle ) @@ -4009,6 +4013,7 @@ class Scheduler( if res == AddReqResult.NO_TOKEN: if ( self.enable_hierarchical_cache + or self.enable_lmcache or self.enable_unified_cache_external_linker ): # Set batch_is_full after making sure there are requests that can be served @@ -4879,12 +4884,12 @@ class Scheduler( return self.external_corpus_manager.list(recv_req) def clear_hicache_storage_wrapped(self, recv_req: ClearHiCacheReqInput): - if self.enable_hierarchical_cache: + if self.enable_hierarchical_cache or self.enable_lmcache: self.tree_cache.clear_storage_backend() - logger.info("Hierarchical cache cleared successfully!") + logger.info("Hierarchical cache or LMCache cleared successfully!") if_success = True else: - logging.warning("Hierarchical cache is not enabled.") + logging.warning("Hierarchical cache or LMCache is not enabled.") if_success = False return ClearHiCacheReqOutput(success=if_success) @@ -4914,6 +4919,7 @@ class Scheduler( if ( self.enable_hicache_storage or self.disaggregation_mode != DisaggregationMode.NULL + or self.enable_lmcache ): # Storage and transfer workers need the GIL between I/O calls. # Singleton PD polls no longer yield through a collective. @@ -5044,6 +5050,8 @@ class Scheduler( # storage writes still hold host staging # (buffer-mode unified tree only). idle &= tc.buffer_pipeline.is_idle() + elif self.enable_lmcache: + idle &= not self.tree_cache.has_pending_cache_operations() return idle diff --git a/python/sglang/srt/managers/scheduler_components/output_streamer.py b/python/sglang/srt/managers/scheduler_components/output_streamer.py index 89cf4e53d1..752d456b9b 100644 --- a/python/sglang/srt/managers/scheduler_components/output_streamer.py +++ b/python/sglang/srt/managers/scheduler_components/output_streamer.py @@ -79,6 +79,8 @@ class SchedulerOutputStreamer: storage_backend = cache_controller.storage_backend if storage_backend is not None: storage_backend_type = type(storage_backend).__name__ + elif self.server_args.enable_lmcache: + storage_backend_type = "LMCache" return storage_backend_type def get_cached_tokens_details(self, req: Req) -> Optional[CachedTokensDetails]: diff --git a/python/sglang/srt/mem_cache/registry.py b/python/sglang/srt/mem_cache/registry.py index 6c3655f180..732e24d358 100644 --- a/python/sglang/srt/mem_cache/registry.py +++ b/python/sglang/srt/mem_cache/registry.py @@ -100,6 +100,29 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: return SWAChunkCache(params) + if get_memory().enable_lmcache: + from sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache import ( + LMCacheUnifiedRadixCache, + ) + from sglang.srt.mem_cache.unified_cache.components import ComponentType + + tree_components = [] + if not (ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0): + tree_components.append(ComponentType.FULL) + if ctx.is_hybrid_swa: + tree_components.append(ComponentType.SWA) + if ctx.is_hybrid_ssm: + tree_components.append(ComponentType.MAMBA) + params.tree_components = tuple(tree_components) + return LMCacheUnifiedRadixCache( + params, + model_config=ctx.model_config, + tp_size=ctx.tp_size, + tp_rank=ctx.tp_rank, + lmcache_config_file=get_memory().lmcache_config_file, + forward_stream=ctx.tp_worker.model_runner.forward_stream, + ) + if get_memory().enable_unified_cache_external_linker: return _create_unified_radix_cache(ctx, server_args, params) @@ -108,19 +131,6 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: return PureSWARadixCache(params=params) - if get_memory().enable_lmcache: - from sglang.srt.mem_cache.storage.lmcache.lmc_radix_cache import ( - LMCRadixCache, - ) - - return LMCRadixCache( - params=params, - model_config=ctx.model_config, - tp_size=ctx.tp_size, - rank=ctx.tp_rank, - tp_group=ctx.tp_group, - ) - if get_memory().enable_flexkv: # Importing the package side-effect registers the explicit # ``--radix-cache-backend=flexkv`` factory; we then call the diff --git a/python/sglang/srt/mem_cache/storage/flexkv/README.md b/python/sglang/srt/mem_cache/storage/flexkv/README.md index b3724540ca..23e0daa76f 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/README.md +++ b/python/sglang/srt/mem_cache/storage/flexkv/README.md @@ -2,9 +2,8 @@ A `RadixCache` subclass that routes sglang's host-tier KV cache through a FlexKV [`KVManager`](https://github.com/taco-project/FlexKV) (CPU / SSD / -Remote offload). Same integration pattern as -[`LMCRadixCache`](../lmcache/README.md): `FlexKVRadixCache` overrides -`match_prefix` / `init_load_back` / `cache_finished_req` / `evict`; a +Remote offload). `FlexKVRadixCache` overrides `match_prefix` / +`init_load_back` / `cache_finished_req` / `evict`; a `FlexKVConnector` façade talks to `KVManager`, `KVTPClient`, and a 3-axis (PP × CP × TP) sync context. diff --git a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py index fdfd2732d9..2b770987f6 100644 --- a/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py +++ b/python/sglang/srt/mem_cache/storage/flexkv/flexkv_radix_cache.py @@ -3,8 +3,8 @@ This module exposes :class:`FlexKVRadixCache`, a subclass of :class:`sglang.srt.mem_cache.radix_cache.RadixCache` that delegates host-side prefix storage to a FlexKV ``KVManager``. The design mirrors -``LMCRadixCache`` (the LMCache integration) so the scheduler-side -contract is identical: +the two-phase external-cache integration pattern, so the scheduler-side +contract is: * MP (synchronous) mode — the default. ``match_prefix`` fires only a FlexKV LOOKUP and returns ``host_hit_length``; @@ -118,7 +118,7 @@ class FlexKVRadixCache(RadixCache): # forward layer blocks on its own eventfd. self.flexkv_connector.register_layer_transfer_counter(kvcache) - # CUDA streams (mirroring LMCRadixCache). + # CUDA streams. self.load_stream = torch.cuda.Stream() self.store_stream = torch.cuda.Stream() @@ -395,7 +395,7 @@ class FlexKVRadixCache(RadixCache): """Base cache_finished_req then fire an async FlexKV store.""" super().cache_finished_req(req, owned_kv_len=owned_kv_len) - # Compute the committed prefix mirroring LMCRadixCache's logic. + # Compute the committed prefix. topk = get_spec().speculative_eagle_topk enable_kv_committed_len = topk is None or topk == 1 if enable_kv_committed_len: diff --git a/python/sglang/srt/mem_cache/storage/lmcache/README.md b/python/sglang/srt/mem_cache/storage/lmcache/README.md index f23954d973..da1571fd10 100644 --- a/python/sglang/srt/mem_cache/storage/lmcache/README.md +++ b/python/sglang/srt/mem_cache/storage/lmcache/README.md @@ -29,11 +29,9 @@ pip install -e . --no-build-isolation ## Use LMCache -LMCache supports two transport modes. **MP (multi-process, default)** issues a single blocking retrieve over ZMQ to a standalone daemon that owns the KV store and survives SGLang restarts. **IP (in-process)** uses an embedded layerwise connector — the cache lives and dies with the SGLang process. Mode selection is currently a code-level setting in `LMCRadixCache.__init__` (`self._mode`); only MP is reachable by default. - -### MP mode (default): multi-process daemon - -Uses `LMCacheMPConnector`. Daemon host/port come from the LMCache YAML config (`mp_host`, `mp_port`). +SGLang uses `LMCacheUnifiedRadixCache` with LMCache's multiprocess connector. +The standalone LMCache daemon owns the external cache and can survive SGLang +process restarts. Daemon host and port come from the LMCache YAML config. Terminal 1 — start the LMCache daemon: @@ -56,16 +54,3 @@ python -m sglang.launch_server \ ``` For full LMCache config options see https://docs.lmcache.ai/api_reference/configurations.html. - -### IP mode: in-process - -Uses `LMCacheLayerwiseConnector`. KV transfer happens per layer inside the SGLang process; the cache lives and dies with the server. To enable, edit `LMCRadixCache.__init__` and set `self._mode = LMCacheMode.IP`. - -The LMCache config still controls chunk_size and storage; `mp_host` / `mp_port` are ignored on this path. Use the bundled `example_config_ip.yaml`: - -```bash -python -m sglang.launch_server \ - --model-path MODEL \ - --enable-lmcache \ - --lmcache-config-file example_config_ip.yaml -``` diff --git a/python/sglang/srt/mem_cache/storage/lmcache/__init__.py b/python/sglang/srt/mem_cache/storage/lmcache/__init__.py new file mode 100644 index 0000000000..01006ad70f --- /dev/null +++ b/python/sglang/srt/mem_cache/storage/lmcache/__init__.py @@ -0,0 +1 @@ +"""LMCache-backed KV cache integrations.""" diff --git a/python/sglang/srt/mem_cache/storage/lmcache/example_config_ip.yaml b/python/sglang/srt/mem_cache/storage/lmcache/example_config_ip.yaml deleted file mode 100644 index 549110b7cd..0000000000 --- a/python/sglang/srt/mem_cache/storage/lmcache/example_config_ip.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Basic configurations -chunk_size: 256 - -# CPU offloading configurations -local_cpu: true -use_layerwise: true -max_local_cpu_size: 10 # number of CPU backend GB diff --git a/python/sglang/srt/mem_cache/storage/lmcache/lmc_radix_cache.py b/python/sglang/srt/mem_cache/storage/lmcache/lmc_radix_cache.py deleted file mode 100644 index 9c2b55c2f5..0000000000 --- a/python/sglang/srt/mem_cache/storage/lmcache/lmc_radix_cache.py +++ /dev/null @@ -1,519 +0,0 @@ -from __future__ import annotations - -import enum -import logging -import threading -from dataclasses import dataclass -from typing import TYPE_CHECKING, Optional, Tuple - -import torch - -from sglang.srt.mem_cache.base_prefix_cache import ( - EvictParams, - EvictResult, - InitLoadBackParams, - MatchPrefixParams, - MatchResult, -) -from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode -from sglang.srt.runtime_context import get_memory, get_spec -from sglang.srt.utils import create_device_stream, device_stream_context - -try: - from lmcache.integration.sglang.multi_process_adapter import LMCacheMPConnector - from lmcache.integration.sglang.sglang_adapter import ( - LMCacheLayerwiseConnector, - LoadMetadata, - StoreMetadata, - ) - from lmcache.integration.sglang.utils import lmcache_get_config -except ImportError as e: - raise RuntimeError( - "LMCache is not installed. Please install it by running `pip install lmcache`" - ) from e - - -if TYPE_CHECKING: - from sglang.srt.configs.model_config import ModelConfig - from sglang.srt.managers.schedule_batch import Req - from sglang.srt.mem_cache.cache_init_params import CacheInitParams - -logger = logging.getLogger(__name__) - - -@dataclass -class _LMCacheLoadBackMarker: - """Carries the data ``init_load_back`` needs from the - ``match_prefix`` call in MP mode. - """ - - key: RadixKey # detached snapshot of the matched key (the live query key - # aliases the req's growing fill_ids and must not be retained) - value_numel: int # number of tokens already in radix at match time - - -class LMCacheMode(enum.Enum): - MP = enum.auto() # multi-process mode - IP = enum.auto() # in-process mode - - -class LayerTransferCounter: - """Minimal adapter that lets the memory pool notify LMCache per-layer. - - The KV pool calls `wait_until(layer_id)` after finishing a layer, which we - translate into a `load_kv_layerwise(layer_id)` call on the LMCache connector - within the provided device stream. - """ - - def __init__( - self, - num_layers: int, - load_stream: torch.Stream, - lmc_connector: LMCacheLayerwiseConnector, - printable: bool = False, - ): - self.num_layers = num_layers - self.load_stream = load_stream - self.lmc_connector = lmc_connector - - def wait_until(self, layer_id: int): - # Ensure ordering of the async loads wrt compute stream(s). - self.load_stream.synchronize() - with device_stream_context(self.load_stream): - self.lmc_connector.load_kv_layerwise(layer_id) - - -class LMCRadixCache(RadixCache): - """RadixCache + LMCache IO. - - IP mode keeps the existing layerwise connector and - its per-layer transfer hook: ``match_prefix`` kicks off the load via - ``start_load_kv`` and SGLang's per-layer KV-pool hook drives subsequent - layers during forward. - - MP mode uses ``LMCacheMPConnector`` with a two-phase - load: ``match_prefix`` fires LOOKUP only (``connector.lookup_kv``) and - returns ``host_hit_length`` on the ``MatchResult``; the SGLang - scheduler then calls `init_load_back` at dispatch time, - which fires the actual RETRIEVE (``connector.retrieve_kv``) into - pre-allocated GPU slots. - """ - - def __init__( - self, - params: CacheInitParams, - model_config: Optional[ModelConfig] = None, - tp_size: int = 1, - rank: int = 0, - tp_group: Optional[torch.distributed.ProcessGroup] = None, - ): - super().__init__(params) - - cli_lmc_cfg = get_memory().lmcache_config_file or "" - - kvcache = self.token_to_kv_pool_allocator.get_kvcache() - connector_kwargs = dict( - sgl_config=model_config, - tp_size=tp_size, - rank=rank, - # NOTE: The original implementation accessed private buffers via - # `_kvcache.k_buffer` / `.v_buffer`. We prefer public accessors when - # available; fall back to private fields if needed. - k_pool=getattr( - kvcache, - "k_buffer", - getattr(self.token_to_kv_pool_allocator._kvcache, "k_buffer"), - ), - v_pool=getattr( - kvcache, - "v_buffer", - getattr(self.token_to_kv_pool_allocator._kvcache, "v_buffer"), - ), - tp_group=tp_group.device_group if tp_group is not None else None, - ) - - self.load_stream = create_device_stream(self.device) - self.store_stream = create_device_stream(self.device) - - # MP (multi-process) is the default. XPU defaults to IP (in-process - # layerwise) because the MP connector shares the KV cache via CUDA IPC - # (``Tensor._share_cuda_``), which is unavailable on XPU. - self._mode = LMCacheMode.IP if self.device.type == "xpu" else LMCacheMode.MP - if self._mode is LMCacheMode.MP: - if not cli_lmc_cfg: - raise ValueError( - "MP mode requires --lmcache-config-file (the YAML " - "supplies mp_host / mp_port)." - ) - lm_cfg = lmcache_get_config(cli_lmc_cfg) - self.lmcache_connector = LMCacheMPConnector( - page_size=params.page_size, - host=lm_cfg.mp_host, - port=lm_cfg.mp_port, - **connector_kwargs, - ) - elif self._mode is LMCacheMode.IP: - self.lmcache_connector = LMCacheLayerwiseConnector( - config_file=cli_lmc_cfg, **connector_kwargs - ) - # Per-layer hook - self.layer_done_executor = LayerTransferCounter( - num_layers=( - model_config.num_hidden_layers if model_config is not None else 0 - ), - load_stream=self.load_stream, - lmc_connector=self.lmcache_connector, - ) - kvcache.register_layer_transfer_counter(self.layer_done_executor) - - self._in_flight_nodes: list[TreeNode] = [] - self._node_lock = threading.Lock() - self._mp_load_back_markers: dict[str, _LMCacheLoadBackMarker] = {} - - def reset(self): - super().reset() - if hasattr(self, "_in_flight_nodes"): - with self._node_lock: - self._in_flight_nodes.clear() - if hasattr(self, "_mp_load_back_markers"): - self._mp_load_back_markers.clear() - - def match_prefix(self, params: MatchPrefixParams) -> MatchResult: - """Dispatch to the mode-specific match_prefix. - - MP mode → ``_mp_match_prefix`` (fires LOOKUP only). - IP mode → ``_ip_match_prefix`` (single-shot ``start_load_kv`` - plus per-layer hook). - """ - key = params.key - if self.disable or not key: - return super().match_prefix(params) - - if self.page_size != 1: - aligned_len = len(key) // self.page_size * self.page_size - key = key[:aligned_len] - - base_res = super().match_prefix(params) - value: torch.Tensor = base_res.device_indices - last_node: TreeNode = base_res.last_device_node - - if self._mode is LMCacheMode.MP: - if params.req is None: - return base_res - return self._mp_match_prefix(key, base_res, value, last_node, params.req) - elif self._mode is LMCacheMode.IP: - return self._ip_match_prefix(key, base_res, value, last_node) - return base_res - - def _mp_match_prefix( - self, - key: RadixKey, - base_res: MatchResult, - value: torch.Tensor, - last_node: TreeNode, - req: Req, - ) -> MatchResult: - """MP LOOKUP - - Returns a ``MatchResult`` with ``host_hit_length`` set when - LMCache has tokens beyond radix. Otherwise releases - the held read locks and returns the radix-only result. - """ - token_ids = key.raw_token_ids() - matched = self.lmcache_connector.lookup_kv(token_ids, req.rid) - if matched <= value.numel(): - # Release the read locks; keep the pending session for end_session. - self.lmcache_connector.release_pending(req.rid) - return base_res - - if token_ids is key.token_ids: - token_ids = token_ids[:] - self._mp_load_back_markers[req.rid] = _LMCacheLoadBackMarker( - key=RadixKey( - token_ids, - key.extra_key, - key.is_bigram, - cache_salt=key.cache_salt, - ), - value_numel=int(value.numel()), - ) - return MatchResult( - device_indices=value, - last_device_node=last_node, - last_host_node=last_node, - best_match_node=last_node, - host_hit_length=matched - int(value.numel()), - ) - - def _ip_match_prefix( - self, - key: RadixKey, - base_res: MatchResult, - value: torch.Tensor, - last_node: TreeNode, - ) -> MatchResult: - """IP mode: ``start_load_kv`` + per-layer hook. - - Allocates slots for the page-aligned uncached tail and kicks off - the layerwise load. Returns ``base_res`` if there's nothing to - fetch or alloc/load fails. - """ - if value.numel() == len(key): - return base_res - - uncached_len = len(key) - value.numel() - if uncached_len == 0: - return base_res - - token_ids = key.raw_token_ids() - result = self._load_back( - key=key, - value_numel=int(value.numel()), - uncached_len=uncached_len, - last_node=last_node, - load_fn=lambda sm, pp: self._ip_load_back( - token_ids=token_ids, - value_numel=int(value.numel()), - slot_mapping=sm, - prefix_pad=pp, - ), - ) - if result is None: - return base_res - new_slots, new_node = result - return MatchResult( - device_indices=torch.cat([value, new_slots]), - last_device_node=new_node, - last_host_node=new_node, - best_match_node=new_node, - ) - - def init_load_back( - self, params: InitLoadBackParams - ) -> Tuple[torch.Tensor, Optional[TreeNode]]: - """MP RETRIEVE. - - Called by the scheduler when ``match_prefix`` returned - ``host_hit_length > 0``. Uses the cached LOOKUP result to - allocate slots and fire RETRIEVE, inserts the resulting - TreeNode into the radix tree, and returns - ``(new_indices, new_last_node)``. - """ - req = params.req - marker = self._mp_load_back_markers.pop(req.rid) - last_node: TreeNode = params.best_match_node - - result = self._load_back( - key=marker.key, - value_numel=marker.value_numel, - uncached_len=params.host_hit_length, - last_node=last_node, - load_fn=lambda sm, pp: self._mp_load_back( - marker=marker, - request_id=req.rid, - slot_mapping=sm, - prefix_pad=pp, - ), - ) - if result is None: - # Either alloc failed (locks still held by lookup_kv) or - # retrieve returned nothing (locks already released by - # retrieve_kv). release_pending is idempotent on locks_held. - self.lmcache_connector.release_pending(req.rid) - return ( - torch.empty((0,), dtype=torch.int64, device=self.device), - last_node, - ) - return result - - def _load_back( - self, - *, - key: RadixKey, - value_numel: int, - uncached_len: int, - last_node: TreeNode, - load_fn, # Callable[[torch.Tensor, int], int] — (slot_mapping, prefix_pad) -> num_retrieved - ) -> Optional[Tuple[torch.Tensor, TreeNode]]: - """Alloc slots, run ``load_fn``, attach a TreeNode for what was loaded. - - Returns ``(slots, new_node)`` on success, ``None`` if alloc fails - or the load returned zero (slots are freed in either case). - """ - chunk_size = self.lmcache_connector.chunk_size() - prefix_pad = value_numel % chunk_size - - if self.token_to_kv_pool_allocator.available_size() < uncached_len: - self.evict(EvictParams(num_tokens=uncached_len)) - - token_slots = self.token_to_kv_pool_allocator.alloc(uncached_len) - if token_slots is None: - return None - - slot_mapping = torch.empty( - value_numel + token_slots.numel(), - dtype=torch.int64, - device=self.device, - ) - slot_mapping[:value_numel].fill_(-1) - slot_mapping[value_numel:].copy_(token_slots) - - # Dispatch to the mode-specific loader (IP: start_load_kv, MP: - # retrieve_kv). Each loader manages its own load_stream context. - num_retrieved = load_fn(slot_mapping, prefix_pad) - logger.debug("num_retrieved_tokens: %s", num_retrieved) - - if num_retrieved > 0: - self.token_to_kv_pool_allocator.free( - token_slots[(num_retrieved - prefix_pad) :] - ) - else: - self.token_to_kv_pool_allocator.free(token_slots) - - if num_retrieved > 0: - fetched = num_retrieved - prefix_pad - new_node = TreeNode(priority=last_node.priority) - start = value_numel - end = start + fetched - new_node.key = key[start:end] - new_node.value = token_slots[:fetched] - new_node.parent = last_node - last_node.children[new_node.key.child_key(self.page_size)] = new_node - self.evictable_size_ += fetched - self._update_leaf_status(last_node) - self._update_leaf_status(new_node) - - self.kv_events.record_store(new_node.parent) - self.kv_events.record_store(new_node) - - return token_slots[:fetched], new_node - - return None - - def _mp_load_back( - self, - *, - marker: _LMCacheLoadBackMarker, - request_id: str, - slot_mapping: torch.Tensor, - prefix_pad: int, - ) -> int: - """MP non-layerwise loader: fire ``retrieve_kv`` and wait for the - load_stream so the compute stream observes the writes. - """ - current_stream = torch.get_device_module(self.device).current_stream() - self.load_stream.wait_stream(current_stream) - with device_stream_context(self.load_stream): - n = self.lmcache_connector.retrieve_kv( - LoadMetadata( - token_ids=marker.key.token_ids, - slot_mapping=slot_mapping, - offset=marker.value_numel - prefix_pad, - prefix_pad=prefix_pad, - request_id=request_id, - ) - ) - current_stream.wait_stream(self.load_stream) - return n - - def _ip_load_back( - self, - *, - token_ids: list[int], - value_numel: int, - slot_mapping: torch.Tensor, - prefix_pad: int, - ) -> int: - """IP layerwise loader: kick off ``start_load_kv`` on ``self.load_stream``. - - ``start_load_kv`` enqueues the first layer's transfer; the - ``LayerTransferCounter`` hook drives the rest during forward. - """ - with device_stream_context(self.load_stream): - return self.lmcache_connector.start_load_kv( - LoadMetadata( - token_ids=token_ids, - slot_mapping=slot_mapping, - offset=value_numel - prefix_pad, - ) - ) - - def on_release(self, req: Req, *, inserted: bool) -> None: - if not inserted and self._mode is LMCacheMode.MP: - self._mp_load_back_markers.pop(req.rid, None) - self.lmcache_connector.end_session(req.rid) - - def cache_finished_req(self, req: Req, *, owned_kv_len: int) -> None: - """On request completion, insert device KV into radix and store to LMCache.""" - - super().cache_finished_req(req, owned_kv_len=owned_kv_len) - - topk = get_spec().speculative_eagle_topk - enable_kv_committed_len = topk is None or topk == 1 - if enable_kv_committed_len: - kv_committed_len = req.kv.kv_committed_len - else: - kv_committed_len = len(req.origin_input_ids) + max( - len(req.output_ids) - 1, 0 - ) - - token_ids = (req.origin_input_ids + req.output_ids)[:kv_committed_len] - kv_indices = self.req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, :kv_committed_len - ] - - # Use super() to avoid a redundant LOOKUP — we only need new_last_node from radix. - match_result = super().match_prefix( - MatchPrefixParams( - key=RadixKey( - token_ids, - req.extra_key, - cache_salt=req.cache_salt, - ) - ) - ) - new_last_node = match_result.last_device_node - assert new_last_node is not None - - self.inc_lock_ref(new_last_node) - store_md = StoreMetadata( - last_node=new_last_node, - token_ids=token_ids, - kv_indices=kv_indices, - offset=0, - request_id=req.rid, - ) - if self._mode is LMCacheMode.MP: - self.lmcache_connector.store_kv(store_md) - # MP store_kv blocks until the daemon's signal event fires, so the slots are safe to evict immediately. - self._mp_load_back_markers.pop(req.rid, None) - self.dec_lock_ref(new_last_node) - self.lmcache_connector.end_session(req.rid) - elif self._mode is LMCacheMode.IP: - with device_stream_context(self.store_stream): - self.lmcache_connector.store_kv(store_md) - # Layerwise store is async on store_stream; defer the unlock to evict()'s store_stream.synchronize(). - with self._node_lock: - self._in_flight_nodes.append(new_last_node) - - def evict(self, params: EvictParams) -> EvictResult: - """Before base eviction, wait for any outstanding stores and release locks.""" - if self.disable: - return EvictResult() - - self.store_stream.synchronize() - with self._node_lock: - for node in self._in_flight_nodes: - self.dec_lock_ref(node) - self._in_flight_nodes.clear() - - return super().evict(params) - - def pretty_print(self): - super().pretty_print() - try: - logger.debug( - "evictable=%d protected=%d", self.evictable_size_, self.protected_size_ - ) - except Exception: # pragma: no cover - pass diff --git a/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py new file mode 100644 index 0000000000..159ff9ad6d --- /dev/null +++ b/python/sglang/srt/mem_cache/storage/lmcache/lmcache_unified_radix_cache.py @@ -0,0 +1,982 @@ +"""Unified radix tree backed by the LMCache multiprocess service. + +LMCache owns external storage; retrieved KV is published after reaching GPU. +""" + +from __future__ import annotations + +import atexit +import logging +from array import array +from typing import TYPE_CHECKING, Any, Optional + +import torch +from lmcache.integration.sglang.lmcache_mp_metadata import ( + LMCacheExternalFlow, + LMCachePendingStore, +) +from lmcache.integration.sglang.unified_lmcache_mp_connector import ( + UnifiedLMCacheMPConnector, +) + +from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedSWAAllocatorBase, +) +from sglang.srt.mem_cache.base_prefix_cache import ( + CacheRequestHandle, + EvictParams, + InitLoadBackParams, + InsertParams, + MatchPrefixParams, + MatchResult, +) +from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.mem_cache.unified_cache.components import ComponentType +from sglang.srt.mem_cache.unified_radix_cache import NodeId, UnifiedRadixCache + +if TYPE_CHECKING: + from sglang.srt.configs.model_config import ModelConfig + from sglang.srt.managers.schedule_batch import Req + from sglang.srt.mem_cache.cache_init_params import CacheInitParams + +logger = logging.getLogger(__name__) + + +class LMCacheUnifiedRadixCache(UnifiedRadixCache): + """Unified radix tree with device-direct LMCache MP KV/state I/O.""" + + def __init__( + self, + params: CacheInitParams, + *, + model_config: ModelConfig, + tp_size: int, + tp_rank: int, + lmcache_config_file: Optional[str], + forward_stream: Any, + ) -> None: + # The parent constructor calls reset() before the connector exists. + self.lmcache_connector: Optional[UnifiedLMCacheMPConnector] = None + super().__init__(params) + self._mamba_component = self._find_mamba_component() + self.lmcache_connector = UnifiedLMCacheMPConnector( + config_file=lmcache_config_file, + model_config=model_config, + tp_size=tp_size, + tp_rank=tp_rank, + tp_group=params.tp_cache_group, + pp_size=params.pp_size, + pp_rank=params.pp_rank, + pp_group=params.pp_cache_group, + page_size=self.page_size, + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + req_to_token_pool=self.req_to_token_pool, + tree_components=self.tree_components, + mamba_component=self._mamba_component, + sliding_window_size=self._sliding_window_size, + ) + self._external_flows: dict[str, LMCacheExternalFlow] = {} + self._forward_stream = forward_stream + self._pending_stores: list[LMCachePendingStore] = [] + self._pending_store_counts: dict[str, int] = {} + self._session_finish_requested: set[str] = set() + self._lmcache_closed = False + atexit.register(self.shutdown) + + def is_backuped(self, node_id: NodeId) -> bool: + # LMCache rebuilds lookup keys from tokens, so any L1 node is valid. + return True + + def match_prefix(self, params: MatchPrefixParams) -> MatchResult: + """Report LMCache hits through host fields until GPU slots are ready.""" + requested_key_len = len(params.key) + req = params.req + flow = self._external_flows.get(req.rid) if req is not None else None + if ( + flow is not None + and flow.total_hit is not None + and flow.local_hit_tokens is not None + and flow.load is None + ): + # Use the shortest cross-rank L1 prefix for consistent admission. + params = MatchPrefixParams( + key=params.key[: flow.local_hit_tokens], + cow_mamba=params.cow_mamba, + req=req, + ) + elif flow is not None and flow.load is not None and params.cow_mamba: + # Re-arm Mamba COW from the retrieved checkpoint on retry. + self._arm_external_mamba_cow(flow, req) + params = MatchPrefixParams(key=params.key, cow_mamba=False, req=req) + result = super().match_prefix(params) + if req is None: + return result + if flow is None or flow.total_hit is None or flow.cancelled: + return result + + total_hit = min(flow.total_hit, requested_key_len, len(flow.key)) + local_hit = len(result.device_indices) + skip = 0 + if flow.load is not None and not flow.prefix_published: + # Track private slots shadowed by a concurrent tree insertion. + skip = max( + min(local_hit, total_hit) - flow.load.local_hit_tokens, + 0, + ) + flow.loaded_skip_tokens = max(flow.loaded_skip_tokens, skip) + if flow.load.result: + self._release_unused_loaded_slots(flow) + if total_hit <= local_hit: + return result + + if flow.load is None: + external_hit = total_hit - local_hit + return result._replace( + last_host_node=result.last_device_node, + best_match_node=result.last_device_node, + host_hit_length=external_hit, + swa_host_hit_length=( + min( + external_hit, + self.lmcache_connector.aligned_swa_window_size(), + ) + if self.is_swa_enabled + else 0 + ), + mamba_host_hit_length=(1 if self._mamba_component is not None else 0), + mamba_branching_seqlen=None, + full_kv_hit_length=max(result.full_kv_hit_length, total_hit), + ) + + # Reuse an already submitted H2D load on the next admission attempt. + suffix = flow.load.device_indices[skip : skip + max(total_hit - local_hit, 0)] + return result._replace( + device_indices=torch.cat([result.device_indices, suffix]), + last_host_node=result.last_device_node, + best_match_node=result.last_device_node, + host_hit_length=0, + swa_host_hit_length=0, + mamba_host_hit_length=0, + mamba_branching_seqlen=None, + full_kv_hit_length=max(result.full_kv_hit_length, total_hit), + ) + + def prefetch_from_storage( + self, + request: CacheRequestHandle, + last_host_node_id: NodeId, + new_input_tokens: list[int], + last_hash: Optional[str] = None, + prefix_keys: Optional[list[str]] = None, + matched_prefix_tokens: Optional[list[int]] = None, + extra_key: Optional[str] = None, + cache_salt: Optional[str] = None, + storage_hit_end: Optional[int] = None, + ) -> None: + del last_hash, prefix_keys, storage_hit_end + req_id = request.rid + if req_id in self._external_flows: + return + local_tokens = list(matched_prefix_tokens or []) + token_ids = local_tokens + list(new_input_tokens) + anchor_extra_key, anchor_cache_salt = self.tree_core.prefetch_anchor_info( + last_host_node_id + ) + extra_key = extra_key or anchor_extra_key + cache_salt = cache_salt or anchor_cache_salt + key = RadixKey( + array("q", token_ids), + extra_key=extra_key, + is_bigram=self.tree_core.is_eagle, + cache_salt=cache_salt, + ).page_aligned(self.page_size) + if len(key) == 0: + return + token_ids = key.raw_token_ids()[: len(key)] + lookup = self.lmcache_connector.submit_lookup( + req_id, + token_ids, + local_hit_tokens=min(len(local_tokens), len(key)), + cache_salt=self.lmcache_connector.build_cache_salt(cache_salt, extra_key), + ) + self._external_flows[req_id] = LMCacheExternalFlow( + key=key, + lookup=lookup, + ) + + def check_prefetch_progress(self, handle: CacheRequestHandle) -> bool: + req_id = handle.rid + flow = self._external_flows.get(req_id) + if flow is None: + return True + if flow.cancelled: + return False + + if flow.total_hit is None: + total_hit = self.lmcache_connector.poll_lookup(flow.lookup) + if total_hit is None: + return False + latest = super().match_prefix(MatchPrefixParams(key=flow.key)) + local_hit = torch.tensor( + [len(latest.device_indices)], dtype=torch.int64, device="cpu" + ) + self.lmcache_connector.parallel_all_reduce( + local_hit, torch.distributed.ReduceOp.MIN + ) + total_hit = min(total_hit, len(flow.key)) + local_hit_tokens = int(local_hit.item()) + release_end = min( + total_hit, + local_hit_tokens + // self.lmcache_connector.chunk_size + * self.lmcache_connector.chunk_size, + ) + if release_end > 0: + # All ranks advance state; only the leader releases remote locks. + self.lmcache_connector.free_lookup_locks( + req_id, start=0, end=release_end + ) + if total_hit <= local_hit_tokens: + self._external_flows.pop(req_id, None) + self.prefetch_loaded_tokens_by_reqid[handle] = 0 + self.prefetch_loaded_storage_start_by_reqid.pop(handle, None) + return True + flow.total_hit = total_hit + flow.local_hit_tokens = local_hit_tokens + self.prefetch_loaded_tokens_by_reqid[handle] = total_hit - local_hit_tokens + self.prefetch_loaded_storage_start_by_reqid[handle] = local_hit_tokens + + return True + + def cache_unfinished_req(self, req: Req, chunked: bool = False, **kwargs) -> None: + self._publish_external_loaded_prefix(req, token_ids_len=len(req.get_fill_ids())) + super().cache_unfinished_req(req, chunked=chunked, **kwargs) + self._retire_loaded_flow(req.rid) + self._submit_store(req, req.get_fill_ids()) + + def on_release(self, req: Req, *, inserted: bool) -> None: + super().on_release(req, inserted=inserted) + if not inserted: + self.release_aborted_request(req.cache_request_handle) + + def cache_finished_req(self, req: Req, *, owned_kv_len: int, **kwargs) -> None: + self._publish_external_loaded_prefix(req, token_ids_len=owned_kv_len) + super().cache_finished_req(req, owned_kv_len=owned_kv_len, **kwargs) + self._retire_loaded_flow(req.rid) + token_ids = (req.origin_input_ids + req.output_ids)[:owned_kv_len] + self._submit_store(req, token_ids) + self._request_session_finish(req.rid) + + def check_hicache_events(self) -> None: + """Poll LMCache retrieve/store futures at the scheduler safe point.""" + load_flows = [ + flow + for flow in self._external_flows.values() + if flow.load is not None and not flow.load_completed + ] + ready_loads = self.lmcache_connector.ready_prefix_count( + [f.load for f in load_flows] + ) + for flow in load_flows[:ready_loads]: + assert flow.load is not None + success = self.lmcache_connector.complete_load(flow.load) + flow.load_completed = True + cancelled = flow.cancelled + if cancelled or not success: + self._finish_failed_load(flow) + if not cancelled: + logger.warning( + "LMCache retrieve failed after admission for request %s", + flow.lookup.request_id, + ) + elif flow.retire_requested: + self._finalize_retired_flow(flow) + else: + self._finish_successful_load(flow) + + ready_stores = self.lmcache_connector.ready_prefix_count( + [pending.operation for pending in self._pending_stores] + ) + for _ in range(ready_stores): + pending = self._pending_stores.pop(0) + rid = pending.operation.request_id + try: + self.lmcache_connector.complete_store(pending.operation) + finally: + # Always unlock the source node after store completion. + self.dec_lock_ref(pending.node_id, pending.lock_params) + remaining = self._pending_store_counts[rid] - 1 + if remaining > 0: + self._pending_store_counts[rid] = remaining + else: + self._pending_store_counts.pop(rid) + self._finish_session_if_store_idle(rid) + + def has_pending_cache_operations(self) -> bool: + return bool(self._external_flows or self._pending_stores) + + def release_aborted_request(self, handle: CacheRequestHandle) -> None: + rid = handle.rid + self.prefetch_loaded_tokens_by_reqid.pop(handle, None) + self.prefetch_loaded_storage_start_by_reqid.pop(handle, None) + flow = self._external_flows.get(rid) + if flow is None: + self._request_session_finish(rid) + return + flow.cancelled = True + # Keep the checkpoint alive until H2D completes. + if flow.load is not None and flow.mamba_value is not None: + flow.free_mamba_after_load = True + if ( + flow.load_req is not None + and flow.load_req.kv.mamba_cow_src_index is flow.mamba_value + ): + flow.load_req.kv.mamba_cow_src_index = None + if flow.load is None: + if flow.total_hit is not None: + # No retrieve will release locks from this completed lookup. + self._retire_loaded_flow(rid) + else: + # END_SESSION follows the in-flight lookup and releases its locks. + self._external_flows.pop(rid, None) + elif flow.load_completed: + self._finish_failed_load(flow) + else: + # Complete the cross-rank load only in check_hicache_events(). + flow.retire_requested = True + self._request_session_finish(rid) + + def init_load_back(self, params: InitLoadBackParams) -> tuple[torch.Tensor, NodeId]: + req = params.req + if req is None: + return ( + self.tree_core.empty_match_result.device_indices, + params.best_match_node, + ) + flow = self._external_flows.get(req.rid) + if flow is None or flow.total_hit is None or flow.load is not None: + return ( + self.tree_core.empty_match_result.device_indices, + params.best_match_node, + ) + + device_indices = self._start_external_load(flow, req) + if device_indices is None: + req.storage_hit_length = 0 + req.host_hit_length = 0 + req.swa_host_hit_length = 0 + req.mamba_host_hit_length = 0 + self.prefetch_loaded_tokens_by_reqid.pop(req.cache_request_handle, None) + self.prefetch_loaded_storage_start_by_reqid.pop( + req.cache_request_handle, None + ) + # Retire the unloaded flow to release its lookup locks. + self._retire_loaded_flow(req.rid) + return ( + self.tree_core.empty_match_result.device_indices, + params.best_match_node, + ) + return device_indices, params.best_match_node + + def ready_to_load_host_cache(self) -> int: + # H2D is submitted in init_load_back to preserve prefill fallback. + return -1 + + def supports_retraction_backup(self) -> bool: + # TODO(chunxiaozheng): implement retraction backup + return False + + def clear_storage_backend(self) -> bool: + return self.lmcache_connector.clear() + + def reset(self) -> None: + connector = self.lmcache_connector + if connector is not None: + for flow in list(self._external_flows.values()): + if flow.load is not None: + try: + flow.load.future.result(timeout=connector.operation_timeout) + except Exception: + pass + connector.complete_load(flow.load, synchronize=False) + flow.load_completed = True + if flow.prefix_published: + # Free only private slots; the tree owns the published suffix. + self._release_unused_loaded_slots(flow) + else: + self.token_to_kv_pool_allocator.free( + flow.load.device_indices[flow.released_skip_tokens :] + ) + if flow.mamba_value is not None: + self.req_to_token_pool.mamba_allocator.free(flow.mamba_value) + flow.mamba_value = None + self._release_flow_anchor(flow) + for pending in list(self._pending_stores): + try: + pending.operation.future.result(timeout=connector.operation_timeout) + except Exception: + pass + connector.complete_store(pending.operation, synchronize=False) + self.dec_lock_ref(pending.node_id, pending.lock_params) + self._external_flows.clear() + self._pending_stores.clear() + self._pending_store_counts.clear() + self._session_finish_requested.clear() + self.prefetch_loaded_tokens_by_reqid.clear() + connector.end_all_sessions() + super().reset() + + def shutdown(self) -> None: + if self._lmcache_closed: + return + self.reset() + self.lmcache_connector.close() + self._lmcache_closed = True + + def release_host_resources(self) -> None: + self.shutdown() + + def _find_mamba_component(self): + return next( + ( + component + for component in self._components_tuple + if component.component_type is ComponentType.MAMBA + ), + None, + ) + + def _allocate_external_mamba_slot(self) -> Optional[torch.Tensor]: + if self._mamba_component is None: + return None + allocator = self.req_to_token_pool.mamba_allocator + slot = allocator.alloc(1) + if slot is None: + self.evict(EvictParams(num_tokens=0, mamba_num=1)) + slot = allocator.alloc(1) + if slot is not None: + physical = self.req_to_token_pool.translate_mamba_indices(slot) + self.lmcache_connector.reset_mamba_checkpoint_metadata(physical) + return slot + + def _arm_external_mamba_cow(self, flow: LMCacheExternalFlow, req: Req) -> None: + """Give a request a mutable state slot and COW from the loaded checkpoint.""" + checkpoint = flow.mamba_value + if checkpoint is None: + return + if req.kv.mamba_pool_idx is None: + active = self._allocate_external_mamba_slot() + assert active is not None, "Cannot allocate Mamba request state for LMCache" + req.kv.mamba_pool_idx = active[0] + flow.request_mamba_value = active + flow.allocated_request_mamba_for_load = True + elif flow.request_mamba_value is None: + flow.request_mamba_value = req.kv.mamba_pool_idx.reshape(1) + flow.allocated_request_mamba_for_load = False + flow.load_req = req + req.kv.mamba_cow_src_index = checkpoint + req.kv.mamba_needs_clear = False + + def _allocate_external_slots(self, num_tokens: int) -> Optional[torch.Tensor]: + allocator = self.token_to_kv_pool_allocator + if not self.is_swa_enabled: + if allocator.available_size() < num_tokens: + self.evict(EvictParams(num_tokens=num_tokens)) + return allocator.alloc(num_tokens) + + # Allocate SWA only for the trailing window; older pages use dummy page 0. + assert self._sliding_window_size is not None + swa_tail_tokens = min( + num_tokens, + self.lmcache_connector.aligned_swa_window_size(), + ) + full_allocator = allocator.full_attn_allocator + swa_allocator = allocator.swa_attn_allocator + if full_allocator.available_size() < num_tokens: + self.evict(EvictParams(num_tokens=num_tokens)) + if swa_allocator.available_size() < swa_tail_tokens: + self.evict(EvictParams(swa_num_tokens=swa_tail_tokens)) + + full_indices = full_allocator.alloc(num_tokens) + if full_indices is None: + return None + if swa_tail_tokens == 0: + return full_indices + + tail_full_indices = full_indices[-swa_tail_tokens:] + if isinstance(allocator, UnifiedSWAAllocatorBase): + # Unified-memory FULL and SWA share virtual page IDs. + virtual_pages = torch.unique(tail_full_indices // self.page_size) + try: + allocator.swa_attn_allocator.alloc_with_virtual(virtual_pages) + except Exception: + full_allocator.free(full_indices) + logger.exception("Failed to allocate unified SWA slots for LMCache") + return None + else: + swa_indices = swa_allocator.alloc(swa_tail_tokens) + if swa_indices is None: + full_allocator.free(full_indices) + return None + allocator.set_full_to_swa_mapping(tail_full_indices, swa_indices) + return full_indices + + def _start_external_load( + self, flow: LMCacheExternalFlow, req: Req + ) -> Optional[torch.Tensor]: + assert flow.total_hit is not None + assert flow.local_hit_tokens is not None + local_hit = flow.local_hit_tokens + latest = super().match_prefix(MatchPrefixParams(key=flow.key[:local_hit])) + total_hit = min(flow.total_hit, len(flow.key)) + if total_hit <= local_hit: + return self.tree_core.empty_match_result.device_indices + + # Pin the common L1 boundary before allocation can trigger eviction. + flow.anchor_node = latest.last_device_node + flow.anchor_lock = self.inc_lock_ref(flow.anchor_node).to_dec_params() + num_tokens = total_hit - local_hit + device_indices = self._allocate_external_slots(num_tokens) + mamba_value = None + allocated_mamba_for_load = False + request_mamba_value = None + allocated_request_mamba_for_load = False + if device_indices is not None and self._mamba_component is not None: + # Keep the immutable checkpoint separate from mutable request state. + mamba_value = self._allocate_external_mamba_slot() + allocated_mamba_for_load = mamba_value is not None + if req.kv.mamba_pool_idx is None: + request_mamba_value = self._allocate_external_mamba_slot() + allocated_request_mamba_for_load = request_mamba_value is not None + else: + request_mamba_value = req.kv.mamba_pool_idx.reshape(1) + allocation_ok = torch.tensor( + [ + int( + device_indices is not None + and ( + self._mamba_component is None + or (mamba_value is not None and request_mamba_value is not None) + ) + ) + ], + dtype=torch.int32, + device="cpu", + ) + self.lmcache_connector.parallel_all_reduce( + allocation_ok, torch.distributed.ReduceOp.MIN + ) + if not allocation_ok.item(): + if device_indices is not None: + self.token_to_kv_pool_allocator.free(device_indices) + if allocated_mamba_for_load and mamba_value is not None: + self.req_to_token_pool.mamba_allocator.free(mamba_value) + if allocated_request_mamba_for_load and request_mamba_value is not None: + self.req_to_token_pool.mamba_allocator.free(request_mamba_value) + self._release_flow_anchor(flow) + logger.debug( + "LMCache retrieve declined for %s: a parallel rank cannot allocate " + "%d GPU slots", + flow.lookup.request_id, + num_tokens, + ) + return None + assert device_indices is not None + flow.mamba_value = mamba_value + flow.request_mamba_value = request_mamba_value + flow.allocated_request_mamba_for_load = allocated_request_mamba_for_load + flow.load_req = req + flow.free_mamba_after_load = allocated_mamba_for_load + flow.loaded_skip_tokens = 0 + if allocated_request_mamba_for_load: + assert request_mamba_value is not None + req.kv.mamba_pool_idx = request_mamba_value[0] + req.kv.mamba_needs_clear = False + + try: + load_start = local_hit // self.lmcache_connector.chunk_size + load_start *= self.lmcache_connector.chunk_size + # Submit asynchronously while admission can still fall back to prefill. + flow.load = self.lmcache_connector.submit_load( + flow.lookup, + self.lmcache_connector.device_indices_by_group( + device_indices, + mamba_value=mamba_value, + mamba_transfer_tokens=total_hit - load_start, + ), + local_hit_tokens=local_hit, + owned_device_indices=device_indices, + producer_stream=self._forward_stream, + ) + if not self.lmcache_connector.prepare_load_on_stream( + flow.load, self._forward_stream + ): + raise RuntimeError("LMCache server rejected the retrieve request") + if mamba_value is not None: + # Forward waits for H2D before deferred Mamba COW runs. + self._arm_external_mamba_cow(flow, req) + except Exception: + self.token_to_kv_pool_allocator.free(device_indices) + if allocated_mamba_for_load and mamba_value is not None: + self.req_to_token_pool.mamba_allocator.free(mamba_value) + if allocated_request_mamba_for_load and request_mamba_value is not None: + self.req_to_token_pool.mamba_allocator.free(request_mamba_value) + req.kv.mamba_pool_idx = None + if req.kv.mamba_cow_src_index is mamba_value: + req.kv.mamba_cow_src_index = None + flow.mamba_value = None + flow.request_mamba_value = None + flow.allocated_request_mamba_for_load = False + flow.load_req = None + flow.free_mamba_after_load = False + self._release_flow_anchor(flow) + logger.exception( + "LMCache retrieve submission failed for %s", flow.lookup.request_id + ) + flow.load = None + return None + return device_indices[flow.loaded_skip_tokens :] + + def _release_flow_anchor(self, flow: LMCacheExternalFlow) -> None: + assert (flow.anchor_node is None) == (flow.anchor_lock is None), ( + "LMCache flow anchor node and lock must be set together" + ) + if flow.anchor_node is not None: + assert flow.anchor_lock is not None + self.dec_lock_ref(flow.anchor_node, flow.anchor_lock) + flow.anchor_node = None + flow.anchor_lock = None + + def _release_unused_loaded_slots(self, flow: LMCacheExternalFlow) -> None: + """Free retrieved slots shadowed by a longer rank-local L1 prefix.""" + assert flow.load is not None + release_end = min(flow.loaded_skip_tokens, len(flow.load.device_indices)) + if release_end <= flow.released_skip_tokens: + return + self.token_to_kv_pool_allocator.free( + flow.load.device_indices[flow.released_skip_tokens : release_end] + ) + flow.released_skip_tokens = release_end + + def _finish_failed_load(self, flow: LMCacheExternalFlow) -> None: + assert flow.load is not None + handle = ( + flow.load_req.cache_request_handle if flow.load_req is not None else None + ) + if not flow.prefix_published: + self.token_to_kv_pool_allocator.free( + flow.load.device_indices[flow.released_skip_tokens :] + ) + if flow.free_mamba_after_load and flow.mamba_value is not None: + self.req_to_token_pool.mamba_allocator.free(flow.mamba_value) + if ( + flow.load_req is not None + and flow.load_req.kv.mamba_cow_src_index is flow.mamba_value + ): + flow.load_req.kv.mamba_cow_src_index = None + flow.mamba_value = None + if ( + not flow.cancelled + and flow.allocated_request_mamba_for_load + and flow.request_mamba_value is not None + and flow.load_req is not None + and flow.load_req.kv.mamba_pool_idx is not None + and int(flow.load_req.kv.mamba_pool_idx.item()) + == int(flow.request_mamba_value[0].item()) + ): + self.req_to_token_pool.mamba_allocator.free(flow.request_mamba_value) + flow.load_req.kv.mamba_pool_idx = None + flow.request_mamba_value = None + flow.allocated_request_mamba_for_load = False + flow.load_req = None + flow.free_mamba_after_load = False + self._release_flow_anchor(flow) + rid = flow.lookup.request_id + self._external_flows.pop(rid, None) + if handle is not None: + if flow.cancelled: + self.prefetch_loaded_tokens_by_reqid.pop(handle, None) + else: + self.prefetch_loaded_tokens_by_reqid[handle] = 0 + self.prefetch_loaded_storage_start_by_reqid.pop(handle, None) + + def _finish_successful_load(self, flow: LMCacheExternalFlow) -> None: + """Finish local bookkeeping after LMCache has completed the retrieve.""" + assert flow.load is not None and flow.load_completed and flow.load.result + self._release_unused_loaded_slots(flow) + # Keep Mamba state until the cache callback publishes it. + + def _finalize_retired_flow(self, flow: LMCacheExternalFlow) -> None: + """Release a loaded flow after its cache callback retires it.""" + assert flow.load is not None and flow.load_completed and flow.load.result + self._finish_successful_load(flow) + if flow.mamba_value is not None: + # Free checkpoints from request paths that retire without insertion. + if ( + flow.load_req is not None + and flow.load_req.kv.mamba_cow_src_index is flow.mamba_value + ): + flow.load_req.kv.mamba_cow_src_index = None + self.req_to_token_pool.mamba_allocator.free(flow.mamba_value) + flow.mamba_value = None + flow.free_mamba_after_load = False + flow.request_mamba_value = None + flow.allocated_request_mamba_for_load = False + flow.load_req = None + self._release_flow_anchor(flow) + rid = flow.lookup.request_id + self._external_flows.pop(rid, None) + + def _retire_loaded_flow(self, rid: str) -> None: + """Mark a load for retirement without entering a collective.""" + flow = self._external_flows.get(rid) + if flow is None: + return + if flow.load is None: + if flow.total_hit is None: + return + self.lmcache_connector.free_lookup_locks( + rid, + start=flow.lookup.lock_start, + end=flow.total_hit, + ) + self._external_flows.pop(rid, None) + return + + flow.retire_requested = True + if not flow.load_completed: + return + if flow.load.result: + self._finalize_retired_flow(flow) + else: + cancelled = flow.cancelled + self._finish_failed_load(flow) + if not cancelled: + logger.warning( + "LMCache retrieve failed after admission for request %s", rid + ) + + def _prepare_external_slots_for_insert(self, req: Req) -> None: + """Restore tree ownership and mark the missing SWA prefix as evicted.""" + flow = self._external_flows.get(req.rid) + if flow is None or flow.load is None: + return + tree_owned_len = flow.load.local_hit_tokens + flow.loaded_skip_tokens + req.kv.cache_protected_len = min(req.kv.cache_protected_len, tree_owned_len) + if self.is_swa_enabled: + external_tokens = len(flow.load.device_indices) + swa_missing_end = flow.load.local_hit_tokens + max( + external_tokens - self.lmcache_connector.aligned_swa_window_size(), + 0, + ) + req.kv.swa_evicted_seqlen = max(req.kv.swa_evicted_seqlen, swa_missing_end) + + def _publish_external_loaded_prefix(self, req: Req, *, token_ids_len: int) -> None: + """Publish retrieved KV and immutable Mamba state into the device tree.""" + flow = self._external_flows.get(req.rid) + if ( + flow is None + or flow.load is None + or flow.total_hit is None + or flow.prefix_published + ): + return + # Restore the ownership boundary once before the parent cache callback. + self._prepare_external_slots_for_insert(req) + if flow.mamba_value is None: + # The parent callback will adopt the retrieved Full/SWA slots. + flow.prefix_published = True + return + + total_hit = min(flow.total_hit, len(flow.key)) + assert total_hit > 0, "LMCache loaded prefix must contain at least one token" + if total_hit > token_ids_len: + raise RuntimeError( + "LMCache Mamba checkpoint lies beyond the request KV boundary: " + f"{total_hit}>{token_ids_len} for request {req.rid}" + ) + prev_prefix_len = min(req.kv.cache_protected_len, total_hit) + kv_indices = self.req_to_token_pool.req_to_token[ + req.kv.req_pool_idx, :token_ids_len + ] + key = flow.key[:total_hit] + checkpoint = flow.mamba_value + result = self.insert( + InsertParams( + key=key, + value=kv_indices[:total_hit].to(dtype=torch.int64, copy=True), + mamba_value=checkpoint, + prev_prefix_len=prev_prefix_len, + swa_evicted_seqlen=req.kv.swa_evicted_seqlen, + chunked=True, + priority=req.priority or 0, + ) + ) + + # Free our checkpoint if a concurrent insertion already owns this boundary. + if result.mamba_exist: + self.req_to_token_pool.mamba_allocator.free(checkpoint) + flow.mamba_value = None + flow.free_mamba_after_load = False + + matched = super().match_prefix( + MatchPrefixParams(key=key, req=req, cow_mamba=False) + ) + if len(matched.device_indices) < total_hit: + raise RuntimeError( + "LMCache loaded prefix was inserted but is not reusable across " + f"all UnifiedRadixCache components: {len(matched.device_indices)}" + f"/{total_hit} tokens for request {req.rid}" + ) + canonical = matched.device_indices[:total_hit] + self.req_to_token_pool.write( + (req.kv.req_pool_idx, slice(prev_prefix_len, total_hit)), + canonical[prev_prefix_len:], + ) + + # Move the request lock to the published boundary. + if req.last_node is not None: + self._dec_req_lock(req) + lock_result = self.inc_lock_ref(matched.last_device_node) + if total_hit < token_ids_len: + req.prefix_indices = torch.cat( + [canonical, kv_indices[total_hit:].to(dtype=torch.int64, copy=True)] + ) + else: + req.prefix_indices = canonical + req.kv.cache_protected_len = total_hit + req.last_node = matched.last_device_node + req.lock_receipt = lock_result.to_dec_params() + req.swa_prefix_lock_released = False + flow.request_mamba_value = None + flow.allocated_request_mamba_for_load = False + flow.load_req = None + flow.prefix_published = True + + def _submit_store(self, req: Req, token_ids: list[int]) -> None: + key = RadixKey( + token_ids, + req.extra_key, + is_bigram=self.tree_core.is_eagle, + cache_salt=req.cache_salt, + ).page_aligned(self.page_size) + aligned_len = ( + len(key) + // self.lmcache_connector.chunk_size + * self.lmcache_connector.chunk_size + ) + key = key[:aligned_len] + if len(key) == 0: + return + matched = super().match_prefix(MatchPrefixParams(key=key)) + # Store the currently resident SWA prefix; later callbacks may extend it. + resident_len = torch.tensor( + [min(len(matched.device_indices), len(key))], + dtype=torch.int64, + device="cpu", + ) + self.lmcache_connector.parallel_all_reduce( + resident_len, torch.distributed.ReduceOp.MIN + ) + resident_len = ( + int(resident_len.item()) + // self.lmcache_connector.chunk_size + * self.lmcache_connector.chunk_size + ) + store_start = self.lmcache_connector.get_store_start(req.rid, len(key)) + if resident_len <= store_start: + logger.debug( + "LMCache store skipped for %s: no new resident chunk " + "after token %d (%d/%d tokens)", + req.rid, + store_start, + resident_len, + len(key), + ) + return + if resident_len < len(key): + logger.debug( + "LMCache store for %s is limited to the common resident prefix: " + "%d/%d tokens", + req.rid, + resident_len, + len(key), + ) + key = key[:resident_len] + # Re-match so the lock and Mamba state use the shortened boundary. + matched = super().match_prefix(MatchPrefixParams(key=key)) + store_indices = matched.device_indices[store_start : len(key)] + if len(store_indices) != len(key) - store_start: + logger.debug( + "LMCache store skipped for %s: store indices length: %d, " + "store start: %d, key length: %d", + req.rid, + len(store_indices), + store_start, + len(key), + ) + return + lock_params = self.inc_lock_ref(matched.last_device_node).to_dec_params() + mamba_value = ( + self.tree_core.get_component_device_value( + matched.best_match_node, ComponentType.MAMBA + ) + if self._mamba_component is not None + else None + ) + mamba_is_resident = torch.tensor( + [int(self._mamba_component is None or mamba_value is not None)], + dtype=torch.int32, + device="cpu", + ) + self.lmcache_connector.parallel_all_reduce( + mamba_is_resident, torch.distributed.ReduceOp.MIN + ) + if not mamba_is_resident.item(): + self.dec_lock_ref(matched.last_device_node, lock_params) + logger.debug( + "LMCache store skipped for %s: no Mamba checkpoint at token %d", + req.rid, + len(key), + ) + return + try: + operation = self.lmcache_connector.submit_store( + req.rid, + key.raw_token_ids()[: len(key)], + self.lmcache_connector.device_indices_by_group( + store_indices, + mamba_value=mamba_value, + ), + device_indices_start=store_start, + cache_salt=self.lmcache_connector.build_cache_salt( + req.cache_salt, req.extra_key + ), + ) + except Exception: + self.dec_lock_ref(matched.last_device_node, lock_params) + logger.exception("LMCache store submission failed for %s", req.rid) + return + if operation is None: + self.dec_lock_ref(matched.last_device_node, lock_params) + return + self._pending_stores.append( + LMCachePendingStore(operation, matched.last_device_node, lock_params) + ) + self._pending_store_counts[req.rid] = ( + self._pending_store_counts.get(req.rid, 0) + 1 + ) + + def _request_session_finish(self, rid: str) -> None: + """End the LMCache session after this request's stores have completed.""" + self._session_finish_requested.add(rid) + self._finish_session_if_store_idle(rid) + + def _finish_session_if_store_idle(self, rid: str) -> None: + if ( + rid not in self._session_finish_requested + or self._pending_store_counts.get(rid, 0) > 0 + ): + return + self._session_finish_requested.remove(rid) + self.lmcache_connector.finish_request(rid) diff --git a/python/sglang/srt/mem_cache/storage/lmcache/unit_test.py b/python/sglang/srt/mem_cache/storage/lmcache/unit_test.py deleted file mode 100644 index b6dae15d08..0000000000 --- a/python/sglang/srt/mem_cache/storage/lmcache/unit_test.py +++ /dev/null @@ -1,118 +0,0 @@ -try: - from lmcache.integration.sglang.sglang_adapter import ( - LMCacheLayerwiseConnector, - LoadMetadata, - StoreMetadata, - ) -except ImportError: - raise RuntimeError( - "LMCache is not installed. Please install it by running `pip install lmcache` in the root directory of LMCache" - ) - -import torch - -from sglang.srt.configs.model_config import ModelConfig - - -def test_load_store_metadata(): - model_config = ModelConfig( - model_path="Qwen/Qwen3-4B", - ) - - # Generate Dummy KV Cache - head_num = model_config.num_key_value_heads - head_dim = model_config.head_dim - layer_num = model_config.num_hidden_layers - buffer_size = 256 - input_id_len = 16 - - k_buffer = [ - torch.randn(buffer_size, head_num, head_dim, dtype=torch.bfloat16).cuda() - for _ in range(layer_num) - ] - v_buffer = [ - torch.randn(buffer_size, head_num, head_dim, dtype=torch.bfloat16).cuda() - for _ in range(layer_num) - ] - - connector = LMCacheLayerwiseConnector( - model_config, 1, 0, k_buffer, v_buffer, config_file="example_config_ip.yaml" - ) - - fake_token_ids = torch.randint(0, model_config.vocab_size, (input_id_len,)).tolist() - fake_kv_indices = torch.randint(0, buffer_size, (input_id_len,)) - offset = 0 - - store_metadata = StoreMetadata( - last_node=None, - token_ids=fake_token_ids, - kv_indices=fake_kv_indices, - offset=offset, - ) - - load_metadata = LoadMetadata( - token_ids=fake_token_ids, - slot_mapping=fake_kv_indices, - offset=offset, - ) - - current_stream = torch.cuda.current_stream() - - retrieve_token_num = connector.start_load_kv(load_metadata) - assert retrieve_token_num == 0 - - connector.store_kv(store_metadata) - current_stream.synchronize() - - # check retrieve - gt_key_buffer = [ - torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda() - for _ in range(layer_num) - ] - gt_value_buffer = [ - torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda() - for _ in range(layer_num) - ] - - for i in range(layer_num): - gt_key_buffer[i] = k_buffer[i][fake_kv_indices] - gt_value_buffer[i] = v_buffer[i][fake_kv_indices] - - # clear the k_buffer and v_buffer - for _ in range(layer_num): - k_buffer[i].zero_() - v_buffer[i].zero_() - - retrieve_token_num = connector.start_load_kv(load_metadata) - assert retrieve_token_num == input_id_len - - for i in range(layer_num): - current_stream.synchronize() - connector.load_kv_layerwise(i) - - current_stream.synchronize() - test_key_buffer = [ - torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda() - for _ in range(layer_num) - ] - test_value_buffer = [ - torch.zeros(input_id_len, head_num, head_dim, dtype=torch.bfloat16).cuda() - for _ in range(layer_num) - ] - - for i in range(layer_num): - test_key_buffer[i] = k_buffer[i][fake_kv_indices] - test_value_buffer[i] = v_buffer[i][fake_kv_indices] - - for i in range(layer_num): - assert torch.allclose(test_key_buffer[i], gt_key_buffer[i]) - assert torch.allclose(test_value_buffer[i], gt_value_buffer[i]) - - print("================================================") - print("TEST_LOAD_STORE_METADATA PASSED!") - print("================================================") - connector.close() - - -if __name__ == "__main__": - test_load_store_metadata() diff --git a/test/registered/unit/disaggregation/test_decode_queue_cleanup.py b/test/registered/unit/disaggregation/test_decode_queue_cleanup.py index b7e6ecf160..cadd589df4 100644 --- a/test/registered/unit/disaggregation/test_decode_queue_cleanup.py +++ b/test/registered/unit/disaggregation/test_decode_queue_cleanup.py @@ -482,6 +482,7 @@ class TestDecodeQueueCleanup(CustomTestCase): scheduler.decode_offload_manager = None scheduler.enable_hisparse = False scheduler.enable_hierarchical_cache = False + scheduler.enable_lmcache = False self.assertFalse(scheduler.is_fully_idle()) diff --git a/test/registered/unit/managers/test_scheduler_hicache_events.py b/test/registered/unit/managers/test_scheduler_hicache_events.py index 8c81c07927..9054de5ded 100644 --- a/test/registered/unit/managers/test_scheduler_hicache_events.py +++ b/test/registered/unit/managers/test_scheduler_hicache_events.py @@ -26,6 +26,7 @@ class TestSchedulerHiCacheEvents(unittest.TestCase): s.scheduler_stage_metrics = None s.enable_hierarchical_cache = True s.enable_unified_cache_external_linker = False + s.enable_lmcache = False s.enable_hicache_storage = True s.tree_cache = SimpleNamespace(check_hicache_events=self.calls.drain) s._process_storage_prefetch_retries = self.calls.retry diff --git a/test/registered/unit/managers/test_scheduler_on_idle_load.py b/test/registered/unit/managers/test_scheduler_on_idle_load.py index 9cba96e57c..185dd30eff 100644 --- a/test/registered/unit/managers/test_scheduler_on_idle_load.py +++ b/test/registered/unit/managers/test_scheduler_on_idle_load.py @@ -26,6 +26,7 @@ class TestOnIdleStallPublish(CustomTestCase): s.scheduler_stage_metrics = None s.enable_hicache_storage = False s.disaggregation_mode = DisaggregationMode.NULL + s.enable_lmcache = False s.maybe_send_health_check_signal = MagicMock() s.is_fully_idle = MagicMock(return_value=False) # stalled, not idle s.publish_load_snapshot = MagicMock(return_value=None) diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index 18ce7ae80b..53c1cd8e3c 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -406,25 +406,91 @@ class TestDefaultRadixCacheFactory(CustomTestCase): fake_radix.UnifiedRadixCache.assert_called_once_with(ctx.params) self.assertIs(result, fake_radix.UnifiedRadixCache.return_value) - def test_lmc_radix_cache_when_enable_lmcache(self): + def test_lmcache_unified_radix_cache_when_enable_lmcache(self): ctx = _make_ctx(self, enable_lmcache=True) - # The lmcache backend raises at import time when the `lmcache` - # package isn't installed, so inject a stand-in module instead - # of letting patch() trigger the real import. fake_module = MagicMock() + fake_components = MagicMock() with patch.dict( "sys.modules", - {"sglang.srt.mem_cache.storage.lmcache.lmc_radix_cache": fake_module}, + { + "sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache": fake_module, + "sglang.srt.mem_cache.unified_cache.components": fake_components, + }, ): result = default_radix_cache_factory(ctx) - fake_module.LMCRadixCache.assert_called_once_with( - params=ctx.params, + fake_module.LMCacheUnifiedRadixCache.assert_called_once_with( + ctx.params, model_config=ctx.model_config, tp_size=ctx.tp_size, - rank=ctx.tp_rank, - tp_group=ctx.tp_group, + tp_rank=ctx.tp_rank, + lmcache_config_file=None, + forward_stream=ctx.tp_worker.model_runner.forward_stream, ) - self.assertIs(result, fake_module.LMCRadixCache.return_value) + self.assertEqual( + ctx.params.tree_components, + (fake_components.ComponentType.FULL,), + ) + self.assertIs(result, fake_module.LMCacheUnifiedRadixCache.return_value) + + def test_lmcache_supports_hybrid_swa_components(self): + ctx = _make_ctx(self, enable_lmcache=True, is_hybrid_swa=True) + fake_module = MagicMock() + fake_components = MagicMock() + with patch.dict( + "sys.modules", + { + "sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache": fake_module, + "sglang.srt.mem_cache.unified_cache.components": fake_components, + }, + ): + default_radix_cache_factory(ctx) + + self.assertEqual( + ctx.params.tree_components, + ( + fake_components.ComponentType.FULL, + fake_components.ComponentType.SWA, + ), + ) + + def test_lmcache_supports_hybrid_ssm_components(self): + ctx = _make_ctx(self, enable_lmcache=True, is_hybrid_ssm=True) + fake_module = MagicMock() + fake_components = MagicMock() + with patch.dict( + "sys.modules", + { + "sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache": fake_module, + "sglang.srt.mem_cache.unified_cache.components": fake_components, + }, + ): + default_radix_cache_factory(ctx) + + self.assertEqual( + ctx.params.tree_components, + ( + fake_components.ComponentType.FULL, + fake_components.ComponentType.MAMBA, + ), + ) + + def test_lmcache_supports_dsa_as_full_sidecar(self): + ctx = _make_ctx(self, enable_lmcache=True, is_dsa=True) + fake_module = MagicMock() + fake_components = MagicMock() + with patch.dict( + "sys.modules", + { + "sglang.srt.mem_cache.storage.lmcache.lmcache_unified_radix_cache": fake_module, + "sglang.srt.mem_cache.unified_cache.components": fake_components, + }, + ): + default_radix_cache_factory(ctx) + + self.assertEqual( + ctx.params.tree_components, + (fake_components.ComponentType.FULL,), + ) if __name__ == "__main__": diff --git a/test/registered/xpu/test_lmcache_connector.py b/test/registered/xpu/test_lmcache_connector.py deleted file mode 100644 index 2b52241142..0000000000 --- a/test/registered/xpu/test_lmcache_connector.py +++ /dev/null @@ -1,361 +0,0 @@ -""" -XPU integration tests for LMCache connector in SGLang. - -Tests store/retrieve round-trip on Intel XPU using pure PyTorch ops -(index_copy_, index_select) instead of CUDA lmc_ops kernels. - -Uses a single shared connector to avoid LMCacheEngineBuilder singleton -issues (close() does not remove from _instances, so re-creating a -connector returns a dead engine). - -Usage: - python3 -m unittest registered.xpu.test_lmcache_connector -""" - -import os -import unittest - -import torch - -from sglang.test.ci.ci_register import register_xpu_ci - -# Must be set before lmcache imports. Save prior values so tearDownModule can -# restore them and avoid leaking into other tests in the same process. -_PATCHED_ENV = { - "LMCACHE_USE_EXPERIMENTAL": "True", - "LMCACHE_CONFIG_FILE": os.path.join( - os.path.dirname(__file__), "test_lmcache_connector_config.yaml" - ), -} -_OLD_ENV = {k: os.environ.get(k) for k in _PATCHED_ENV} -os.environ["LMCACHE_USE_EXPERIMENTAL"] = _PATCHED_ENV["LMCACHE_USE_EXPERIMENTAL"] -os.environ.setdefault("LMCACHE_CONFIG_FILE", _PATCHED_ENV["LMCACHE_CONFIG_FILE"]) - - -def tearDownModule(): - for key, old_value in _OLD_ENV.items(): - if old_value is None: - os.environ.pop(key, None) - else: - os.environ[key] = old_value - - -try: - from lmcache.integration.sglang.sglang_adapter import ( - LMCacheLayerwiseConnector, - LoadMetadata, - StoreMetadata, - ) -except ImportError: - raise RuntimeError( - "LMCache is not installed. " - "Install with: NO_CUDA_EXT=1 pip install -e . --no-build-isolation" - ) - -from sglang.srt.configs.model_config import ModelConfig - -XPU_AVAILABLE = hasattr(torch, "xpu") and torch.xpu.is_available() - -register_xpu_ci(est_time=60, suite="stage-b-test-1-gpu-xpu") - - -@unittest.skipUnless(XPU_AVAILABLE, "Intel XPU not available") -class TestLMCacheXPUConnector(unittest.TestCase): - """Test LMCache layerwise connector store/retrieve on XPU. - - All tests share a single connector instance to avoid the - LMCacheEngineBuilder singleton issue where close() does not - remove the engine from _instances. - """ - - DEVICE = "xpu:0" - BUFFER_SIZE = 256 - INPUT_LEN = 16 - - @classmethod - def setUpClass(cls): - cls.model_config = ModelConfig(model_path="Qwen/Qwen3-4B") - cls.head_num = cls.model_config.num_key_value_heads - cls.head_dim = cls.model_config.head_dim - cls.layer_num = cls.model_config.num_hidden_layers - cls.vocab_size = cls.model_config.vocab_size - - # Shared KV buffers and connector (created once) - cls.k_buffer = [ - torch.randn( - cls.BUFFER_SIZE, - cls.head_num, - cls.head_dim, - dtype=torch.bfloat16, - device=cls.DEVICE, - ) - for _ in range(cls.layer_num) - ] - cls.v_buffer = [ - torch.randn( - cls.BUFFER_SIZE, - cls.head_num, - cls.head_dim, - dtype=torch.bfloat16, - device=cls.DEVICE, - ) - for _ in range(cls.layer_num) - ] - cls.connector = LMCacheLayerwiseConnector( - cls.model_config, - tp_size=1, - rank=0, - k_pool=cls.k_buffer, - v_pool=cls.v_buffer, - config_file=os.environ["LMCACHE_CONFIG_FILE"], - ) - - @classmethod - def tearDownClass(cls): - cls.connector.close() - - def setUp(self): - """Re-randomize buffers before each test for isolation.""" - for i in range(self.layer_num): - self.k_buffer[i].normal_() - self.v_buffer[i].normal_() - - def _unique_tokens(self, length=None, salt=0): - """Generate unique token ids unlikely to collide across tests.""" - n = length or self.INPUT_LEN - base = torch.randint(0, self.vocab_size, (n,)) - return [(t.item() + salt) % self.vocab_size for t in base] - - def test_store_then_retrieve(self): - """Basic: store KV, clear buffers, retrieve and verify match.""" - token_ids = self._unique_tokens(salt=100) - kv_indices = torch.randint(0, self.BUFFER_SIZE, (self.INPUT_LEN,)) - - # First retrieve should return 0 (cold cache) - load_meta = LoadMetadata( - token_ids=token_ids, - slot_mapping=kv_indices, - offset=0, - ) - self.assertEqual(self.connector.start_load_kv(load_meta), 0) - - # Store - store_meta = StoreMetadata( - last_node=None, - token_ids=token_ids, - kv_indices=kv_indices, - offset=0, - ) - self.connector.store_kv(store_meta) - torch.xpu.synchronize() - - # Save ground truth before clearing - gt_k = [self.k_buffer[i][kv_indices].clone() for i in range(self.layer_num)] - gt_v = [self.v_buffer[i][kv_indices].clone() for i in range(self.layer_num)] - - # Clear buffers - for i in range(self.layer_num): - self.k_buffer[i].zero_() - self.v_buffer[i].zero_() - - # Retrieve - ret = self.connector.start_load_kv(load_meta) - self.assertEqual(ret, self.INPUT_LEN) - - for i in range(self.layer_num): - torch.xpu.synchronize() - self.connector.load_kv_layerwise(i) - - torch.xpu.synchronize() - - # Verify - for i in range(self.layer_num): - actual_k = self.k_buffer[i][kv_indices] - actual_v = self.v_buffer[i][kv_indices] - self.assertTrue( - torch.allclose(actual_k, gt_k[i]), - f"Layer {i}: K mismatch (max diff {(actual_k - gt_k[i]).abs().max():.6f})", - ) - self.assertTrue( - torch.allclose(actual_v, gt_v[i]), - f"Layer {i}: V mismatch (max diff {(actual_v - gt_v[i]).abs().max():.6f})", - ) - - def test_retrieve_cold_cache_returns_zero(self): - """Retrieve from empty cache should return 0 tokens.""" - token_ids = self._unique_tokens(salt=200) - kv_indices = torch.randint(0, self.BUFFER_SIZE, (self.INPUT_LEN,)) - - load_meta = LoadMetadata( - token_ids=token_ids, - slot_mapping=kv_indices, - offset=0, - ) - self.assertEqual(self.connector.start_load_kv(load_meta), 0) - - def test_slot_mapping_with_negative_indices(self): - """slot_mapping may contain -1 for an already-cached prefix. - - The -1 filtering fix must (a) not crash on XPU and (b) leave the -1 - slots untouched while correctly restoring the valid slots. - """ - # Distinct index for every token so -1 slots and valid slots never - # alias each other, keeping the untouched/restored assertions exact. - kv_indices = torch.randperm(self.BUFFER_SIZE)[: self.INPUT_LEN] - token_ids = self._unique_tokens(salt=400) - - # Store first - store_meta = StoreMetadata( - last_node=None, - token_ids=token_ids, - kv_indices=kv_indices, - offset=0, - ) - self.connector.store_kv(store_meta) - torch.xpu.synchronize() - - # Build slot_mapping with -1 prefix (simulating already-cached tokens). - num_cached = 8 - slot_mapping_with_neg = kv_indices.clone() - slot_mapping_with_neg[:num_cached] = -1 - cached_slots = kv_indices[:num_cached] - valid_slots = kv_indices[num_cached:] - - # Ground truth for the valid tail (what retrieve must restore). - gt_k = [self.k_buffer[i][valid_slots].clone() for i in range(self.layer_num)] - gt_v = [self.v_buffer[i][valid_slots].clone() for i in range(self.layer_num)] - - # Clear buffers: untouched -1 slots stay zero, valid slots get restored. - for i in range(self.layer_num): - self.k_buffer[i].zero_() - self.v_buffer[i].zero_() - - load_meta = LoadMetadata( - token_ids=token_ids, - slot_mapping=slot_mapping_with_neg, - # offset marks how many leading tokens are already cached (the - # -1 prefix in slot_mapping); this mirrors how lmc_radix_cache's - # _ip_load_back derives offset from the already-matched prefix. - offset=num_cached, - ) - # Should not crash on XPU (the -1 filtering fix is critical here). - ret = self.connector.start_load_kv(load_meta) - self.assertEqual(ret, self.INPUT_LEN - num_cached) - for i in range(self.layer_num): - torch.xpu.synchronize() - self.connector.load_kv_layerwise(i) - torch.xpu.synchronize() - - for i in range(self.layer_num): - # -1 positions must be untouched (still zero). - self.assertTrue( - torch.all(self.k_buffer[i][cached_slots] == 0), - f"Layer {i}: -1 K slots were written", - ) - self.assertTrue( - torch.all(self.v_buffer[i][cached_slots] == 0), - f"Layer {i}: -1 V slots were written", - ) - # Valid positions must be restored. - self.assertTrue( - torch.allclose(self.k_buffer[i][valid_slots], gt_k[i]), - f"Layer {i}: valid K slots not restored", - ) - self.assertTrue( - torch.allclose(self.v_buffer[i][valid_slots], gt_v[i]), - f"Layer {i}: valid V slots not restored", - ) - - def test_multiple_store_retrieve_cycles(self): - """Multiple store/retrieve cycles should not leak or corrupt.""" - for cycle in range(3): - token_ids = self._unique_tokens(salt=500 + cycle * 1000) - kv_indices = torch.randint(0, self.BUFFER_SIZE, (self.INPUT_LEN,)) - - store_meta = StoreMetadata( - last_node=None, - token_ids=token_ids, - kv_indices=kv_indices, - offset=0, - ) - self.connector.store_kv(store_meta) - torch.xpu.synchronize() - - gt_k = [self.k_buffer[i][kv_indices].clone() for i in range(self.layer_num)] - gt_v = [self.v_buffer[i][kv_indices].clone() for i in range(self.layer_num)] - - for i in range(self.layer_num): - self.k_buffer[i].zero_() - self.v_buffer[i].zero_() - - load_meta = LoadMetadata( - token_ids=token_ids, - slot_mapping=kv_indices, - offset=0, - ) - ret = self.connector.start_load_kv(load_meta) - self.assertEqual( - ret, - self.INPUT_LEN, - f"Cycle {cycle}: expected {self.INPUT_LEN}, got {ret}", - ) - - for i in range(self.layer_num): - torch.xpu.synchronize() - self.connector.load_kv_layerwise(i) - torch.xpu.synchronize() - - for i in range(self.layer_num): - actual_k = self.k_buffer[i][kv_indices] - actual_v = self.v_buffer[i][kv_indices] - self.assertTrue( - torch.allclose(actual_k, gt_k[i]), - f"Cycle {cycle}, Layer {i}: K mismatch", - ) - self.assertTrue( - torch.allclose(actual_v, gt_v[i]), - f"Cycle {cycle}, Layer {i}: V mismatch", - ) - - def test_bf16_dtype_preserved(self): - """Verify bf16 dtype is preserved through store->retrieve cycle. - - TODO: once XPU LMCache supports KV quantization, extend the dtype - coverage here to the quantized store/retrieve dtypes (e.g. fp8). - """ - token_ids = self._unique_tokens(salt=600) - kv_indices = torch.randint(0, self.BUFFER_SIZE, (self.INPUT_LEN,)) - - store_meta = StoreMetadata( - last_node=None, - token_ids=token_ids, - kv_indices=kv_indices, - offset=0, - ) - self.connector.store_kv(store_meta) - torch.xpu.synchronize() - - for i in range(self.layer_num): - self.k_buffer[i].zero_() - self.v_buffer[i].zero_() - - load_meta = LoadMetadata( - token_ids=token_ids, - slot_mapping=kv_indices, - offset=0, - ) - ret = self.connector.start_load_kv(load_meta) - self.assertEqual(ret, self.INPUT_LEN) - - for i in range(self.layer_num): - torch.xpu.synchronize() - self.connector.load_kv_layerwise(i) - torch.xpu.synchronize() - - for i in range(self.layer_num): - self.assertEqual(self.k_buffer[i].dtype, torch.bfloat16) - self.assertEqual(self.v_buffer[i].dtype, torch.bfloat16) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/registered/xpu/test_lmcache_connector_config.yaml b/test/registered/xpu/test_lmcache_connector_config.yaml deleted file mode 100644 index a256fa84d4..0000000000 --- a/test/registered/xpu/test_lmcache_connector_config.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# LMCache config for XPU unit tests -# save_unfull_chunk enables storing sequences shorter than chunk_size -chunk_size: 8 -local_cpu: true -use_layerwise: true -max_local_cpu_size: 1 -save_unfull_chunk: true diff --git a/test/registered/xpu/test_lmcache_radix_cache.py b/test/registered/xpu/test_lmcache_radix_cache.py deleted file mode 100644 index bf8842b19b..0000000000 --- a/test/registered/xpu/test_lmcache_radix_cache.py +++ /dev/null @@ -1,235 +0,0 @@ -""" -XPU integration tests for LMCRadixCache (IP mode). - -Unlike test_lmcache_connector.py, which drives LMCacheLayerwiseConnector -directly, this test builds a real ReqToTokenPool + MHATokenToKVPool + -TokenToKVPoolAllocator and drives LMCRadixCache itself through the request -lifecycle (match_prefix -> cache_finished_req, then evict a fresh cache to -force a real LMCache load-back through match_prefix again). This exercises -LayerTransferCounter.wait_until, _load_back's slot/offset math, and the -store/load stream synchronization that the connector-level tests never touch. - -Usage: - python3 -m unittest registered.xpu.test_lmcache_radix_cache -""" - -import os -import unittest -from types import SimpleNamespace - -import torch - -from sglang.srt.managers.schedule_batch import ReqKvInfo -from sglang.test.ci.ci_register import register_xpu_ci - -# Must be set before lmcache imports. Save prior values so tearDownModule can -# restore them and avoid leaking into other tests in the same process. -_PATCHED_ENV = { - "LMCACHE_USE_EXPERIMENTAL": "True", - "LMCACHE_CONFIG_FILE": os.path.join( - os.path.dirname(__file__), "test_lmcache_connector_config.yaml" - ), -} -_OLD_ENV = {k: os.environ.get(k) for k in _PATCHED_ENV} -os.environ["LMCACHE_USE_EXPERIMENTAL"] = _PATCHED_ENV["LMCACHE_USE_EXPERIMENTAL"] -os.environ.setdefault("LMCACHE_CONFIG_FILE", _PATCHED_ENV["LMCACHE_CONFIG_FILE"]) - - -def tearDownModule(): - for key, old_value in _OLD_ENV.items(): - if old_value is None: - os.environ.pop(key, None) - else: - os.environ[key] = old_value - - -try: - import lmcache.integration.sglang.sglang_adapter # noqa: F401 -except ImportError: - raise RuntimeError("LMCache is not installed. Install with: pip install lmcache") - -from sglang.srt.configs.model_config import ModelConfig -from sglang.srt.mem_cache.allocator.token import TokenToKVPoolAllocator -from sglang.srt.mem_cache.base_prefix_cache import EvictParams, MatchPrefixParams -from sglang.srt.mem_cache.cache_init_params import CacheInitParams -from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, ReqToTokenPool -from sglang.srt.mem_cache.radix_cache import RadixKey -from sglang.srt.mem_cache.storage.lmcache.lmc_radix_cache import LMCRadixCache -from sglang.srt.runtime_context import get_context - -XPU_AVAILABLE = hasattr(torch, "xpu") and torch.xpu.is_available() - -register_xpu_ci(est_time=60, suite="stage-b-test-1-gpu-xpu") - - -def _make_req(rid, req_pool_idx, token_ids, tree): - """Fake Req with the fields LMCRadixCache/RadixCache read (mirrors the - SimpleNamespace pattern in test_swa_eviction_boundary.py).""" - req = SimpleNamespace( - rid=rid, - origin_input_ids=token_ids, - output_ids=[], - extra_key=None, - cache_salt=None, - last_node=tree.root_node, - kv=ReqKvInfo( - req_pool_idx=req_pool_idx, - cache_protected_len=0, - kv_committed_len=len(token_ids), - ), - priority=0, - kv_committed_freed=False, - ) - req.pop_committed_kv_cache = lambda: len(token_ids) - return req - - -@unittest.skipUnless(XPU_AVAILABLE, "Intel XPU not available") -class TestLMCRadixCacheXPU(unittest.TestCase): - """Drive LMCRadixCache (IP mode) through match_prefix/cache_finished_req - with a real KV pool, to cover the code path test_lmcache_connector.py - (which talks to the connector directly) never exercises.""" - - DEVICE = "xpu:0" - BUFFER_SIZE = 256 - MAX_CONTEXT_LEN = 64 - INPUT_LEN = 16 - - @classmethod - def setUpClass(cls): - cls.model_config = ModelConfig(model_path="Qwen/Qwen3-4B") - cls._override = get_context().override_server_args( - lmcache_config_file=os.environ["LMCACHE_CONFIG_FILE"], - speculative_eagle_topk=None, - ) - cls._override.install() - - @classmethod - def tearDownClass(cls): - cls._override.restore() - - def _build_tree(self): - model_config = self.model_config - kv_pool = MHATokenToKVPool( - size=self.BUFFER_SIZE, - page_size=1, - dtype=torch.bfloat16, - head_num=model_config.num_key_value_heads, - head_dim=model_config.head_dim, - layer_num=model_config.num_hidden_layers, - device=self.DEVICE, - enable_memory_saver=False, - ) - allocator = TokenToKVPoolAllocator( - size=self.BUFFER_SIZE, - dtype=torch.bfloat16, - device=self.DEVICE, - kvcache=kv_pool, - need_sort=False, - ) - req_to_token_pool = ReqToTokenPool( - size=8, - max_context_len=self.MAX_CONTEXT_LEN, - device=self.DEVICE, - enable_memory_saver=False, - ) - tree = LMCRadixCache( - params=CacheInitParams( - disable=False, - req_to_token_pool=req_to_token_pool, - token_to_kv_pool_allocator=allocator, - page_size=1, - ), - model_config=model_config, - tp_size=1, - rank=0, - ) - return tree, allocator, req_to_token_pool, kv_pool - - def test_store_then_load_back_through_match_prefix(self): - """Full lifecycle: match_prefix (miss) -> cache_finished_req (store to - LMCache + insert into radix) -> evict the radix entry -> match_prefix - again must retrieve from LMCache via LayerTransferCounter and restore - the original KV content, proving _load_back's offset/slot math and - the load_stream synchronization are correct end-to-end.""" - tree, allocator, req_to_token_pool, kv_pool = self._build_tree() - try: - token_ids = torch.randint( - 0, self.model_config.vocab_size, (self.INPUT_LEN,) - ).tolist() - - # No prior entries: match_prefix should be a full miss. - miss_res = tree.match_prefix(MatchPrefixParams(key=RadixKey(token_ids))) - self.assertEqual(miss_res.device_indices.numel(), 0) - - # Allocate KV slots for the request, write ground-truth K/V, then - # commit it as a finished request (inserts into radix + stores to - # LMCache on tree.store_stream). - req_pool_idx = req_to_token_pool.alloc( - [ - SimpleNamespace( - inflight_middle_chunks=0, kv=ReqKvInfo(req_pool_idx=None) - ) - ] - )[0] - kv_slots = allocator.alloc(self.INPUT_LEN) - self.assertIsNotNone(kv_slots) - req_to_token_pool.write((req_pool_idx, slice(0, self.INPUT_LEN)), kv_slots) - - gt_k = [] - gt_v = [] - for layer_id in range(self.model_config.num_hidden_layers): - k = torch.randn( - self.INPUT_LEN, - self.model_config.num_key_value_heads, - self.model_config.head_dim, - dtype=torch.bfloat16, - device=self.DEVICE, - ) - v = torch.randn_like(k) - kv_pool.k_buffer[layer_id][kv_slots] = k - kv_pool.v_buffer[layer_id][kv_slots] = v - gt_k.append(k.clone()) - gt_v.append(v.clone()) - - req = _make_req("req-0", req_pool_idx, token_ids, tree) - tree.cache_finished_req(req, owned_kv_len=len(token_ids)) - # IP-mode store is async on tree.store_stream; evict()'s - # synchronize() is what the real scheduler relies on to make the - # store visible before slots are reused. - tree.evict(EvictParams(num_tokens=0)) - - # Evict everything from the radix tree so the only remaining copy - # of this KV is inside LMCache, forcing a real load-back on the - # next match_prefix. - tree.evict(EvictParams(num_tokens=self.INPUT_LEN)) - self.assertEqual(tree.total_size(), 0) - - for layer_id in range(self.model_config.num_hidden_layers): - kv_pool.k_buffer[layer_id].zero_() - kv_pool.v_buffer[layer_id].zero_() - - reload_res = tree.match_prefix(MatchPrefixParams(key=RadixKey(token_ids))) - self.assertEqual(reload_res.device_indices.numel(), self.INPUT_LEN) - - new_slots = reload_res.device_indices - for layer_id in range(self.model_config.num_hidden_layers): - # get_key_buffer/get_value_buffer (not the raw k_buffer/v_buffer - # list) is what invokes layer_transfer_counter.wait_until — - # the real per-layer forward hook this test exists to cover. - actual_k = kv_pool.get_key_buffer(layer_id)[new_slots] - actual_v = kv_pool.get_value_buffer(layer_id)[new_slots] - self.assertTrue( - torch.allclose(actual_k, gt_k[layer_id]), - f"Layer {layer_id}: K not restored via LMCache load-back", - ) - self.assertTrue( - torch.allclose(actual_v, gt_v[layer_id]), - f"Layer {layer_id}: V not restored via LMCache load-back", - ) - finally: - tree.lmcache_connector.close() - - -if __name__ == "__main__": - unittest.main()