From 4ce23542bfc9efa82487594ef9ac952356032a20 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Tue, 22 Sep 2026 16:17:12 -0700 Subject: [PATCH] [HiCache] Remove the unused HiRadixCache (#40787) --- .github/labeler.yml | 1 - .../hicache_storage_runtime_attach_detach.mdx | 4 +- .../srt/distributed/communication_tags.py | 2 +- .../sglang/srt/managers/cache_controller.py | 4 - python/sglang/srt/mem_cache/README.md | 2 +- .../sglang/srt/mem_cache/base_prefix_cache.py | 4 - .../sglang/srt/mem_cache/hicache_storage.py | 9 - python/sglang/srt/mem_cache/hiradix_cache.py | 2034 ----------------- .../hybrid_cache/hybrid_pool_assembler.py | 124 +- python/sglang/srt/mem_cache/radix_cache.py | 22 - .../srt/mem_cache/unified_radix_cache.py | 8 +- .../test_hicache_load_back_timing.py | 40 +- ...drain.py => test_hicache_pp_sync_drain.py} | 26 +- .../unit/mem_cache/test_hiradix_cache_unit.py | 123 - .../test_minimax_sparse_pool_host_unit.py | 19 - .../unit/mem_cache/test_radix_cache_unit.py | 46 +- .../simulation/sglang/cache_controller.py | 2 +- .../simulation/sglang/hiradix_cache.py | 25 - .../simulation/sglang/hook_bootstrap.py | 2 - 19 files changed, 47 insertions(+), 2450 deletions(-) delete mode 100644 python/sglang/srt/mem_cache/hiradix_cache.py rename test/registered/unit/mem_cache/{test_hiradix_pp_sync_drain.py => test_hicache_pp_sync_drain.py} (87%) delete mode 100644 test/registered/unit/mem_cache/test_hiradix_cache_unit.py delete mode 100644 tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hiradix_cache.py diff --git a/.github/labeler.yml b/.github/labeler.yml index 16ff878feb..377b148ef2 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -124,7 +124,6 @@ hicache: - changed-files: - any-glob-to-any-file: - '**/*hicache*' - - '**/*hiradix*' - '**/HiCache/**/*' - 'benchmark/hicache/**/*' - 'test/registered/hicache/**/*' diff --git a/docs/docs/advanced_features/hicache_storage_runtime_attach_detach.mdx b/docs/docs/advanced_features/hicache_storage_runtime_attach_detach.mdx index 1f36e06776..54da9d4181 100644 --- a/docs/docs/advanced_features/hicache_storage_runtime_attach_detach.mdx +++ b/docs/docs/advanced_features/hicache_storage_runtime_attach_detach.mdx @@ -26,7 +26,7 @@ The control path is: 3. **Scheduler** (`python/sglang/srt/managers/scheduler.py`) - Performs a **strict idle check** - Calls `tree_cache.attach_storage_backend(...)` / `detach_storage_backend(...)` -4. **HiRadixCache** (`python/sglang/srt/mem_cache/hiradix_cache.py`) +4. **UnifiedRadixCache** (`python/sglang/srt/mem_cache/unified_radix_cache.py`) - Parses `hicache_storage_backend_extra_config_json` (supports both backend config and prefetch knobs) - Calls `cache_controller.attach_storage_backend(...)` / `detach_storage_backend(...)` 5. **HiCacheController** (`python/sglang/srt/managers/cache_controller.py`) @@ -130,4 +130,4 @@ Notes: - **Host KV layout constraints still apply**: for example, Mooncake still requires layouts like `page_first/page_first_direct/page_head`; if the server's HiCache host-memory layout does not satisfy the backend requirements, attach will fail with an error - **Observability**: - After attach, `server_args.hicache_storage_backend*` is updated on both the tokenizer and scheduler sides - - If metrics are enabled, attach will create a storage metrics collector in `HiRadixCache` on demand + - If metrics are enabled, attach will create a storage metrics collector in `UnifiedRadixCache` on demand diff --git a/python/sglang/srt/distributed/communication_tags.py b/python/sglang/srt/distributed/communication_tags.py index 9b8cf90e0e..55962e7d69 100644 --- a/python/sglang/srt/distributed/communication_tags.py +++ b/python/sglang/srt/distributed/communication_tags.py @@ -11,5 +11,5 @@ class P2PTag(IntEnum): """ DEFAULT = 0 - HIRADIX_PP_SYNC = int.from_bytes(b"PpHi", byteorder="big") + HICACHE_PP_SYNC = int.from_bytes(b"PpHi", byteorder="big") GRAMMAR_PP_SYNC = int.from_bytes(b"PpGr", byteorder="big") diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index c652c1b099..ba49418fe9 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -987,10 +987,6 @@ class HiCacheController: ) return producer_id - def evict_device(self, device_indices: torch.Tensor) -> int: - self.mem_pool_device_allocator.free(device_indices) - return len(device_indices) - def evict_host(self, host_indices: torch.Tensor, backup_only: bool = True) -> int: if not backup_only: raise ValueError("Other eviction policies are not supported yet.") diff --git a/python/sglang/srt/mem_cache/README.md b/python/sglang/srt/mem_cache/README.md index ea2ff10d4e..bc08fb6dde 100644 --- a/python/sglang/srt/mem_cache/README.md +++ b/python/sglang/srt/mem_cache/README.md @@ -38,7 +38,7 @@ to keep. The layout is specified in Two groups sit outside that stack: - **Radix cache** is its own axis. The per-model variants (`radix_cache.py`, - `hiradix_cache.py`, `chunk_cache.py`) + `chunk_cache.py`) are converging onto the **Unified Radix Cache** (`unified_cache/`, [#20415](https://github.com/sgl-project/sglang/issues/20415)), whose Full/SWA/Mamba component model is documented in diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 81a1a5d95d..a5950ad4f6 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -406,10 +406,6 @@ class BasePrefixCache(ABC, PrefixCacheTrait): implementations that shard trees per cache namespace.""" return self.root_node - def is_backuped(self, node: Any) -> bool: - """Whether the node's Full KV is present on host.""" - return node.backuped - def is_root(self, node: Any) -> bool: """Whether the node is a tree root.""" return node is self.root_node diff --git a/python/sglang/srt/mem_cache/hicache_storage.py b/python/sglang/srt/mem_cache/hicache_storage.py index 04e5ab8c84..1450da14fd 100644 --- a/python/sglang/srt/mem_cache/hicache_storage.py +++ b/python/sglang/srt/mem_cache/hicache_storage.py @@ -48,15 +48,6 @@ class HiCacheStorageExtraInfo: extra_info: Optional[dict] = None -@dataclass(frozen=True) -class PrefetchTimeoutConfig: - """Knobs for the linear prefetch-timeout policy used by HiCache.""" - - base: float = 2.0 # seconds, fixed overhead unrelated to token count - per_ki_token: float = 0.1 # seconds per 1024 tokens - max: float = 30.0 # seconds, upper bound for the linear timeout - - class PoolName(str, Enum): """Well-known pool names used as PoolTransfer/PoolEntry identifiers.""" diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py deleted file mode 100644 index 9c2e95b568..0000000000 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ /dev/null @@ -1,2034 +0,0 @@ -from __future__ import annotations - -import atexit -import heapq -import json -import logging -import os -import queue -import threading -import time -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple - -import torch - -from sglang.srt.disaggregation.kv_events import StorageMedium -from sglang.srt.distributed.communication_tags import P2PTag -from sglang.srt.managers.cache_controller import HiCacheController, PrefetchOperation -from sglang.srt.mem_cache.base_prefix_cache import ( - CacheRequestHandle, - DecLockRefParams, - DecLockRefResult, - EvictParams, - EvictResult, - IncLockRefResult, - InitLoadBackParams, - InsertParams, - InsertResult, - MatchPrefixParams, - MatchResult, -) -from sglang.srt.mem_cache.hicache_storage import ( - PoolHitPolicy, - PoolName, - PoolTransfer, - PrefetchTimeoutConfig, -) -from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( - HybridCacheController, -) -from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( - PrefetchOperation as HybridPrefetchOperation, -) -from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( - attach_hybrid_dsa_pool_to_hiradix_cache, -) -from sglang.srt.mem_cache.memory_pool import ( - DSATokenToKVPool, - MHATokenToKVPool, - MiniMaxSparseKVPool, - MLATokenToKVPool, -) -from sglang.srt.mem_cache.pool_host.common import get_allocator_type -from sglang.srt.mem_cache.pool_host.mha import get_mha_host_pool_cls -from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost -from sglang.srt.mem_cache.radix_cache import ( - RadixCache, - RadixKey, - TreeNode, -) -from sglang.srt.mem_cache.utils import ( - compute_node_hash_values, - split_node_hash_value, -) -from sglang.srt.observability.metrics_collector import ( - STAT_LOGGER_ROLE_STORAGE, - StorageMetricsCollector, - resolve_collector_class, -) -from sglang.srt.runtime_context import ( - get_memory, - get_observability, - get_parallel, - get_serving, -) - -if TYPE_CHECKING: - from sglang.srt.mem_cache.cache_init_params import CacheInitParams - from sglang.srt.server_args import ServerArgs - -logger = logging.getLogger(__name__) - - -class HiRadixCache(RadixCache): - def __init__(self, params: CacheInitParams, server_args: ServerArgs): - self._enable_metrics_flag = params.enable_metrics - - self.page_size = params.page_size - self.kv_cache = params.token_to_kv_pool_allocator.get_kvcache() - - allocator_type = get_allocator_type() - - if isinstance(self.kv_cache, MHATokenToKVPool): - self.token_to_kv_pool_host = get_mha_host_pool_cls(self.kv_cache)( - self.kv_cache, - get_memory().hicache_ratio, - get_memory().hicache_size, - self.page_size, - get_memory().hicache_mem_layout, - allocator_type=allocator_type, - ) - elif isinstance(self.kv_cache, DSATokenToKVPool): - # Filled by attach_hybrid_dsa_pool_to_hiradix_cache after storage extra_config is parsed. - self.token_to_kv_pool_host = None - elif isinstance(self.kv_cache, MiniMaxSparseKVPool): - # Filled by attach_hybrid_minimax_sparse_pool_to_hiradix_cache. - self.token_to_kv_pool_host = None - elif isinstance(self.kv_cache, MLATokenToKVPool): - _parallel = get_parallel() - self.token_to_kv_pool_host = MLATokenToKVPoolHost( - self.kv_cache, - get_memory().hicache_ratio, - get_memory().hicache_size, - self.page_size, - get_memory().hicache_mem_layout, - allocator_type=allocator_type, - dcp_size=_parallel.attn_dcp_size, - dcp_rank=_parallel.attn_dcp_rank, - ) - else: - raise ValueError("HiRadixCache only supports MHA, MLA, DSA, and MSA models") - - self.tp_group = params.tp_cache_group - self.attn_cp_group = params.attn_cp_cache_group - self.attn_tp_group = params.attn_tp_cache_group - self.pp_group = params.pp_cache_group - self.tp_world_size = torch.distributed.get_world_size(group=self.tp_group) - self.pp_rank = params.pp_rank - self.pp_size = params.pp_size - self.enable_storage = get_memory().hicache_storage_backend is not None - self.enable_storage_metrics = self.enable_storage and params.enable_metrics - self.extra_metric_labels = get_observability().extra_metric_labels - - ( - extra_config, - prefetch_threshold, - prefetch_timeout_config, - hicache_storage_pass_prefix_keys, - ) = self._parse_storage_backend_extra_config( - get_memory().hicache_storage_backend_extra_config - ) - # TODO: support more timeout check functions - self.is_prefetch_timeout = self._prefetch_timeout_check_linear_func - self.prefetch_stop_policy = get_memory().hicache_storage_prefetch_policy - - self.load_cache_event = threading.Event() - if isinstance(self.kv_cache, DSATokenToKVPool): - attach_hybrid_dsa_pool_to_hiradix_cache( - self, - params, - extra_config=extra_config, - prefetch_threshold=prefetch_threshold, - enable_storage_metrics=self.enable_storage_metrics, - load_cache_event=self.load_cache_event, - ) - elif isinstance(self.kv_cache, MiniMaxSparseKVPool): - from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( - attach_hybrid_minimax_sparse_pool_to_hiradix_cache, - ) - - attach_hybrid_minimax_sparse_pool_to_hiradix_cache( - self, - params, - extra_config=extra_config, - prefetch_threshold=prefetch_threshold, - enable_storage_metrics=self.enable_storage_metrics, - load_cache_event=self.load_cache_event, - ) - else: - self.cache_controller = HiCacheController( - params.token_to_kv_pool_allocator, - self.token_to_kv_pool_host, - self.page_size, - self.tp_group, - load_cache_event=self.load_cache_event, - attn_cp_group=self.attn_cp_group, - attn_tp_group=self.attn_tp_group, - pp_group=self.pp_group, - write_policy=get_memory().hicache_write_policy, - io_backend=get_memory().hicache_io_backend, - storage_backend=get_memory().hicache_storage_backend, - prefetch_threshold=prefetch_threshold, - model_name=get_serving().served_model_name, - storage_backend_extra_config=extra_config, - enable_storage_metrics=self.enable_storage_metrics, - ) - self._apply_storage_runtime_config( - storage_backend=get_memory().hicache_storage_backend, - prefetch_threshold=prefetch_threshold, - prefetch_timeout_config=prefetch_timeout_config, - hicache_storage_pass_prefix_keys=hicache_storage_pass_prefix_keys, - enable_storage=self.enable_storage, - enable_storage_metrics=self.enable_storage_metrics, - extra_metric_labels=self.extra_metric_labels, - ) - - # record the nodes with ongoing write through - self.ongoing_write_through = {} - # record the node segments with ongoing load back - self.ongoing_load_back = {} - # record the ongoing prefetch requests - self.ongoing_prefetch = {} - self.ongoing_backup = {} - # track per-request tokens loaded from storage (L3 hits) - # key: request_id, value: number of tokens actually loaded from storage - self.prefetch_loaded_tokens_by_reqid: dict[str, int] = {} - self.work_list: List[torch.distributed.Work] = [] - # todo: dynamically adjust the threshold - self.write_through_threshold = ( - 1 if get_memory().hicache_write_policy == "write_through" else 2 - ) - self.load_back_threshold = 10 - # Detach storage backend automatically on process shutdown - atexit.register(self.shutdown) - - self.evictable_host_leaves = set() - - super().__init__(params=params) - - def _all_reduce_attn_groups(self, tensor: torch.Tensor, op): - reduced = False - for group in (self.attn_cp_group, self.attn_tp_group): - if group is not None and torch.distributed.get_world_size(group=group) > 1: - torch.distributed.all_reduce(tensor, op=op, group=group) - reduced = True - if not reduced and self.tp_world_size > 1: - torch.distributed.all_reduce(tensor, op=op, group=self.tp_group) - - def _drain_async_work(self): - """ - Block until all outstanding async sends are consumed, then clear. - - Called at the start of each event round, so work_list holds the sends - accumulated since the last round. This bounds it and applies - backpressure when a downstream PP rank lags. Scheduler thread only. - """ - for work in self.work_list: - work.wait() - self.work_list.clear() - - def _all_reduce(self, data: torch.Tensor, tp_reduce_op: torch.distributed.ReduceOp): - """ - Synchronize data across all TP and PP ranks. - - In particular, "tp_reduce_op" is performed on all TP ranks of the first PP rank, - and then the result is propagated to all following PP ranks. - - Must be called in the scheduler thread. - """ - if self.pp_rank == 0: - self._all_reduce_attn_groups(data, tp_reduce_op) - self._pp_sync(data) - - def _pp_sync(self, data: torch.Tensor) -> None: - """ - Synchronize data across the PP pipeline, where PPn (n>0) will receive PP0's data. - - The following diagram illustrates the behavior of _pp_sync. - - time | pp0 | pp1 | pp2 - ------|-------------------------|-------------------------|----------------------------- - 0 | _pp_sync(data=1) starts | _pp_sync(data=?) starts | _pp_sync(data=?) starts - 1 | _pp_sync(data=1) ends | | - 2 | | _pp_sync(data=1) ends | - 3 | | | _pp_sync(data=1) ends - - _pp_sync requires no synchronization point among ranks. The following case may also happen. - - time | pp0 | pp1 | pp2 - ------|-------------------------|-------------------------|----------------------------- - 0 | _pp_sync(data=1) starts | | - 1 | _pp_sync(data=1) ends | | - 2 | | _pp_sync(data=?) starts | - 3 | | _pp_sync(data=1) ends | - 4 | | | _pp_sync(data=?) starts - 5 | | | _pp_sync(data=1) ends - """ - if self.pp_size <= 1 or self.pp_group is None: - return - if self.pp_rank > 0: - torch.distributed.recv( - data, - group_src=self.pp_rank - 1, - group=self.pp_group, - tag=P2PTag.HIRADIX_PP_SYNC, - ) - if self.pp_rank + 1 < self.pp_size: - # Make a copy of data, so that the caller is safe to modify `data` after this call. - # This is cheap, as _pp_sync is not to be used for transmitting large data. - copy_of_data = data.clone() - send_work = torch.distributed.isend( - copy_of_data, - group_dst=self.pp_rank + 1, - group=self.pp_group, - tag=P2PTag.HIRADIX_PP_SYNC, - ) - self.work_list.append(send_work) - - def shutdown(self): - """Best-effort auto-detach of storage backend on process shutdown. - - This keeps startup and runtime behavior consistent: if a backend was attached - (either via CLI args or via admin API), we attempt to detach it on exit. - """ - try: - if self.enable_storage: - self.detach_storage_backend() - except Exception: - logger.exception("Failed to detach storage backend on process shutdown.") - - def _apply_storage_runtime_config( - self, - *, - storage_backend: Optional[str], - prefetch_threshold: int, - prefetch_timeout_config: PrefetchTimeoutConfig, - hicache_storage_pass_prefix_keys: bool, - enable_storage: bool, - enable_storage_metrics: bool, - extra_metric_labels: Optional[Dict[str, str]], - ) -> None: - self.enable_storage = enable_storage - self.prefetch_threshold = prefetch_threshold - self.prefetch_timeout_config = prefetch_timeout_config - self.hicache_storage_pass_prefix_keys = hicache_storage_pass_prefix_keys - self.enable_storage_metrics = enable_storage_metrics - - if self.enable_storage_metrics: - attn_cp_rank, attn_cp_size = ( - self.cache_controller.get_attn_cp_rank_and_size() - ) - labels = { - "storage_backend": storage_backend, - "tp_rank": self.cache_controller.tp_rank, - "dp_rank": self.cache_controller.dp_rank, - "pp_rank": self.cache_controller.pp_rank, - "pp_size": self.cache_controller.pp_size, - "attn_cp_rank": attn_cp_rank, - "attn_cp_size": attn_cp_size, - } - if extra_metric_labels: - labels.update(extra_metric_labels) - existing_collector = getattr(self, "storage_metrics_collector", None) - if existing_collector is None: - storage_cls = resolve_collector_class( - STAT_LOGGER_ROLE_STORAGE, - StorageMetricsCollector, - ) - self.storage_metrics_collector = storage_cls(labels=labels) - elif set(existing_collector.labels.keys()) == set(labels.keys()): - existing_collector.labels = labels - else: - logger.warning( - "Storage metrics labels changed (%s -> %s). Keep existing labels to " - "avoid duplicate metric registration.", - sorted(existing_collector.labels.keys()), - sorted(labels.keys()), - ) - - def attach_storage_backend( - self, - storage_backend: str, - storage_backend_extra_config_json: Optional[str] = None, - served_model_name: Optional[str] = None, - hicache_storage_prefetch_policy: Optional[str] = None, - hicache_write_policy: Optional[str] = None, - ) -> tuple[bool, str]: - """Attach (enable) storage backend at runtime. - - This will start storage threads inside `HiCacheController` and enable - prefetch/backup paths. Caller must ensure there are no running/queued - requests to avoid races. - """ - # Validate inputs first (no side effects). - if hicache_storage_prefetch_policy is not None: - allowed = ["best_effort", "wait_complete", "timeout"] - if hicache_storage_prefetch_policy not in allowed: - return ( - False, - f"Invalid hicache_storage_prefetch_policy: {hicache_storage_prefetch_policy!r}. " - f"Expected one of {allowed}.", - ) - - if hicache_write_policy is not None: - allowed = ["write_back", "write_through", "write_through_selective"] - if hicache_write_policy not in allowed: - return ( - False, - f"Invalid hicache_write_policy: {hicache_write_policy!r}. " - f"Expected one of {allowed}.", - ) - - # If already enabled: - # - backend unchanged: treat as success, update policies only. - # - backend changed: treat as failure, do NOT update policies. - if self.enable_storage: - current_backend = self.cache_controller.storage_backend_type - - if current_backend == storage_backend: - if hicache_storage_prefetch_policy is not None: - self.prefetch_stop_policy = hicache_storage_prefetch_policy - logger.info( - f"Set hicache_storage_prefetch_policy to {hicache_storage_prefetch_policy}" - ) - if hicache_write_policy is not None: - self.cache_controller.write_policy = hicache_write_policy - self.write_through_threshold = ( - 1 if hicache_write_policy == "write_through" else 2 - ) - logger.info(f"Set hicache_write_policy to {hicache_write_policy}") - return ( - True, - "HiCache storage backend already enabled with same backend; policies updated.", - ) - - return ( - False, - f"HiCache storage backend is already enabled with backend '{current_backend}'. " - f"Cannot attach different backend '{storage_backend}'. Detach first.", - ) - - # Not enabled: update policies before controller attach so storage threads observe new values. - if hicache_storage_prefetch_policy is not None: - self.prefetch_stop_policy = hicache_storage_prefetch_policy - logger.info( - f"Set hicache_storage_prefetch_policy to {hicache_storage_prefetch_policy}" - ) - - if hicache_write_policy is not None: - self.cache_controller.write_policy = hicache_write_policy - self.write_through_threshold = ( - 1 if hicache_write_policy == "write_through" else 2 - ) - logger.info(f"Set hicache_write_policy to {hicache_write_policy}") - - logger.info(f"Attaching HiCache storage backend: {storage_backend}") - try: - ( - extra_config, - prefetch_threshold, - prefetch_timeout_config, - hicache_storage_pass_prefix_keys, - ) = self._parse_storage_backend_extra_config( - storage_backend_extra_config_json - ) - except Exception as e: - logger.exception(f"Failed to parse storage_backend_extra_config_json: {e}") - return ( - False, - f"Failed to parse storage_backend_extra_config_json '{storage_backend_extra_config_json}': {e}", - ) - - try: - self.cache_controller.attach_storage_backend( - storage_backend=storage_backend, - prefetch_threshold=prefetch_threshold, - model_name=served_model_name, - storage_backend_extra_config=extra_config, - **self._get_hybrid_storage_attach_kwargs(), - ) - except Exception as e: - logger.exception( - f"Failed to attach storage backend '{storage_backend}': {e}" - ) - return False, f"Failed to attach storage backend '{storage_backend}': {e}" - - self._apply_storage_runtime_config( - storage_backend=storage_backend, - prefetch_threshold=prefetch_threshold, - prefetch_timeout_config=prefetch_timeout_config, - hicache_storage_pass_prefix_keys=hicache_storage_pass_prefix_keys, - enable_storage=True, - enable_storage_metrics=self._enable_metrics_flag, - extra_metric_labels=self.extra_metric_labels, - ) - return True, "Attached HiCache storage backend successfully." - - def detach_storage_backend(self) -> tuple[bool, str]: - """Detach (disable) storage backend at runtime. - - Caller must ensure there are no running/queued requests to avoid races. - """ - try: - # Drain any pending control queues before tearing down storage threads/backend. - # IMPORTANT: this must happen before we clear `ongoing_*`, otherwise acks/releases - # cannot be matched to nodes and may leak host pages / locks. - self._drain_storage_control_queues_local() - # Idempotent detach: always ask controller to best-effort cleanup, even if - # `self.enable_storage` is already False (may be leftover state from a - # previous partial detach). - self.cache_controller.detach_storage_backend() - except Exception as e: - logger.exception("Failed to detach storage backend.") - # Do NOT crash the server for admin operations. Return failure with detail. - return False, f"Failed to detach HiCache storage backend: {e}" - - # Best-effort cleanup of any leftover bookkeeping. - self._drain_storage_control_queues_local() - # After controller threads are fully stopped, it's safe to force-release any - # leftover pending ops (e.g., async prefetch/backup that didn't get a revoke/ack). - self._force_release_pending_storage_ops() - - self.enable_storage = False - self.enable_storage_metrics = False - return True, "Detached HiCache storage backend successfully." - - def _force_release_pending_storage_ops(self): - """Force release any leftover pending prefetch/backup bookkeeping. - - This is a safety net for detach/shutdown paths. It assumes storage threads - have been stopped already (via controller.detach), so no concurrent access - to these structures should happen. - """ - cc = self.cache_controller - - # Force release leftover prefetch ops: free pre-allocated host pages and - # drop the host protection on the matched prefix node. - try: - for req_id, info in list(self.ongoing_prefetch.items()): - try: - last_host_node, prefetch_key, _operation = info - except Exception: - # Unexpected shape; just drop it. - self.ongoing_prefetch.pop(req_id, None) - continue - - try: - if _operation.host_indices is not None: - cc.mem_pool_host.free(_operation.host_indices) - except Exception: - logger.exception( - "Failed to free host indices for prefetch %s", req_id - ) - - try: - last_host_node.release_host() - except Exception: - logger.exception( - "Failed to release host protection for prefetch %s", req_id - ) - - try: - cc.prefetch_tokens_occupied -= len(prefetch_key) - if cc.prefetch_tokens_occupied < 0: - cc.prefetch_tokens_occupied = 0 - except Exception: - pass - - self.ongoing_prefetch.pop(req_id, None) - except Exception: - logger.exception("Force release pending prefetch ops failed.") - - # Force release leftover backup ops: drop host protection on nodes. - try: - for ack_id, node in list(self.ongoing_backup.items()): - try: - node.release_host() - except Exception: - logger.exception( - "Failed to release host protection for backup op %s", ack_id - ) - self.ongoing_backup.pop(ack_id, None) - except Exception: - logger.exception("Force release pending backup ops failed.") - - def _drain_storage_control_queues_local(self): - """Drain storage control queues without TP synchronization. - - This is intended for shutdown/detach paths where we want to make best-effort - cleanup even if queue sizes temporarily differ across ranks. - """ - self._drain_storage_control_queues_impl( - n_storage_hit=0, - n_ack_prefetch=None, - n_backup=None, - n_release=None, - log_metrics=False, - ) - - def _drain_storage_control_queues_impl( - self, - n_storage_hit: Optional[int], - n_ack_prefetch: Optional[int], - n_backup: Optional[int], - n_release: Optional[int], - log_metrics: bool, - ): - cc = self.cache_controller - - def _drain_queue(q: queue.Queue, n: Optional[int]): - """If n is None, consume all items from the queue. - Otherwise, consume n items from the queue. - """ - if n is None: - while not q.empty(): - item = q.get() - yield item - else: - for _ in range(n): - # Block when there is no enough elements. - # All TP/PP ranks must consume the same number of elements. - item = q.get() - yield item - - def _drain_and_alloc_storage_hit(): - # The L3 hit count is now known, so reserve exactly that much host - # memory (this is the whole point: no over-allocation up front). - # NOTE: alloc/evict here is rank-local but deterministic across TP - # ranks without extra synchronization: host pool mutations only - # happen on the scheduler thread at lockstep points (releases are - # page-granular and drained by the TP-min count), so every rank - # reaches the same success / fallback / revoke decision. - for operation in _drain_queue(cc.prefetch_hit_queue, n_storage_hit): - req_id = operation.request_id - info = self.ongoing_prefetch.get(req_id) - if info is None: - # request already aborted/cleaned up, skip - continue - if operation.is_terminated(): - # request was aborted while the storage query was in flight - self._revoke_pending_prefetch(req_id) - continue - if operation.storage_hit_count < self.prefetch_threshold: - # not to prefetch if not enough benefits - self._revoke_pending_prefetch(req_id) - logger.debug( - f"Revoking prefetch for request {req_id} due to insufficient hits ({operation.storage_hit_count})." - ) - continue - - alloc_len = operation.storage_hit_count - host_indices = cc.mem_pool_host.alloc(alloc_len) - if host_indices is None: - self.evict_host(alloc_len) - host_indices = cc.mem_pool_host.alloc(alloc_len) - if host_indices is None: - # Memory-pressure fallback: a shorter page-aligned prefix. - available_size = cc.mem_pool_host.available_size() - alloc_len = min( - operation.storage_hit_count, - available_size - (available_size % self.page_size), - ) - if alloc_len >= self.prefetch_threshold: - host_indices = cc.mem_pool_host.alloc(alloc_len) - if host_indices is None: - self._revoke_pending_prefetch(req_id) - logger.debug( - f"Revoking prefetch for request {req_id} due to host memory allocation failure." - ) - continue - - operation.storage_hit_count = alloc_len - operation.hash_value = operation.hash_value[ - : alloc_len // self.page_size - ] - operation.host_indices = host_indices - cc.prefetch_buffer.put(operation) - - def _drain_ack_prefetch(): - for ack in _drain_queue(cc.ack_prefetch_queue, n_ack_prefetch): - operation = ack.operation - if ack.completed_tokens is not None: - if operation.request_id in self.ongoing_prefetch: - assert operation.completed_tokens <= ack.completed_tokens - operation.completed_tokens = ack.completed_tokens - if ack.pool_hits is not None: - if operation.request_id in self.ongoing_prefetch: - operation.pool_storage_result.update_extra_pool_hit_pages( - ack.pool_hits - ) - operation.pool_transfers_done = True - if ack.completed_req: - if operation.request_id in self.ongoing_prefetch: - self._handle_prefetch_result(operation) - tail = operation.host_indices[operation.completed_tokens :] - self.cache_controller.mem_pool_host.free(tail) - - def _drain_backup(): - for operation in _drain_queue(cc.ack_backup_queue, n_backup): - ack_id = operation.id - entry = self.ongoing_backup.pop(ack_id, None) - if entry is not None: - entry.release_host() - if log_metrics and self.enable_storage_metrics: - self.storage_metrics_collector.log_backuped_tokens( - operation.completed_tokens - ) - - def _drain_release(): - host_indices_list = [] - for host_indices in _drain_queue(cc.host_mem_release_queue, n_release): - host_indices_list.append(host_indices) - if host_indices_list: - host_indices = torch.cat(host_indices_list, dim=0) - cc.mem_pool_host.free(host_indices) - - _drain_and_alloc_storage_hit() - _drain_ack_prefetch() - _drain_backup() - _drain_release() - - def _parse_storage_backend_extra_config( - self, storage_backend_extra_config: Optional[str] - ): - """ - Parse storage backend extra config JSON and extract specific parameters. - - Args: - storage_backend_extra_config: JSON string containing extra configuration - - Returns: - tuple: (extra_config_dict, prefetch_threshold, prefetch_timeout_config, hicache_storage_pass_prefix_keys) - """ - # Parse extra config if provided. Extra config can be a JSON string or a json/toml/yaml file path prefixed with "@". - extra_config = {} - if storage_backend_extra_config: - try: - if storage_backend_extra_config.startswith("@"): - # Read config from a json/toml/yaml file - path = storage_backend_extra_config[1:] - ext = os.path.splitext(path)[1].lower() - with open(path, "rb" if ext == ".toml" else "r") as f: - if ext == ".json": - extra_config = json.load(f) - elif ext == ".toml": - import tomllib - - extra_config = tomllib.load(f) - elif ext in (".yaml", ".yml"): - import yaml - - extra_config = yaml.safe_load(f) - else: - raise ValueError( - f"Unsupported config file {path} (config format: {ext})" - ) - else: - # read config from JSON string - extra_config = json.loads(storage_backend_extra_config) - except Exception as e: - logger.error(f"Invalid backend extra config JSON: {e}") - raise e - - defaults = PrefetchTimeoutConfig() - prefetch_threshold = extra_config.pop("prefetch_threshold", 256) # tokens - prefetch_timeout_base = extra_config.pop( - "prefetch_timeout_base", defaults.base - ) # seconds - prefetch_timeout_per_ki_token = extra_config.pop( - "prefetch_timeout_per_ki_token", defaults.per_ki_token - ) # seconds per 1024 tokens - prefetch_timeout_max = extra_config.pop( - "prefetch_timeout_max", defaults.max - ) # seconds, upper bound for the linear timeout - hicache_storage_pass_prefix_keys = extra_config.pop( - "hicache_storage_pass_prefix_keys", False - ) - - if not isinstance(prefetch_threshold, int): - raise ValueError( - f"prefetch_threshold must be int, got {type(prefetch_threshold).__name__}" - ) - if not isinstance(prefetch_timeout_base, (int, float)): - raise ValueError( - f"prefetch_timeout_base must be number, got {type(prefetch_timeout_base).__name__}" - ) - if not isinstance(prefetch_timeout_per_ki_token, (int, float)): - raise ValueError( - f"prefetch_timeout_per_ki_token must be number, got {type(prefetch_timeout_per_ki_token).__name__}" - ) - if not isinstance(prefetch_timeout_max, (int, float)): - raise ValueError( - f"prefetch_timeout_max must be number, got {type(prefetch_timeout_max).__name__}" - ) - if not isinstance(hicache_storage_pass_prefix_keys, bool): - raise ValueError( - "hicache_storage_pass_prefix_keys must be bool, got " - f"{type(hicache_storage_pass_prefix_keys).__name__}" - ) - - prefetch_timeout_config = PrefetchTimeoutConfig( - base=float(prefetch_timeout_base), - per_ki_token=float(prefetch_timeout_per_ki_token), - max=float(prefetch_timeout_max), - ) - - return ( - extra_config, - prefetch_threshold, - prefetch_timeout_config, - hicache_storage_pass_prefix_keys, - ) - - def reset(self): - TreeNode.counter = 0 - self.cache_controller.reset() - self.token_to_kv_pool_host.clear() - # Clear per-request tracking dicts - self.prefetch_loaded_tokens_by_reqid.clear() - self.evictable_host_leaves.clear() - super().reset() - - def release_host_resources(self) -> None: - if self.token_to_kv_pool_host is not None: - self.token_to_kv_pool_host.destroy() - - def get_height(self, node: TreeNode): - height = 0 - while node != self.root_node: - node = node.parent - height += 1 - return height - - def _get_extra_pools(self) -> dict: - if not isinstance(self.cache_controller, HybridCacheController): - return {} - if isinstance(self.kv_cache, DSATokenToKVPool) or ( - isinstance(self.kv_cache, MiniMaxSparseKVPool) - and self.kv_cache.index_k_pool is not None - ): - pool = PoolTransfer( - name=PoolName.INDEXER, - hit_policy=PoolHitPolicy.ALL_PAGES, - indices_from_pool=PoolName.KV, - ) - return {"extra_pools": [pool]} - else: - return {} - - def _get_hybrid_storage_attach_kwargs(self) -> dict: - """Extra kwargs for attach_storage_backend when controller is HybridCacheController.""" - if isinstance(self.cache_controller, HybridCacheController): - return {"host_pools": self.cache_controller.mem_pool_host.entries} - return {} - - def clear_storage_backend(self) -> bool: - if self.enable_storage: - try: - # Check if the storage backend has a clear method (for nixl backends) - if hasattr(self.cache_controller.storage_backend, "clear"): - self.cache_controller.storage_backend.clear() - logger.info( - "Hierarchical cache storage backend cleared successfully!" - ) - return True - else: - logger.warning( - f"Storage backend {type(self.cache_controller.storage_backend).__name__} does not support clear operation." - ) - return False - except Exception as e: - logger.error(f"Failed to clear hierarchical cache storage backend: {e}") - return False - else: - logger.warning("Hierarchical cache storage backend is not enabled.") - return False - - def write_backup(self, node: TreeNode, write_back=False) -> int: - # Backup invariant (for write-through mode): backed-up nodes must form a - # contiguous prefix from root — no gaps. Skip if parent isn't backed - # up yet; - if not write_back and ( - node.parent != self.root_node and not node.parent.backuped - ): - return 0 - - host_indices = self.cache_controller.write( - device_indices=node.value, - node_id=node.id, - **self._get_extra_pools(), - ) - if host_indices is None: - self.evict_host(len(node.value)) - host_indices = self.cache_controller.write( - device_indices=node.value, - node_id=node.id, - **self._get_extra_pools(), - ) - if host_indices is not None: - node.host_value = host_indices.clone() - assert len(node.host_value) > 0 - self._track_write_through_node(node, len(node.key)) - if not write_back: - self.inc_lock_ref(node) - else: - return 0 - - return len(host_indices) - - def _track_write_through_node(self, node: TreeNode, backup_len: int) -> None: - node.write_through_pending_id = node.id - self.ongoing_write_through[node.id] = (node, backup_len, [node]) - - def _replace_pending_write_through_node( - self, old_node: TreeNode, new_nodes: List[TreeNode] - ) -> None: - ack_id = old_node.write_through_pending_id - if ack_id is None: - return - - pending = self.ongoing_write_through.get(ack_id) - if pending is None: - return - - lock_node, backup_len, publish_nodes = pending - updated_nodes = [] - replaced = False - for node in publish_nodes: - if node is old_node: - updated_nodes.extend(new_nodes) - replaced = True - else: - updated_nodes.append(node) - - if not replaced: - return - - for node in new_nodes: - node.write_through_pending_id = ack_id - self.ongoing_write_through[ack_id] = (lock_node, backup_len, updated_nodes) - - def _finish_write_through_ack(self, ack_id: int, *, release_lock: bool) -> None: - lock_node, backup_len, publish_nodes = self.ongoing_write_through.pop(ack_id) - for node in publish_nodes: - if node.write_through_pending_id == ack_id: - node.write_through_pending_id = None - # DMA confirmed -- block is now on host. - self.kv_events.record_store(node, medium=StorageMedium.CPU) - if self.enable_storage: - self.write_backup_storage(lock_node, backup_len) - if release_lock: - self.dec_lock_ref(lock_node) - - def write_backup_storage(self, node: TreeNode, backup_len: Optional[int] = None): - # Recover pre-split data via walk-and-concat if node was split. - # prefix_keys anchored at chain top to avoid double-counting. - if backup_len is None or len(node.key) == backup_len: - top, key, hash_value, host_value = ( - node, - node.key, - node.hash_value, - node.host_value, - ) - else: - top, key, hash_value, host_value = self._concat_split_chain( - node, backup_len - ) - - prefix_keys = ( - top.get_prefix_hash_values(top.parent) - if self.hicache_storage_pass_prefix_keys - else None - ) - - operation_id = self.cache_controller.write_storage( - host_value, key, hash_value, prefix_keys, **self._get_extra_pools() - ) - self.ongoing_backup[operation_id] = node - node.protect_host() - - def _concat_split_chain(self, node: TreeNode, backup_len: int): - """Recover enqueue-time key/hash/host by walking the split chain.""" - chain, accumulated = [], 0 - current = node - while current is not self.root_node and accumulated < backup_len: - chain.append(current) - accumulated += len(current.key) - current = current.parent - assert accumulated == backup_len, ( - f"backup chain length mismatch for node {node.id}: " - f"expected {backup_len}, got {accumulated}" - ) - chain.reverse() # parent-first - top = chain[0] - if top.key.is_bigram: - # Bigram segments share boundary tokens; drop overlap after first. - token_ids = list(chain[0].key.token_ids) - for n in chain[1:]: - token_ids.extend(n.key.token_ids[1:]) - else: - token_ids = [] - for n in chain: - token_ids.extend(n.key.token_ids) - key = RadixKey( - token_ids, - top.key.extra_key, - top.key.is_bigram, - cache_salt=top.key.cache_salt, - ) - - if all(n.hash_value is not None for n in chain): - hash_value = [] - for n in chain: - hash_value.extend(n.hash_value) - else: - hash_value = None - host_value = torch.cat([n.host_value for n in chain]) - return top, key, hash_value, host_value - - def _inc_hit_count(self, node: TreeNode, chunked=False): - # skip the hit count update for chunked requests - if self.cache_controller.write_policy == "write_back" or chunked: - return - node.hit_count += 1 - - if not node.backuped: - if node.hit_count >= self.write_through_threshold: - # write to host if the node is not backuped - self.write_backup(node) - - def _count_ready_acks(self, ack_queue) -> int: - ready_count = 0 - for ack in ack_queue: - if not ack.finish_event.query(): - break - ready_count += 1 - return ready_count - - def _sync_hicache_ready_counts(self) -> tuple[int, int, tuple[int, ...]]: - cache_controller = self.cache_controller - storage_queue_sizes = ( - ( - cache_controller.prefetch_hit_queue.qsize(), - cache_controller.ack_prefetch_queue.qsize(), - cache_controller.ack_backup_queue.qsize(), - cache_controller.host_mem_release_queue.qsize(), - ) - if self.enable_storage - else () - ) - - ready_counts = torch.tensor( - [ - self._count_ready_acks(cache_controller.ack_write_queue), - self._count_ready_acks(cache_controller.ack_load_queue), - *storage_queue_sizes, - ], - dtype=torch.int, - device="cpu", - ) - self._all_reduce(ready_counts, torch.distributed.ReduceOp.MIN) - - count_values = list(map(int, ready_counts.tolist())) - return count_values[0], count_values[1], tuple(count_values[2:]) - - def writing_check(self, write_back=False, finish_count: Optional[int] = None): - if write_back: - # blocking till all write back complete - while len(self.ongoing_write_through) > 0: - for ack in self.cache_controller.ack_write_queue: - ack.finish_event.synchronize() - for ack_id in ack.node_ids: - self._finish_write_through_ack(ack_id, release_lock=False) - self._log_write_ack_metrics(ack) - self.cache_controller.ack_write_queue.clear() - assert len(self.ongoing_write_through) == 0 - return - - if finish_count is None: - # Every rank must enter the all_reduce below; ongoing_write_through can - # diverge across ranks (e.g. write_backup returning 0 on a subset under - # host memory pressure), so a conditional skip desyncs the NCCL op - # sequence and deadlocks under TP > 1. (Matches UnifiedRadixCache.) - finish_count = 0 - if self.pp_rank == 0: - finish_count = self._count_ready_acks( - self.cache_controller.ack_write_queue - ) - finish_count_tensor = torch.tensor( - finish_count, dtype=torch.int, device="cpu" - ) - self._all_reduce(finish_count_tensor, torch.distributed.ReduceOp.MIN) - finish_count = finish_count_tensor.item() - - if finish_count > 0: - logger.debug(f"Process {finish_count} write back operations") - while finish_count > 0: - ack = self.cache_controller.ack_write_queue.pop(0) - ack.finish_event.synchronize() - for ack_id in ack.node_ids: - self._finish_write_through_ack(ack_id, release_lock=True) - self._log_write_ack_metrics(ack) - finish_count -= 1 - - def _log_write_ack_metrics(self, ack) -> None: - """Record D->H backup volume and duration for a completed write ack.""" - if self.metrics_collector is None: - return - for pool, num_tokens in (ack.num_tokens_by_pool or {}).items(): - if num_tokens > 0: - self.metrics_collector.increment_backup_num_tokens( - num_tokens=num_tokens, pool=pool - ) - if ack.num_bytes > 0: - self.metrics_collector.increment_backup_num_bytes(ack.num_bytes) - if ack.timing_enabled: - duration_ms = ack.start_event.elapsed_time(ack.finish_event) - self.metrics_collector.observe_backup_duration(duration_ms / 1000.0) - - def loading_check(self, finish_count: Optional[int] = None): - if finish_count is None: - finish_count = 0 - if self.pp_rank == 0: - finish_count = self._count_ready_acks( - self.cache_controller.ack_load_queue - ) - finish_count_tensor = torch.tensor( - finish_count, dtype=torch.int, device="cpu" - ) - self._all_reduce(finish_count_tensor, torch.distributed.ReduceOp.MIN) - finish_count = finish_count_tensor.item() - - if finish_count > 0: - logger.debug(f"Process {finish_count} load operations") - while finish_count > 0: - ack = self.cache_controller.ack_load_queue.pop(0) - ack.finish_event.synchronize() - for ack_id in ack.node_ids: - end_node = self.ongoing_load_back.pop(ack_id) - self.dec_lock_ref(end_node) - - if self.metrics_collector is not None: - for pool, num_tokens in (ack.num_tokens_by_pool or {}).items(): - if num_tokens > 0: - self.metrics_collector.increment_load_back_num_tokens( - num_tokens=num_tokens, pool=pool - ) - if ack.num_bytes > 0: - self.metrics_collector.increment_load_back_num_bytes(ack.num_bytes) - if ack.timing_enabled: - duration_ms = ack.start_event.elapsed_time(ack.finish_event) - self.metrics_collector.observe_load_back_duration( - duration_ms / 1000.0 - ) - finish_count -= 1 - - def is_load_back_event_done(self, consumer_index: int) -> bool: - """Return True after the local load-back event is complete.""" - if consumer_index < 0: - return True - - finish_event = self.cache_controller.layer_done_counter.events[ - consumer_index - ].finish_event - if not finish_event.query(): - return False - - self.loading_check() - return True - - def evictable_size(self): - return self.evictable_size_ - - def inc_lock_ref(self, node: TreeNode) -> IncLockRefResult: - if self.disable: - return IncLockRefResult(delta=0) - - delta = 0 - while node != self.root_node: - if node.lock_ref == 0: - self.evictable_size_ -= len(node.key) - self.protected_size_ += len(node.key) - delta -= len(node.key) - node.lock_ref += 1 - self._update_leaf_status(node) - self._update_host_leaf_status(node) - node = node.parent - return IncLockRefResult(delta=delta) - - def dec_lock_ref( - self, node: TreeNode, params: Optional[DecLockRefParams] = None - ) -> DecLockRefResult: - if self.disable: - return DecLockRefResult(delta=0) - - delta = 0 - while node != self.root_node: - if node.lock_ref == 1: - self.evictable_size_ += len(node.key) - self.protected_size_ -= len(node.key) - delta += len(node.key) - node.lock_ref -= 1 - self._update_leaf_status(node) - self._update_host_leaf_status(node) - if node.parent is None: - assert node is self.root_node, ( - f"This request holds the node from another tree" - ) - node = node.parent - return DecLockRefResult(delta=delta) - - def _update_host_leaf_status(self, node: TreeNode): - if not node.evicted or node.lock_ref > 0: - if node in self.evictable_host_leaves: - self.evictable_host_leaves.remove(node) - return - - for child in node.children.values(): - if child.backuped: - if node in self.evictable_host_leaves: - self.evictable_host_leaves.remove(node) - return - - if node not in self.evictable_host_leaves: - self.evictable_host_leaves.add(node) - - def evict(self, params: EvictParams) -> EvictResult: - start_time = time.perf_counter() - num_tokens = params.num_tokens - if self.cache_controller.write_policy == "write_back": - num_evicted = self._evict_write_back(num_tokens) - else: - num_evicted = self._evict_write_through(num_tokens) - self.update_eviction_metrics(num_evicted, start_time) - return EvictResult(num_tokens_evicted=num_evicted) - - def _make_eviction_heap(self): - heap = [ - (self.eviction_strategy.get_priority(node), node) - for node in self.evictable_leaves - ] - heapq.heapify(heap) - return heap - - def _promote_parent(self, node: TreeNode, heap) -> None: - # Once all of a node's children are evicted, it becomes a device leaf. - p = node.parent - if p is not self.root_node and all(c.evicted for c in p.children.values()): - heapq.heappush(heap, (self.eviction_strategy.get_priority(p), p)) - - def _evict_write_through(self, num_tokens: int) -> int: - """write_through / write_through_selective: drop non-backuped leaves, - demote already-backuped ones. Nothing is staged to host during eviction, - so this is a plain on-the-fly pass. - """ - heap = self._make_eviction_heap() - num_evicted = 0 - while num_evicted < num_tokens and heap: - _, x = heapq.heappop(heap) - if x.lock_ref > 0: - continue - if x.backuped: - num_evicted += self._evict_backuped(x) - else: - num_evicted += self._evict_regular(x) - self._promote_parent(x, heap) - return num_evicted - - def _evict_write_back(self, num_tokens: int) -> int: - """eviction for write_back mode: demote already-backuped leaves, stage non-backuped ones to host if possible, otherwise drop them. - note this path will be deprecated in the future. - """ - heap = self._make_eviction_heap() - num_evicted = 0 - staged: List[Tuple[TreeNode, torch.Tensor]] = [] - - def flush_staged() -> None: - if not staged: - return - self.writing_check(write_back=True) - for node, device_indices in staged: - self.cache_controller.evict_device(device_indices) - node.release_host() - staged.clear() - - while num_evicted < num_tokens and heap: - _, x = heapq.heappop(heap) - if x.lock_ref > 0: - continue - if x.backuped: - num_evicted += self._evict_backuped(x) - elif self.write_backup(x, write_back=True) > 0: - x.protect_host() - staged.append((x, x.value)) - num_evicted += self._detach_backuped(x) - else: - flush_staged() - num_evicted += self._drop_subtree_no_host(x) - self._promote_parent(x, heap) - flush_staged() - return num_evicted - - def _detach_backuped(self, node: TreeNode) -> int: - # detach nodes from tree while keeping device slots, for write-back eviction - self.kv_events.record_remove(node, medium=StorageMedium.GPU) - num_evicted = len(node.value) - assert num_evicted > 0 - self.evictable_size_ -= num_evicted - node.value = None - self._update_leaf_status(node) - self._update_host_leaf_status(node) - # update leaf status for the parent because the node is evicted - self._update_leaf_status(node.parent) - return num_evicted - - def _evict_backuped(self, node: TreeNode): - device_indices = node.value - num_evicted = self._detach_backuped(node) - self.cache_controller.evict_device(device_indices) - return num_evicted - - def _evict_regular(self, node: TreeNode): - # evict a node not initiated write to host -- emit BlockRemoved - assert len(node.children) == 0, f"non-leaf, {node.id=}" - - self.kv_events.record_remove(node) - self.cache_controller.mem_pool_device_allocator.free(node.value) - num_evicted = len(node.value) - self._delete_leaf(node) - return num_evicted - - def _drop_subtree_no_host(self, root: TreeNode) -> int: - nodes = [] - stack = [root] - while stack: - n = stack.pop() - nodes.append(n) - stack.extend(n.children.values()) - - if any(n.host_ref_counter > 0 for n in nodes): - return 0 - - logger.warning( - "write_back: KV cache on device are dropped without backup due to host memory pressure, subtree root %d, num_nodes %d", - root.id, - len(nodes), - ) - - freed_device = 0 - for n in nodes: - if n.host_value is not None: - self.kv_events.record_remove(n, medium=StorageMedium.CPU) - self.cache_controller.evict_host(n.host_value) - n.host_value = None - if n.value is not None: - self.kv_events.record_remove(n, medium=StorageMedium.GPU) - self.cache_controller.mem_pool_device_allocator.free(n.value) - freed_device += len(n.value) - self.evictable_size_ -= len(n.value) - n.value = None - self.ongoing_write_through.pop(n.id, None) - self.evictable_leaves.discard(n) - self.evictable_host_leaves.discard(n) - - key = root.key.child_key(self.page_size) - root.parent.children.pop(key, None) - self._update_leaf_status(root.parent) - self._update_host_leaf_status(root.parent) - if freed_device > 0 and self.metrics_collector is not None: - self.metrics_collector.increment_dropped_tokens( - num_tokens=freed_device, - reason="host_pressure", - pool=PoolName.KV.value, - ) - return freed_device - - def evict_host(self, num_tokens: int): - leaves = list(self.evictable_host_leaves) - eviction_heap = [ - (self.eviction_strategy.get_priority(node), node) for node in leaves - ] - heapq.heapify(eviction_heap) - - num_evicted = 0 - while num_evicted < num_tokens and len(eviction_heap): - _, x = heapq.heappop(eviction_heap) - if x == self.root_node: - break - # only evict the host value of evicted nodes - if not x.evicted: - continue - - if x.host_ref_counter > 0: - continue - - # Block deleted entirely (GPU already evicted, now CPU freed) -- - # emit remove(CPU) so the router drops the host-tier entry. - self.kv_events.record_remove(x, medium=StorageMedium.CPU) - num_evicted += self.cache_controller.evict_host(x.host_value) - - key = x.key.child_key(self.page_size) - v = x.parent.children.pop(key, None) - assert v == x, f"parent does not have child key, {key}" - if x in self.evictable_host_leaves: - self.evictable_host_leaves.remove(x) - self._update_host_leaf_status(x.parent) - - if len(x.parent.children) == 0 and x.parent.evicted: - new_priority = self.eviction_strategy.get_priority(x.parent) - heapq.heappush(eviction_heap, (new_priority, x.parent)) - - def load_back( - self, node: TreeNode, mem_quota: Optional[int] = None - ) -> Optional[torch.Tensor]: - - last_hit_node = node - nodes_to_load = [] - while node.evicted: - assert node.backuped, ( - "No backup available on evicted nodes, should not happen" - ) - nodes_to_load.insert(0, node) - node = node.parent - else: - ancester_node = node - - # protect the ancestor nodes from eviction - result = self.inc_lock_ref(ancester_node) - delta = result.delta - - # load it all or not at all - host_indices = torch.cat([n.host_value for n in nodes_to_load]) - if len(host_indices) < self.load_back_threshold or ( - len(host_indices) > mem_quota + delta if mem_quota is not None else False - ): - # skip loading back if the total size is too small or exceeding the memory quota - self.dec_lock_ref(ancester_node) - return None - - # Protect the nodes being loaded from host eviction. - for n in nodes_to_load: - n.protect_host() - - device_indices = self.cache_controller.load( - host_indices=host_indices, - node_id=last_hit_node.id, - **self._get_extra_pools(), - ) - if device_indices is None: - self.evict(EvictParams(num_tokens=len(host_indices))) - device_indices = self.cache_controller.load( - host_indices=host_indices, - node_id=last_hit_node.id, - **self._get_extra_pools(), - ) - self.dec_lock_ref(ancester_node) - if device_indices is None: - # no sufficient GPU memory to load back KV caches - for n in nodes_to_load: - n.release_host() - logger.warning( - "load_back: FAILED to load %d tokens for node %d " - "even after eviction (evictable_size=%d)", - len(host_indices), - last_hit_node.id, - self.evictable_size_, - ) - return None - - for n in nodes_to_load: - n.release_host() - self.ongoing_load_back[last_hit_node.id] = last_hit_node - offset = 0 - for node in nodes_to_load: - node.value = device_indices[offset : offset + len(node.host_value)].clone() - offset += len(node.host_value) - # Block promoted from host to GPU -- emit store(GPU) so downstream - # indexers see it as device-local again. - self.kv_events.record_store(node, medium=StorageMedium.GPU) - self.evictable_size_ += len(device_indices) - self.inc_lock_ref(last_hit_node) - - return device_indices - - def init_load_back( - self, - params: InitLoadBackParams, - ): - last_node = params.best_match_node - mem_quota = params.mem_quota - if last_node.evicted: - loading_values = self.load_back(last_node, mem_quota) - if loading_values is not None: - logger.debug( - f"loading back {len(loading_values)} tokens for node {last_node.id}" - ) - return loading_values, last_node - - while last_node.evicted: - last_node = last_node.parent - - return ( - self._empty_match_result.device_indices, - last_node, - ) - - def query_storage_hit_length( - self, - last_host_node: TreeNode, - new_input_tokens: List[int], - last_hash: Optional[str] = None, - prefix_keys: Optional[List[str]] = None, - extra_key: Optional[str] = None, - cache_salt: Optional[str] = None, - ) -> int: - if not self.enable_storage or self.cache_controller.prefetch_rate_limited(): - return 0 - - prefetch_key = RadixKey( - new_input_tokens, - extra_key=extra_key, - is_bigram=self.is_eagle, - cache_salt=cache_salt, - ).page_aligned(self.page_size) - if len(prefetch_key) < self.prefetch_threshold: - return 0 - - prefetch_op_cls = ( - HybridPrefetchOperation - if isinstance(self.cache_controller, HybridCacheController) - else PrefetchOperation - ) - extra_kwargs = {} - if prefetch_op_cls is HybridPrefetchOperation: - extra_kwargs["pool_transfers"] = self._get_extra_pools().get("extra_pools") - request = ( - CacheRequestHandle("__storage_hit_query__", 0) - if prefetch_op_cls is HybridPrefetchOperation - else "__storage_hit_query__" - ) - operation = prefetch_op_cls( - request, - prefetch_key, - last_hash, - prefix_keys, - **extra_kwargs, - ) - hash_values, storage_hit_count = self.cache_controller._storage_hit_query( - operation - ) - storage_hit_count_tensor = torch.tensor(storage_hit_count, dtype=torch.int) - self._all_reduce_attn_groups( - storage_hit_count_tensor, torch.distributed.ReduceOp.MIN - ) - storage_hit_count = storage_hit_count_tensor.item() - storage_hit_count = storage_hit_count - (storage_hit_count % self.page_size) - return storage_hit_count - - def ready_to_load_host_cache(self) -> int: - """ - Notify the cache controller to start the KV cache loading. - Return the consumer index for the schedule batch manager to track. - """ - return self.cache_controller.start_loading() - - def check_hicache_events(self): - # Reap the previous round's PP-sync sends before issuing new ones. - self._drain_async_work() - - if self.pp_size != 1: - self.writing_check() - self.loading_check() - if self.enable_storage: - self.drain_storage_control_queues() - else: - ( - write_finish_count, - load_finish_count, - storage_queue_sizes, - ) = self._sync_hicache_ready_counts() - self.writing_check(finish_count=write_finish_count) - self.loading_check(finish_count=load_finish_count) - - if self.enable_storage and storage_queue_sizes: - n_storage_hit, n_ack_prefetch, n_backup, n_release = ( - storage_queue_sizes[:4] - ) - self._drain_storage_control_queues_impl( - n_storage_hit=n_storage_hit, - n_ack_prefetch=n_ack_prefetch, - n_backup=n_backup, - n_release=n_release, - log_metrics=True, - ) - if self.enable_storage_metrics: - self.storage_metrics_collector.log_storage_metrics( - self.cache_controller.storage_backend.get_stats() - ) - - def drain_storage_control_queues(self): - """ - Combine prefetch revoke, backup ack, and host mem release checks - to minimize TP synchronization and Python overhead. - """ - cc = self.cache_controller - - qsizes = torch.tensor( - [ - cc.prefetch_hit_queue.qsize(), - cc.ack_prefetch_queue.qsize(), - cc.ack_backup_queue.qsize(), - cc.host_mem_release_queue.qsize(), - ], - dtype=torch.int, - ) - - self._all_reduce(qsizes, torch.distributed.ReduceOp.MIN) - n_storage_hit, n_ack_prefetch, n_backup, n_release = map(int, qsizes.tolist()) - self._drain_storage_control_queues_impl( - n_storage_hit=n_storage_hit, - n_ack_prefetch=n_ack_prefetch, - n_backup=n_backup, - n_release=n_release, - log_metrics=True, - ) - - # Timeout is linearly increasing with the number of pages - def _prefetch_timeout_check_linear_func(self, operation: PrefetchOperation): - cfg = self.prefetch_timeout_config - num_tokens = len(operation.hash_value) * self.page_size - timeout = min(cfg.max, cfg.base + cfg.per_ki_token * num_tokens / 1024) - return time.monotonic() - operation.start_time > timeout - - def can_terminate_prefetch(self, operation: PrefetchOperation) -> bool: - if self.prefetch_stop_policy == "best_effort": - return True - if self.prefetch_stop_policy == "wait_complete": - return False - elif self.prefetch_stop_policy == "timeout": - return self.is_prefetch_timeout(operation) - else: - # unknown prefetch stop policy, just return True - return True - - def _revoke_pending_prefetch(self, req_id: str): - info = self.ongoing_prefetch.pop(req_id, None) - if info is None: - return - last_host_node, prefetch_key, _ = info - last_host_node.release_host() - cc = self.cache_controller - cc.prefetch_tokens_occupied = max( - 0, cc.prefetch_tokens_occupied - len(prefetch_key) - ) - - def has_ongoing_prefetch(self, handle: CacheRequestHandle) -> bool: - return handle.rid in self.ongoing_prefetch - - def check_prefetch_progress(self, handle: CacheRequestHandle) -> bool: - req_id = handle.rid - if req_id not in self.ongoing_prefetch: - # there is no ongoing prefetch for this request or it has been revoked - return True - - _, _, operation = self.ongoing_prefetch[req_id] - - # Determine whether or not we should terminate this prefetch request. Make all - # ranks agree on the decision. When running with PP, PPn will follow PP0's decision. - should_terminate = False - if self.pp_rank == 0: - should_terminate = operation.is_terminated() or self.can_terminate_prefetch( - operation - ) - should_terminate_tensor = torch.tensor( - int(should_terminate), dtype=torch.int, device="cpu" - ) - self._all_reduce(should_terminate_tensor, torch.distributed.ReduceOp.MAX) - should_terminate = should_terminate_tensor.item() == 1 - - if not should_terminate: - return False - - # Terminate in-flight prefetch. - self.cache_controller.terminate_prefetch(operation) - if operation.host_indices is None: - # Stopping before host memory was committed (best_effort, timeout, or - # still mid-query): signal the worker to stop, then release the request. - self._revoke_pending_prefetch(req_id) - else: - # Handle partial or full completion. - self._handle_prefetch_result(operation) - return True - - def _handle_prefetch_result(self, operation: PrefetchOperation) -> None: - req_id = operation.request_id - # All PP/TP ranks will get the same `min_completed_tokens`, because `completed_tokens` - # and `pool_hits` in their operations are same. No need to sync cross-rank here. - min_completed_tokens = self._clamp_prefetch_result(operation) - logger.debug( - f"Prefetch {req_id} completed with {operation.completed_tokens} tokens" - ) - last_host_node, prefetch_key, operation = self.ongoing_prefetch.pop(req_id) - host_indices = operation.host_indices - fetched_key = prefetch_key[:min_completed_tokens] - written_indices = host_indices[:min_completed_tokens] - matched_length = self._insert_helper_host( - last_host_node, - fetched_key, - written_indices, - operation.hash_value[: min_completed_tokens // self.page_size], - ) - - self.cache_controller.mem_pool_host.free(host_indices[:matched_length]) - last_host_node.release_host() - self.cache_controller.prefetch_tokens_occupied -= len(prefetch_key) - - # Track tokens actually loaded from storage for this request (L3 hits) - loaded_from_storage = min_completed_tokens - matched_length - self.prefetch_loaded_tokens_by_reqid[req_id] = loaded_from_storage - - if self.enable_storage_metrics: - self.storage_metrics_collector.log_prefetched_tokens(loaded_from_storage) - return - - def _clamp_prefetch_result( - self, - operation: PrefetchOperation, - ) -> int: - """Determine the minimal number of tokens from full KV hits and sidecar hits. - - HiRadixCache only wires DSA-style stacks (Full attention + a KV-derived - ALL_PAGES sidecar such as the DSA / MiniMax indexer); For the DSA case we *clamp* - to the minimum fetched prefix shared by the Full KV pool and every - sidecar rather than discarding everything. With no sidecar (FULL-only) - this is just Full KV completion. - """ - # Sync completed tokens and per-pool hit pages across ATTN groups, taking - # the minimum so every rank agrees on the same usable prefix length. - pool_transfers = getattr(operation, "pool_transfers", None) or [] - hit_pages = ( - operation.pool_storage_result.extra_pool_hit_pages if pool_transfers else {} - ) - completed_tokens = operation.completed_tokens - pool_hit_pages = [hit_pages.get(t.name, 0) for t in pool_transfers] - - # Clamp to the shared minimum prefix of the Full KV completion and each - # KV-derived ALL_PAGES sidecar (e.g. the DSA indexer). FULL-only has no - # sidecar, so the usable prefix is just the Full KV completion. - usable_pages = completed_tokens // self.page_size - if pool_transfers: - usable_pages = min(usable_pages, *pool_hit_pages) - return usable_pages * self.page_size - - def pop_prefetch_loaded_tokens(self, handle: CacheRequestHandle) -> int: - """ - Pop and return the number of tokens loaded from storage for a request. - Returns 0 if no prefetch was done or was revoked. - This should be called after check_prefetch_progress() returns True. - """ - return self.prefetch_loaded_tokens_by_reqid.pop(handle.rid, 0) - - def match_prefix(self, params: MatchPrefixParams): - if self.disable: - return self._empty_match_result - - key = params.key - key, _ = key.maybe_to_bigram_view(self.is_eagle) - key = key.page_aligned(self.page_size) - if len(key) == 0: - return self._empty_match_result - - value, last_node = self._match_prefix_helper(self.root_node, key) - if value: - value = torch.cat(value) - else: - value = self._empty_match_result.device_indices - - host_hit_length = 0 - last_host_node = last_node - while last_node.evicted: - host_hit_length += len(last_node.host_value) - last_node = last_node.parent - while not last_host_node.backuped: - last_host_node = last_host_node.parent - - return MatchResult( - device_indices=value, - last_device_node=last_node, - last_host_node=last_host_node, - # TODO(ispobock): use best_match_node as start node for load_back - best_match_node=last_host_node, - host_hit_length=host_hit_length, - ) - - def prefetch_from_storage( - self, - handle: CacheRequestHandle, - last_host_node: TreeNode, - new_input_tokens: List[int], - last_hash: Optional[str] = None, - prefix_keys: Optional[List[str]] = None, - # Scheduler-call parity with UnifiedRadixCache; unused in cache mode. - matched_prefix_tokens: Optional[List[int]] = None, - extra_key: Optional[str] = None, - cache_salt: Optional[str] = None, - storage_hit_end: Optional[int] = None, - ): - req_id = handle.rid - prefetch_key = RadixKey( - new_input_tokens, - extra_key=extra_key, - is_bigram=self.is_eagle, - cache_salt=cache_salt, - ) - # align the number of fetching tokens to the page size - prefetch_key = prefetch_key.page_aligned(self.page_size) - prefetch_length = len(prefetch_key) - if ( - not self.enable_storage - or prefetch_length < self.prefetch_threshold - or self.cache_controller.prefetch_rate_limited() - ): - return - - last_host_node.protect_host() - # NOTE: host_indices is no longer pre-allocated here. It is allocated - # lazily in _drain_and_alloc_storage_hit() once the L3 storage hit count is known, - # so we only reserve host memory for pages that actually hit. - request = ( - handle - if isinstance(self.cache_controller, HybridCacheController) - else req_id - ) - operation = self.cache_controller.prefetch( - request, - prefetch_key, - last_hash, - prefix_keys, - **self._get_extra_pools(), - ) - self.ongoing_prefetch[req_id] = ( - last_host_node, - prefetch_key, - operation, - ) - self.cache_controller.prefetch_tokens_occupied += len(prefetch_key) - - def _insert_helper_host( - self, node: TreeNode, key: RadixKey, host_value, hash_value - ): - node.last_access_time = time.monotonic() - if len(key) == 0: - return 0 - - child_key = key.child_key(self.page_size) - - matched_length = 0 - while len(key) > 0 and child_key in node.children.keys(): - node = node.children[child_key] - node.last_access_time = time.monotonic() - prefix_len = node.key.match(key, page_size=self.page_size) - key = key[prefix_len:] - host_value = host_value[prefix_len:] - hash_value = hash_value[prefix_len // self.page_size :] - matched_length += prefix_len - - if prefix_len < len(node.key): - new_node = self._split_node(node.key, node, prefix_len) - node = new_node - - if len(key): - child_key = key.child_key(self.page_size) - - if len(key): - new_node = TreeNode(priority=node.priority) - new_node.parent = node - new_node.key = key - new_node.value = None - new_node.host_value = host_value.clone() - new_node.hash_value = hash_value - node.children[child_key] = new_node - self._update_host_leaf_status(new_node) - self._update_leaf_status(node) - self._update_host_leaf_status(node) - # Publish the newly materialized host suffix immediately so downstream - # cache indexers can resolve descendants that extend this L2-only prefix. - self.kv_events.record_store(new_node, medium=StorageMedium.CPU) - - return matched_length - - def _match_prefix_helper(self, node: TreeNode, key: RadixKey): - node.last_access_time = time.monotonic() - child_key = key.child_key(self.page_size) - value = [] - - while len(key) > 0 and child_key in node.children.keys(): - child = node.children[child_key] - child.last_access_time = time.monotonic() - prefix_len = child.key.match(key, page_size=self.page_size) - if prefix_len < len(child.key): - new_node = self._split_node(child.key, child, prefix_len) - if not new_node.evicted: - value.append(new_node.value) - node = new_node - break - else: - if not child.evicted: - value.append(child.value) - node = child - key = key[prefix_len:] - - if len(key): - child_key = key.child_key(self.page_size) - - return value, node - - def _split_node(self, key: RadixKey, child: TreeNode, split_len: int): - # child node split into new_node -> child - new_node = TreeNode(priority=child.priority) - new_node.children = {key[split_len:].child_key(self.page_size): child} - new_node.parent = child.parent - new_node.lock_ref = child.lock_ref - new_node.key = child.key[:split_len] - new_node.hit_count = child.hit_count - - # split value and host value if exists - if child.evicted: - new_node.value = None - else: - new_node.value = child.value[:split_len].clone() - child.value = child.value[split_len:].clone() - if child.backuped: - new_node.host_value = child.host_value[:split_len].clone() - child.host_value = child.host_value[split_len:].clone() - - new_node.hash_value, child.hash_value = split_node_hash_value( - child.hash_value, split_len, self.page_size - ) - new_node.event_hash_value, child.event_hash_value = split_node_hash_value( - child.event_hash_value, split_len, self.page_size - ) - child.parent = new_node - child.key = child.key[split_len:] - new_node.parent.children[key.child_key(self.page_size)] = new_node - - if child.backuped: - self._replace_pending_write_through_node(child, [new_node, child]) - - return new_node - - def insert(self, params: InsertParams) -> InsertResult: - key = params.key - value = params.value - chunked = params.chunked - priority = params.priority - - if priority is None: - priority = 0 - - key, value = key.maybe_to_bigram_view(self.is_eagle, value) - key = key.page_aligned(self.page_size) - if value is not None: - value = value[: len(key)] - - if len(key) == 0: - return InsertResult(prefix_len=0) - - node = self.root_node - child_key = key.child_key(self.page_size) - total_prefix_length = 0 - - while len(key) > 0 and child_key in node.children.keys(): - node = node.children[child_key] - node.last_access_time = time.monotonic() - node.priority = max(node.priority, priority) - prefix_len = node.key.match(key, page_size=self.page_size) - - if prefix_len == len(node.key): - if node.evicted: - # change the reference if the node is evicted - # this often happens in the case of KV cache recomputation - node.value = value[:prefix_len].clone() - self.evictable_size_ += len(node.value) - self._update_leaf_status(node) - self._update_host_leaf_status(node) - # update parent status as a new leaf is added into device - self._update_leaf_status(node.parent) - else: - self._inc_hit_count(node, chunked) - total_prefix_length += prefix_len - else: - # partial match, split the node - new_node = self._split_node(node.key, node, prefix_len) - # shared-prefix node should also reflect max priority - new_node.priority = max(new_node.priority, priority) - if new_node.evicted: - new_node.value = value[:prefix_len].clone() - self.evictable_size_ += len(new_node.value) - self._update_leaf_status(new_node) - self._update_host_leaf_status(new_node) - # update parent status as a new leaf is added into device - self._update_leaf_status(new_node.parent) - else: - self._inc_hit_count(new_node, chunked) - total_prefix_length += prefix_len - node = new_node - - key = key[prefix_len:] - value = value[prefix_len:] - - if len(key): - child_key = key.child_key(self.page_size) - - if len(key): - new_node = TreeNode(priority=priority) - new_node.parent = node - new_node.key = key - new_node.value = value.clone() - node.children[child_key] = new_node - self.evictable_size_ += len(value) - self._update_leaf_status(node) - self._update_leaf_status(new_node) - - # Compute hash_value if storage or kv events are enabled - if self.enable_storage or self.kv_events.enabled: - new_node.hash_value = compute_node_hash_values(new_node, self.page_size) - - # Emit BlockStored so the router indexes this block. - self.kv_events.record_store(new_node) - - if self.cache_controller.write_policy != "write_back": - self._inc_hit_count(new_node, chunked) - return InsertResult(prefix_len=total_prefix_length) - - def release_aborted_request(self, handle: CacheRequestHandle): - rid = handle.rid - # Clean up storage hit tracking for aborted request - self.prefetch_loaded_tokens_by_reqid.pop(rid, None) - - if rid not in self.ongoing_prefetch: - return - - last_host_node, prefetch_key, operation = self.ongoing_prefetch[rid] - if operation.host_indices is None: - self.cache_controller.terminate_prefetch(operation) - self._revoke_pending_prefetch(rid) - return - - completed_tokens, _ = self.cache_controller.terminate_prefetch(operation) - last_host_node.release_host() - del self.ongoing_prefetch[rid] - self.cache_controller.append_host_mem_release( - operation.host_indices[:completed_tokens] - ) - self.cache_controller.prefetch_tokens_occupied -= len(prefetch_key) diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index 60f281ba34..9cee653925 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -33,7 +33,6 @@ if TYPE_CHECKING: import torch from sglang.srt.mem_cache.cache_init_params import CacheInitParams - from sglang.srt.mem_cache.hiradix_cache import HiRadixCache from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.srt.server_args import ServerArgs @@ -2229,7 +2228,7 @@ def build_minimax_sparse_hicache_stack( "MiniMax-M3 sparse HiCache does not support pipeline parallelism " "(pp_size>1) yet." ) - # mirror HiRadix's guard, which the Unified-tree strategy path otherwise skips. + # The Unified-tree strategy path does not reject value-bearing index layers. if sparse_pool.index_kv_pool is not None: raise ValueError( "MiniMax sparse HiCache currently supports index-k-only sparse layers; " @@ -2301,124 +2300,3 @@ def build_minimax_sparse_hicache_stack( enable_storage_metrics=enable_storage_metrics, ) return host_pool_group, cache_controller - - -def attach_hybrid_minimax_sparse_pool_to_hiradix_cache( - radix_cache: HiRadixCache, - params: CacheInitParams, - *, - extra_config: dict, - prefetch_threshold: int, - enable_storage_metrics: bool, - load_cache_event, -) -> None: - """Attach HostPoolGroup (KV + index K) + HybridCacheController for HiRadixCache.""" - from sglang.srt.mem_cache.memory_pool import MiniMaxSparseKVPool - - try: - sparse_pool = radix_cache.kv_cache - if not isinstance(sparse_pool, MiniMaxSparseKVPool): - raise TypeError( - f"Expected MiniMaxSparseKVPool, got {type(sparse_pool).__name__}" - ) - if sparse_pool.index_kv_pool is not None: - raise ValueError( - "MiniMax M3 HiCache L2 currently supports index-k-only sparse layers " - "(sparse_disable_index_value=1 for all sparse layers). " - "This model has index_kv_pool layers; INDEXER_KV sidecar is not " - "implemented yet." - ) - - main_pool = sparse_pool.main_pool - if sparse_pool.index_k_pool is None: - host_pool_group, cache_controller = build_kv_only_stack( - params=params, - kv_pool=main_pool, - full_layer_mapping={ - layer_id: layer_id for layer_id in range(main_pool.layer_num) - }, - load_cache_event=load_cache_event, - storage_backend=get_memory().hicache_storage_backend, - use_mla=False, - prefetch_threshold=prefetch_threshold, - model_name=get_serving().served_model_name, - storage_backend_extra_config=extra_config, - enable_storage_metrics=enable_storage_metrics, - ) - pools_desc = "KV" - else: - host_pool_group, cache_controller = build_minimax_sparse_hicache_stack( - params=params, - sparse_pool=sparse_pool, - load_cache_event=load_cache_event, - storage_backend=get_memory().hicache_storage_backend, - prefetch_threshold=prefetch_threshold, - model_name=get_serving().served_model_name, - storage_backend_extra_config=extra_config, - enable_storage_metrics=enable_storage_metrics, - ) - pools_desc = "KV + INDEXER(k-only)" - - sparse_pool.register_layer_transfer_counter(cache_controller.layer_done_counter) - radix_cache.full_kv_pool_host = host_pool_group.get_pool(PoolName.KV) - radix_cache.token_to_kv_pool_host = host_pool_group - radix_cache.cache_controller = cache_controller - logger.info( - "Attached hybrid MiniMax sparse pool stack to HiRadixCache: pools=%s, " - "transfer_layer_id_max=%s, sparse_index_k_layers=%s", - pools_desc, - main_pool.layer_num, - len(sparse_pool.index_k_layer_id_mapping), - ) - except Exception: - logger.exception("attach_hybrid_minimax_sparse_pool_to_hiradix_cache failed") - raise - - -def attach_hybrid_dsa_pool_to_hiradix_cache( - radix_cache: HiRadixCache, - params: CacheInitParams, - *, - extra_config: dict, - prefetch_threshold: int, - enable_storage_metrics: bool, - load_cache_event, -) -> None: - """Attach HostPoolGroup (KV + indexer) + HybridCacheController for HiRadixCache. - - This entrypoint is currently intended only for HiRadixCache's DSA path. - """ - try: - kv = radix_cache.kv_cache - layer_mapping = {layer_id: layer_id for layer_id in range(kv.layer_num)} - host_pool_group, cache_controller = build_anchor_sidecar_stack( - params=params, - kv_pool=kv, - sidecar_pool_name=PoolName.INDEXER, - full_layer_mapping=layer_mapping, - load_cache_event=load_cache_event, - storage_backend=get_memory().hicache_storage_backend, - use_mla=True, - override_kv_cache_dim=kv.kv_cache_dim, - prefetch_threshold=prefetch_threshold, - sidecar_host_pool_factory=lambda kv_host_pool: DSAIndexerPoolHost( - kv, - kv_host_pool, - get_memory().hicache_mem_layout, - allocator_type=_get_allocator_type(), - ), - model_name=get_serving().served_model_name, - storage_backend_extra_config=extra_config, - enable_storage_metrics=enable_storage_metrics, - ) - radix_cache.full_kv_pool_host = host_pool_group.get_pool(PoolName.KV) - radix_cache.token_to_kv_pool_host = host_pool_group - radix_cache.cache_controller = cache_controller - logger.info( - "Attached hybrid DSA pool stack to HiRadixCache: pools=KV + INDEXER, " - "transfer_layer_id_max=%s", - len(layer_mapping), - ) - except Exception: - logger.exception("attach_hybrid_dsa_pool_to_hiradix_cache failed") - raise diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index 3eaebc5e96..5719c61498 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -266,12 +266,6 @@ class TreeNode: self.creation_time = time.monotonic() self.hit_count = 0 - # indicating the node is locked to protect from eviction - # incremented when the node is referenced by a storage operation - self.host_ref_counter = 0 - # store the host indices of KV cache - self.host_value: Optional[torch.Tensor] = None - self.write_through_pending_id: Optional[int] = None # store hash values of each pages self.hash_value: Optional[List[str]] = None # Namespace-aware hashes used only for external KV events. @@ -286,21 +280,6 @@ class TreeNode: def evicted(self): return self.value is None - @property - def backuped(self): - return self.host_value is not None - - def protect_host(self): - """Protect the host value from eviction.""" - self.host_ref_counter += 1 - - def release_host(self): - """Release the host value, allowing it to be evicted.""" - if self.host_ref_counter > 0: - self.host_ref_counter -= 1 - else: - raise RuntimeError("Host reference counter is already zero.") - def get_last_hash_value(self) -> Optional[str]: """Returns the hash value of the last page in this node.""" if self.hash_value is None or len(self.hash_value) == 0: @@ -375,7 +354,6 @@ class RadixCache(BasePrefixCache): self.root_node = TreeNode(priority=-sys.maxsize) self.root_node.key = RadixKey(token_ids=array("q"), extra_key=None) self.root_node.value = [] - self.root_node.host_value = [] self.root_node.lock_ref = 1 self.root_node.hash_value = [] self.evictable_size_ = 0 diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 1365e97f50..7803025014 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -355,7 +355,7 @@ class UnifiedRadixCache(BasePrefixCache): data, group_src=self.pp_rank - 1, group=self.pp_group, - tag=P2PTag.HIRADIX_PP_SYNC, + tag=P2PTag.HICACHE_PP_SYNC, ) if self.pp_rank + 1 < self.pp_size: copy_of_data = data.clone() @@ -363,7 +363,7 @@ class UnifiedRadixCache(BasePrefixCache): copy_of_data, group_dst=self.pp_rank + 1, group=self.pp_group, - tag=P2PTag.HIRADIX_PP_SYNC, + tag=P2PTag.HICACHE_PP_SYNC, ) self.work_list.append(send_work) @@ -3477,8 +3477,8 @@ class UnifiedRadixCache(BasePrefixCache): def is_load_back_event_done(self, consumer_index: int) -> bool: """Return True after the local load-back event is complete. - Mirrors ``HiRadixCache`` so the disagg decode restore state machine - (``DecodeHiCacheTransferMixin``) can gate on load-back completion; the + Lets the disagg decode restore state machine + (``DecodeHiCacheTransferMixin``) gate on load-back completion; the controller-level ``layer_done_counter`` event is shared across cache implementations, while the tree-side bookkeeping runs in ``loading_check``. diff --git a/test/registered/unit/mem_cache/test_hicache_load_back_timing.py b/test/registered/unit/mem_cache/test_hicache_load_back_timing.py index dbcbadf706..1b2ad0b8fd 100644 --- a/test/registered/unit/mem_cache/test_hicache_load_back_timing.py +++ b/test/registered/unit/mem_cache/test_hicache_load_back_timing.py @@ -59,9 +59,26 @@ class TestLoadBackDurationMetric(CustomTestCase): self.assertIs(finish, events[1]) self.assertIsNot(start, finish) - def test_loading_check_observes_duration_and_tokens(self): - from sglang.srt.mem_cache.hiradix_cache import HiRadixCache + def _make_unified_stub(self, ack): + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache + stub = object.__new__(UnifiedRadixCache) + stub.cache_controller = SimpleNamespace(ack_load_queue=[ack]) + stub.ongoing_load_back = { + node_id: (object(), None, None) for node_id in ack.node_ids + } + stub.buffer_pipeline = None + stub.tree_core = SimpleNamespace( + write_back_duplicate_reclaim_digest=0, finish_load_back=MagicMock() + ) + stub.dec_lock_ref = MagicMock() + stub.dec_host_lock_ref = MagicMock() + stub.metrics_collector = MagicMock() + stub.pp_rank = 0 + stub._all_reduce = MagicMock() + return stub + + def test_loading_check_observes_duration_and_tokens(self): start, finish = self._completed_pair() ack = self.cc.HiCacheAck( start, @@ -71,13 +88,7 @@ class TestLoadBackDurationMetric(CustomTestCase): timing_enabled=True, num_tokens_by_pool={"kv": 1024}, ) - stub = object.__new__(HiRadixCache) - stub.cache_controller = SimpleNamespace(ack_load_queue=[ack]) - stub.ongoing_load_back = {1: object(), 2: object()} - stub.dec_lock_ref = MagicMock() - stub.metrics_collector = MagicMock() - stub.pp_rank = 0 - stub._all_reduce = MagicMock() + stub = self._make_unified_stub(ack) stub.loading_check() @@ -88,11 +99,10 @@ class TestLoadBackDurationMetric(CustomTestCase): (observed,), _ = stub.metrics_collector.observe_load_back_duration.call_args self.assertGreater(observed, 0.0) self.assertEqual(stub.cache_controller.ack_load_queue, []) + self.assertEqual(stub.ongoing_load_back, {}) def test_loading_check_fallback_when_timing_unsupported(self): """On backends without enable_timing, count tokens but skip duration.""" - from sglang.srt.mem_cache.hiradix_cache import HiRadixCache - start = torch.cuda.Event() finish = torch.cuda.Event() start.record() @@ -107,13 +117,7 @@ class TestLoadBackDurationMetric(CustomTestCase): timing_enabled=False, num_tokens_by_pool={"kv": 512}, ) - stub = object.__new__(HiRadixCache) - stub.cache_controller = SimpleNamespace(ack_load_queue=[ack]) - stub.ongoing_load_back = {7: object()} - stub.dec_lock_ref = MagicMock() - stub.metrics_collector = MagicMock() - stub.pp_rank = 0 - stub._all_reduce = MagicMock() + stub = self._make_unified_stub(ack) stub.loading_check() diff --git a/test/registered/unit/mem_cache/test_hiradix_pp_sync_drain.py b/test/registered/unit/mem_cache/test_hicache_pp_sync_drain.py similarity index 87% rename from test/registered/unit/mem_cache/test_hiradix_pp_sync_drain.py rename to test/registered/unit/mem_cache/test_hicache_pp_sync_drain.py index 05bf95168f..deff151052 100644 --- a/test/registered/unit/mem_cache/test_hiradix_pp_sync_drain.py +++ b/test/registered/unit/mem_cache/test_hicache_pp_sync_drain.py @@ -7,7 +7,6 @@ from unittest.mock import MagicMock import torch -from sglang.srt.mem_cache.hiradix_cache import HiRadixCache from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.test.ci.ci_register import register_cpu_ci @@ -27,28 +26,23 @@ class _Holder: class TestPPSyncDrain(unittest.TestCase): - def _drain_fns(self): - return (HiRadixCache._drain_async_work, UnifiedRadixCache._drain_async_work) - def test_drain_waits_all_and_clears(self): - for drain in self._drain_fns(): - holder = _Holder() - works = [_FakeWork(), _FakeWork(), _FakeWork()] - holder.work_list = list(works) + holder = _Holder() + works = [_FakeWork(), _FakeWork(), _FakeWork()] + holder.work_list = list(works) - drain(holder) + UnifiedRadixCache._drain_async_work(holder) - self.assertTrue(all(w.waited for w in works)) - self.assertEqual(holder.work_list, []) + self.assertTrue(all(w.waited for w in works)) + self.assertEqual(holder.work_list, []) def test_drain_empty_is_noop(self): - for drain in self._drain_fns(): - holder = _Holder() - holder.work_list = [] + holder = _Holder() + holder.work_list = [] - drain(holder) + UnifiedRadixCache._drain_async_work(holder) - self.assertEqual(holder.work_list, []) + self.assertEqual(holder.work_list, []) class TestUnifiedPPSyncBatching(unittest.TestCase): diff --git a/test/registered/unit/mem_cache/test_hiradix_cache_unit.py b/test/registered/unit/mem_cache/test_hiradix_cache_unit.py deleted file mode 100644 index b2ebea42ab..0000000000 --- a/test/registered/unit/mem_cache/test_hiradix_cache_unit.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Unit tests for srt/mem_cache/hiradix_cache.py KV cache events.""" - -import os -import unittest -from array import array - -import torch - -from sglang.srt.disaggregation.kv_events import BlockStored, StorageMedium -from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator -from sglang.srt.mem_cache.base_prefix_cache import InsertParams, MatchPrefixParams -from sglang.srt.mem_cache.cache_init_params import CacheInitParams -from sglang.srt.mem_cache.hiradix_cache import HiRadixCache -from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, ReqToTokenPool -from sglang.srt.mem_cache.radix_cache import RadixKey -from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler -from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci -from sglang.test.test_utils import CustomTestCase - -register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small") -register_amd_ci(est_time=15, stage="stage-b", runner_config="1-gpu-small-amd") - -PAGE_SIZE = 2 - - -class TestHiRadixCacheKVEvents(CustomTestCase): - @classmethod - def setUpClass(cls): - if not torch.cuda.is_available(): - raise unittest.SkipTest("CUDA is required for HiRadixCache tests.") - os.environ.setdefault("MASTER_ADDR", "127.0.0.1") - os.environ.setdefault("MASTER_PORT", "29601") - if not torch.distributed.is_initialized(): - torch.distributed.init_process_group(backend="gloo", rank=0, world_size=1) - - def _build_cache(self): - server_args = ServerArgs( - model_path="dummy", - page_size=PAGE_SIZE, - hicache_io_backend="direct", - hicache_mem_layout="layer_first", - hicache_write_policy="write_through", - ) - set_global_server_args_for_scheduler(server_args) - req_to_token_pool = ReqToTokenPool( - size=10, - max_context_len=512, - device="cuda", - enable_memory_saver=False, - ) - kv_pool = MHATokenToKVPool( - size=256, - page_size=PAGE_SIZE, - dtype=torch.bfloat16, - head_num=2, - head_dim=64, - layer_num=4, - device="cuda", - enable_memory_saver=False, - ) - allocator = TokenToKVPoolAllocator( - size=256, - dtype=torch.bfloat16, - device="cuda", - kvcache=kv_pool, - need_sort=False, - ) - params = CacheInitParams( - req_to_token_pool=req_to_token_pool, - token_to_kv_pool_allocator=allocator, - page_size=PAGE_SIZE, - disable=False, - enable_kv_cache_events=True, - tp_cache_group=torch.distributed.group.WORLD, - ) - cache = HiRadixCache(params, server_args) - # Disable hit-count-driven write-through; tests back up explicitly. - cache.write_through_threshold = 1 << 30 - return cache, allocator - - def _insert(self, cache, allocator, tokens): - key = RadixKey(array("q", tokens)) - value = allocator.alloc(len(tokens)) - self.assertIsNotNone(value) - return cache.insert(InsertParams(key=key, value=value[: len(tokens)])) - - def _leaf_for(self, cache, tokens): - match = cache.match_prefix(MatchPrefixParams(key=RadixKey(array("q", tokens)))) - self.assertIsNot(match.last_device_node, cache.root_node) - return match.last_device_node - - def _stored_cpu_events(self, cache): - return [ - e - for e in cache.take_events() - if isinstance(e, BlockStored) and e.medium == StorageMedium.CPU - ] - - def test_split_pending_write_through_publishes_fragments(self): - cache, allocator = self._build_cache() - cache.take_events() - - self._insert(cache, allocator, [1, 2, 3, 4]) - node = self._leaf_for(cache, [1, 2, 3, 4]) - backed_up = cache.write_backup(node, write_back=True) - self.assertGreater(backed_up, 0) - - # Split the node while its write-through DMA is still pending. - self._insert(cache, allocator, [1, 2, 5, 6]) - self.assertEqual(self._stored_cpu_events(cache), []) - - cache.writing_check(write_back=True) - - # Both split fragments must be published as one parent-linked batch. - stored_cpu = self._stored_cpu_events(cache) - self.assertEqual(len(stored_cpu), 1) - self.assertEqual(list(stored_cpu[0].token_ids), [1, 2, 3, 4]) - self.assertIsNone(stored_cpu[0].parent_block_hash) - self.assertEqual(len(stored_cpu[0].block_hashes), 2) - - -if __name__ == "__main__": - unittest.main(verbosity=2) diff --git a/test/registered/unit/mem_cache/test_minimax_sparse_pool_host_unit.py b/test/registered/unit/mem_cache/test_minimax_sparse_pool_host_unit.py index 94c98edc23..a6cee776b7 100644 --- a/test/registered/unit/mem_cache/test_minimax_sparse_pool_host_unit.py +++ b/test/registered/unit/mem_cache/test_minimax_sparse_pool_host_unit.py @@ -3,11 +3,6 @@ import unittest import psutil import torch -from sglang.srt.mem_cache.hicache_storage import PoolHitPolicy, PoolName -from sglang.srt.mem_cache.hiradix_cache import HiRadixCache -from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( - HybridCacheController, -) from sglang.srt.mem_cache.memory_pool import MiniMaxSparseKVPool from sglang.srt.mem_cache.pool_host.base import HICACHE_HOST_MEMORY_RESERVE_BYTES from sglang.srt.mem_cache.pool_host.common import ( @@ -65,20 +60,6 @@ def _make_cpu_minimax_sparse_pool(start_layer: int = 4) -> MiniMaxSparseKVPool: class TestMiniMaxSparseHiCacheIntegration(unittest.TestCase): - def test_hiradix_extra_pools_include_minimax_indexer(self): - pool = _make_cpu_minimax_sparse_pool() - cache = object.__new__(HiRadixCache) - cache.cache_controller = object.__new__(HybridCacheController) - cache.kv_cache = pool - - extra = HiRadixCache._get_extra_pools(cache) - - transfers = extra["extra_pools"] - self.assertEqual(len(transfers), 1) - self.assertEqual(transfers[0].name, PoolName.INDEXER) - self.assertEqual(transfers[0].indices_from_pool, PoolName.KV) - self.assertEqual(transfers[0].hit_policy, PoolHitPolicy.ALL_PAGES) - def test_index_k_waits_for_full_local_layer(self): pool = _make_cpu_minimax_sparse_pool() counter = _FakeLayerTransferCounter() diff --git a/test/registered/unit/mem_cache/test_radix_cache_unit.py b/test/registered/unit/mem_cache/test_radix_cache_unit.py index 31f165402a..349cfebe1d 100644 --- a/test/registered/unit/mem_cache/test_radix_cache_unit.py +++ b/test/registered/unit/mem_cache/test_radix_cache_unit.py @@ -275,8 +275,6 @@ class TestTreeNode(unittest.TestCase): self.assertIsNone(node.value) self.assertEqual(node.lock_ref, 0) self.assertEqual(node.hit_count, 0) - self.assertEqual(node.host_ref_counter, 0) - self.assertIsNone(node.host_value) self.assertIsNone(node.hash_value) def test_init_with_id(self): @@ -286,46 +284,12 @@ class TestTreeNode(unittest.TestCase): node2 = TreeNode() self.assertEqual(node2.id, 1) # Counter was incremented - def test_evicted_backuped_properties(self): - """Test evicted and backuped properties.""" - test_cases = [ - (False, False, True, False), - (True, False, False, False), - (True, True, False, True), - (False, True, True, True), - ] - - for ( - has_value, - has_host_value, - expected_evicted, - expected_backuped, - ) in test_cases: - with self.subTest(has_value=has_value, has_host_value=has_host_value): - node = TreeNode() - - if has_value: - node.value = torch.tensor([1, 2, 3]) - if has_host_value: - node.host_value = torch.tensor([4, 5, 6]) - - self.assertEqual(node.evicted, expected_evicted) - self.assertEqual(node.backuped, expected_backuped) - - def test_protect_release_host(self): - """Test protect_host and release_host methods.""" + def test_evicted_property(self): + """Test the evicted property.""" node = TreeNode() - self.assertEqual(node.host_ref_counter, 0) - - node.protect_host() - self.assertEqual(node.host_ref_counter, 1) - - node.release_host() - self.assertEqual(node.host_ref_counter, 0) - - # Test error case - with self.assertRaises(RuntimeError): - node.release_host() + self.assertTrue(node.evicted) + node.value = torch.tensor([1, 2, 3]) + self.assertFalse(node.evicted) def test_get_last_hash_value(self): """Test get_last_hash_value method.""" diff --git a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/cache_controller.py b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/cache_controller.py index 8421caf10e..b746744567 100644 --- a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/cache_controller.py +++ b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/cache_controller.py @@ -251,7 +251,7 @@ class C_HiCacheController(BaseHook): def wrapped_terminate_prefetch(self, operator): result = original_terminate_prefetch(self, operator) - # This value may be a float if prefetch progress is interrupted by HiRadixCache.check_prefetch_progress. + # This value may be a float if prefetch progress is interrupted mid-transfer. result = (int(result[0]), result[1]) # operation.completed_tokens, operation.hash_value = result req_stats = request_stats_manager.get_req_stats(operator.request_id) diff --git a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hiradix_cache.py b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hiradix_cache.py deleted file mode 100644 index 0536a12a3a..0000000000 --- a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hiradix_cache.py +++ /dev/null @@ -1,25 +0,0 @@ -from sglang_simulator.hook import BaseHook - - -class C_HiRadixCacheHook(BaseHook): - HOOK_CLASS_NAME = "HiRadixCache" - HOOK_MODULE_NAME = "sglang.srt.mem_cache.hiradix_cache" - REQUIRED = False - - @classmethod - def hook(cls, target): - original_check_hicache_events = target.check_hicache_events - - def wrapped_check_hicache_events(self, *args, **kwargs): - # The async thread for prefetching and backup in `HiCacheController` has been deprecated. - # So we have to handle the backup or prefetch operation manually. - self.cache_controller.handle_backup_operation() - self.cache_controller.handle_prefetch_operation() - result = original_check_hicache_events(self, *args, **kwargs) - # Host pages are allocated after the storage query. Run the - # simulated transfer only after that allocation step. - if hasattr(self.cache_controller, "prefetch_hit_queue"): - self.cache_controller.handle_prefetch_operation() - return result - - target.check_hicache_events = wrapped_check_hicache_events diff --git a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hook_bootstrap.py b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hook_bootstrap.py index dd7725a6e6..2ae6da5b49 100644 --- a/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hook_bootstrap.py +++ b/tools/sglang-simulator/src/sglang_simulator/simulation/sglang/hook_bootstrap.py @@ -9,7 +9,6 @@ import sglang_simulator.hook as sglang_simulator_hook from sglang_simulator.simulation.sglang import ( cache_controller, hicache_storage, - hiradix_cache, mem_cache_allocator, mem_pool_host, model_runner, @@ -46,7 +45,6 @@ def install_simulator_hooks() -> None: hicache_storage.C_StorageBackendFactory, cache_controller.C_HiCacheController, cache_controller.C_HybridCacheController, - hiradix_cache.C_HiRadixCacheHook, unified_radix_cache.C_UnifiedRadixCacheHook, mem_cache_allocator.C_PagedTokenToKVPoolAllocatorHook, mem_pool_host.C_MHATokenToKVPoolHostHook,