Files
sglang/python/sglang/srt/managers/cache_controller.py
T
minke.yu d6f1e1efbd rebase(hicache): per-op async backup on top of #46884 + #48782
Net async change (from exp-async-backup-perop-20261107) applied onto
b300-pick-46884-20261107. Runtime files applied cleanly; the new tests and
the timeline probe are added as-is.

Interaction to validate on hardware: #46884 publishes prefill read-done before
the replay, so the scheduler-stream fence that the index copy waits on is
shorter; the async worker's wait should shrink and the copy may overlap more of
the in-flight forward. Re-baseline the 4-arm A/B on this branch.
2026-11-07 22:04:33 +08:00

2119 lines
84 KiB
Python

from __future__ import annotations
"""
Copyright 2023-2025 SGLang Team
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import logging
import threading
import time
from dataclasses import dataclass
from queue import Empty, Full, Queue
from typing import TYPE_CHECKING, Any, Callable, List, NamedTuple, Optional
import torch
from sglang.srt.mem_cache.hicache_storage import (
STORAGE_BATCH_SIZE,
HiCacheStorageConfig,
HiCacheStorageExtraInfo,
PoolName,
PoolTransfer,
count_pool_hits,
)
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.memory_pool_host import LogicalHostPool
from sglang.srt.mem_cache.pool_host import HostKVCache
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
from sglang.srt.mem_cache.utils import get_storage_hash_str
from sglang.srt.observability import batch_telemetry
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import get_device_module
logger = logging.getLogger(__name__)
device_module = get_device_module()
def storage_model_name(
model_name: Optional[str], host_pool: HostKVCache | LogicalHostPool
) -> Optional[str]:
"""The model name storage backends key pages on, tagged with the host
pool's page format when it has one of its own."""
tag = host_pool.storage_format_tag
if tag is None:
return model_name
return f"{model_name}-{tag}" if model_name else tag
# [HC_PARALLEL_PATCH] knobs for the pipelined HiCache prefetch/backup lanes
import os as _hc_os
_HC_PREFETCH_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_PREFETCH_IO_THREADS", "1") or 1)
_HC_BACKUP_WORKERS = int(_hc_os.getenv("SGLANG_HICACHE_BACKUP_IO_THREADS", "1") or 1)
_HC_TIMING = _hc_os.getenv("SGLANG_HICACHE_TIMING", "0") not in ("", "0", "false", "False")
# [ASYNC_BACKUP] Per-op pinned index copy + worker-side merge/submit.
# Only the direct + page_first_direct shape (the one that does a blocking
# device_indices.cpu() on the scheduler thread) is eligible.
_ASYNC_BACKUP_SUBMIT = _hc_os.getenv(
"SGLANG_ASYNC_BACKUP_SUBMIT", "0"
).strip().lower() not in ("", "0", "false", "no", "off")
_ASYNC_BACKUP_ENQUEUE_TIMEOUT = float(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_ENQUEUE_TIMEOUT", "60") or 60
)
_ASYNC_BACKUP_IDLE_TIMEOUT = float(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_IDLE_TIMEOUT", "120") or 120
)
_ASYNC_BACKUP_PINNED_BYTES_MAX = int(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_PINNED_MB", "2048") or 2048
) * 1024 * 1024
_ASYNC_BACKUP_MAX_OPS = int(
_hc_os.getenv("SGLANG_ASYNC_BACKUP_MAX_OPS", "256") or 256
)
class LayerLoadingEvent:
def __init__(self, num_layers: int):
self._num_layers = num_layers
self.load_events = [device_module.Event() for _ in range(num_layers)]
self.start_event = device_module.Event() # start event on controller stream
def complete(self, layer_index: int):
assert 0 <= layer_index < self._num_layers
self.load_events[layer_index].record()
def wait(self, layer_index: int):
device_module.current_stream().wait_event(self.load_events[layer_index])
@property
def finish_event(self):
return self.load_events[-1]
class LayerDoneCounter:
def __init__(self, num_layers: int):
self.num_layers = num_layers
# extra producer and consumer counters for overlap mode
self.num_counters = 3
self.events = [LayerLoadingEvent(num_layers) for _ in range(self.num_counters)]
self.producer_index = -1
self.consumer_index = -1
def update_producer(self):
self.producer_index = (self.producer_index + 1) % self.num_counters
assert self.events[self.producer_index].finish_event.query(), (
"Producer finish event should be ready before being reused."
)
return self.producer_index
def set_consumer(self, index: int):
self.consumer_index = index
def wait_until(self, threshold: int):
if self.consumer_index < 0:
return
self.events[self.consumer_index].wait(threshold)
def reset(self):
self.producer_index = -1
self.consumer_index = -1
class CacheOperation:
counter = 0
def __init__(
self,
host_indices: torch.Tensor,
device_indices: torch.Tensor,
node_id: int,
priority: Optional[int] = None,
pool_transfers: Optional[List[PoolTransfer]] = None,
):
self.host_indices = host_indices
self.device_indices = device_indices
self.node_ids = [node_id]
self.data = None
self.pool_transfers = pool_transfers
# Set when SGLANG_ASYNC_BACKUP_SUBMIT copied this op's device indices
# into pinned host buffers (see _start_async_index_copy).
self.async_parts = None
self.id = CacheOperation.counter
CacheOperation.counter += 1
# default priority is the order of creation
self.priority = priority if priority is not None else self.id
@staticmethod
def _merge_pool_transfers(
ops: List[CacheOperation],
) -> Optional[List[PoolTransfer]]:
grouped: dict[tuple[PoolName, Optional[PoolName]], List[PoolTransfer]] = {}
for op in ops:
for transfer in op.pool_transfers or []:
grouped.setdefault(
(transfer.name, transfer.indices_from_pool), []
).append(transfer)
if not grouped:
return None
def cat_or_none(tensors):
parts = [tensor for tensor in tensors if tensor is not None]
return torch.cat(parts) if parts else None
return [
PoolTransfer(
name=transfers[0].name,
host_indices=cat_or_none(t.host_indices for t in transfers),
device_indices=cat_or_none(t.device_indices for t in transfers),
keys=[key for t in transfers if t.keys for key in t.keys] or None,
hit_policy=transfers[0].hit_policy,
indices_from_pool=transfers[0].indices_from_pool,
)
for transfers in grouped.values()
]
@staticmethod
def merge_ops(ops: List[CacheOperation]) -> CacheOperation:
assert ops
if len(ops) == 1:
return ops[0]
host_indices = torch.cat([op.host_indices for op in ops])
device_indices = torch.cat([op.device_indices for op in ops])
node_ids = []
priority = min(op.priority for op in ops)
for op in ops:
node_ids.extend(op.node_ids)
merged_op = CacheOperation(
host_indices,
device_indices,
-1,
priority,
pool_transfers=CacheOperation._merge_pool_transfers(ops),
)
merged_op.node_ids = node_ids
return merged_op
def __lt__(self, other: CacheOperation):
return self.priority < other.priority
@dataclass
class AsyncIndexCopy:
"""One device->pinned-host index copy started where the backup op was made."""
key: tuple
src: torch.Tensor
dst: torch.Tensor
nbytes: int
@dataclass
class AsyncIndexParts:
"""Per-op pinned index parts plus the event that guarantees they are final."""
copies: List[AsyncIndexCopy]
ready_event: object
nbytes: int
@dataclass
class WriteSubmission:
ops: List[CacheOperation]
tracer: object
enqueue_mono_ns: int
@dataclass
class WriteResult:
ack: Optional[HiCacheAck]
ops: List[CacheOperation]
error: Optional[BaseException]
class HiCacheAck(NamedTuple):
start_event: device_module.Event
finish_event: device_module.Event
node_ids: List[int]
num_tokens: int = 0
timing_enabled: bool = False
# Tokens transferred per host pool (PoolName value -> count).
num_tokens_by_pool: Optional[dict[str, int]] = None
# Total bytes moved by the op across all pools, including draft piggyback
# and sidecar transfers that the per-pool token counts exclude.
num_bytes: int = 0
@dataclass
class PrefetchAck:
"""ACK for prefetch operation.
A sequence of PrefetchAck is sent to the scheduler thread via ack_prefetch_queue,
indicating progress or completion of the prefetch operation.
For example, a prefetch operation may results into the following sequence of PrefetchAck:
1. PrefetchAck(completed_tokens = 128)
2. PrefetchAck(completed_tokens = 256)
3. PrefetchAck(pool_hits={INDEXER: 256})
4. PrefetchAck(completed_req = True)
The last PrefetchAck always specifies completed_req = True.
"""
rid: str
operation: PrefetchOperation
# Number of hits in KV pool.
completed_tokens: Optional[int] = None
# Number of hits in extra pools.
pool_hits: Optional[dict[str, int]] = None
completed_req: Optional[bool] = None
class StorageOperation:
counter = 0
def __init__(
self,
host_indices: Optional[torch.Tensor],
token_ids: List[int],
last_hash: Optional[str] = None,
hash_value: Optional[List[str]] = None,
prefix_keys: Optional[List[str]] = None,
):
self.host_indices = host_indices
self.token_ids = token_ids
self.last_hash = last_hash
self.completed_tokens = 0
self.hash_value = hash_value if hash_value is not None else []
self.prefix_keys = prefix_keys
# Full queried page-hash chain, set by _storage_hit_query before
# hash_value is truncated to the hit boundary; the tail is the
# absence signal that invalidates buffer-mode existence beliefs.
self.all_hash_values: Optional[List[str]] = None
# Prefetch-outcome accounting, set at enqueue by the tree cache.
self.stats_requested_tokens = 0
# Absolute token offset at which this storage-prefetched span starts.
self.storage_start = 0
self.id = StorageOperation.counter
StorageOperation.counter += 1
def __lt__(self, other: StorageOperation):
return self.id < other.id
# Buffer-mode staging budgets. Prefetch staging is latency-critical
# (wait_complete gates TTFT), so loads may fill the pool up to this fraction
# before new prefetches are declined.
HICACHE_LOAD_POOL_USAGE_FRACTION = 0.9
# Write-staging floor: writes are deferrable, so the flush gate grows the
# write window dynamically into whatever load staging is not using.
HICACHE_WRITE_STAGING_POOL_FRACTION = 0.2
class PrefetchOperation(StorageOperation):
def __init__(
self,
request_id: str,
token_ids: List[int],
last_hash: Optional[str] = None,
prefix_keys: Optional[List[str]] = None,
):
self.request_id = request_id
self._lock = threading.Lock()
self._terminated_flag = False
self.storage_hit_count = 0
self.start_time = time.monotonic()
super().__init__(None, token_ids, last_hash, prefix_keys=prefix_keys)
def mark_terminate(self):
with self._lock:
self._terminated_flag = True
def is_terminated(self) -> bool:
with self._lock:
return self._terminated_flag
class HiCacheController:
def __init__(
self,
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
mem_pool_host: HostKVCache,
page_size: int,
tp_group: torch.distributed.ProcessGroup,
load_cache_event: threading.Event,
attn_cp_group: Optional[torch.distributed.ProcessGroup] = None,
attn_tp_group: Optional[torch.distributed.ProcessGroup] = None,
pp_group: Optional[torch.distributed.ProcessGroup] = None,
write_policy: str = "write_through_selective",
io_backend: str = "",
storage_backend: Optional[str] = None,
prefetch_threshold: int = 256,
model_name: Optional[str] = None,
storage_backend_extra_config: Optional[dict] = None,
enable_storage_metrics: bool = False,
host_memory_mode: str = "cache",
):
self.tp_group = tp_group
self.host_memory_mode = host_memory_mode
self.attn_cp_group = attn_cp_group
self.attn_tp_group = attn_tp_group
self.pp_group = pp_group
self.prefetch_hits_sync_groups: List[torch.distributed.ProcessGroup] = []
self.prefetch_completion_sync_groups: List[torch.distributed.ProcessGroup] = []
self.mem_pool_device_allocator = token_to_kv_pool_allocator
mem_pool_device = token_to_kv_pool_allocator.get_kvcache()
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool
if isinstance(mem_pool_device, HybridLinearKVPool):
mem_pool_device = mem_pool_device.full_kv_pool
self.mem_pool_device = mem_pool_device
self.mem_pool_host = mem_pool_host
self.storage_host_pool = mem_pool_host
self.write_policy = write_policy
self.page_size = page_size
self.io_backend = io_backend
self.enable_storage = False
self.storage_backend = None
self.storage_backend_type = None
self.enable_storage_metrics = enable_storage_metrics
# Buffer mode: wired by the tree cache after attach; the load rate
# limiter subtracts write staging from actual pool usage.
self.host_write_staged_tokens_fn: Optional[Callable[[], int]] = None
# Default storage page IO functions (may be overridden by attach).
self.page_get_func = self._generic_page_get
self.page_set_func = self._generic_page_set
# Dedicated stop event for storage background threads (prefetch/backup).
self.storage_stop_event = threading.Event()
# Storage control queues, (re)created whenever the storage threads start.
self.prefetch_buffer: Optional[Queue[PrefetchOperation]] = None
self.prefetch_sync_queue: Optional[Queue[PrefetchAck]] = None
self.prefetch_hit_queue: Optional[Queue[StorageOperation]] = None
self.ack_prefetch_queue = Queue[PrefetchAck]()
self.ack_backup_queue: Optional[Queue[StorageOperation]] = None
self.host_mem_release_queue: Optional[Queue[torch.Tensor]] = None
self.device = self.mem_pool_device.device
self.transfer_layer_id_max = self.mem_pool_device.layer_num
self.layer_done_counter = LayerDoneCounter(self.transfer_layer_id_max)
self.mem_pool_device.register_layer_transfer_counter(self.layer_done_counter)
if write_policy not in [
"write_through",
"write_through_selective",
"write_back",
]:
raise ValueError(f"Invalid write policy: {write_policy}")
# self.write_queue = PriorityQueue[CacheOperation]()
self.load_queue: List[CacheOperation] = []
self.write_queue: List[CacheOperation] = []
self.ack_load_queue: List[HiCacheAck] = []
# Set by the scheduler to the forward stream; gate load-back H2D and
# write D2H behind in-flight forwards (see start_loading, start_writing).
self.load_fence_stream = None
self.write_fence_stream = None
self.ack_write_queue: List[HiCacheAck] = []
# [ASYNC_BACKUP] Pinned index copies are started where the backup op is
# created; a single worker waits the per-op events, merges the CPU
# index parts and submits the KV copy. Queues are bounded by the state
# lock/condition below; write_queue/ack_write_queue stay scheduler-only.
self._indices_stream = None
self._write_submit_queue: Optional[Queue] = None
self._write_result_queue: Queue = Queue()
self._write_submit_stop = threading.Event()
self._write_submit_thread: Optional[threading.Thread] = None
self._write_submit_state_lock = threading.Lock()
self._write_submit_cond = threading.Condition(self._write_submit_state_lock)
self._write_submit_active = 0
self._async_pinned_bytes = 0
self._retire_parts: List["AsyncIndexParts"] = []
self.l2_transfer_engine = L2TransferEngine(io_backend)
# If a storage backend is provided at startup, treat it as an implicit attach,
# so init/runtime share the same lifecycle semantics and code paths.
if storage_backend is not None:
try:
self.attach_storage_backend(
storage_backend=storage_backend,
prefetch_threshold=prefetch_threshold,
model_name=model_name,
storage_backend_extra_config=storage_backend_extra_config,
)
except ValueError as e:
# Preserve the historical error shape on init for unknown backends.
raise ValueError(f"Failed to create storage backend: {e}") from e
def get_attn_cp_rank_and_size(self) -> tuple[int, int]:
"""Derive CP rank/size from the attn_cp process group."""
if self.attn_cp_group is not None:
return (
torch.distributed.get_rank(group=self.attn_cp_group),
torch.distributed.get_world_size(group=self.attn_cp_group),
)
return 0, 1
def _create_sync_groups(self) -> List[torch.distributed.ProcessGroup]:
from sglang.srt.distributed.parallel_state import create_custom_parallel_group
groups: List[torch.distributed.ProcessGroup] = []
seen_rank_sets = set()
if self.attn_cp_group is not None or self.attn_tp_group is not None:
base_groups = [self.attn_cp_group, self.attn_tp_group]
else:
base_groups = [self.tp_group]
if self.pp_group is not None:
base_groups.append(self.pp_group)
for group in base_groups:
if group is None or torch.distributed.get_world_size(group=group) == 1:
continue
group_ranks = tuple(torch.distributed.get_process_group_ranks(group))
if group_ranks in seen_rank_sets:
continue
seen_rank_sets.add(group_ranks)
groups.append(
create_custom_parallel_group(
group_ranks=list(group_ranks), backend="gloo"
)
)
return groups
def _destroy_sync_groups(
self, groups: List[torch.distributed.ProcessGroup]
) -> None:
for group in groups:
try:
torch.distributed.destroy_process_group(group)
except Exception:
pass
def _all_reduce(
self,
tensor: torch.Tensor,
op,
groups: List[torch.distributed.ProcessGroup],
) -> None:
for group in groups:
torch.distributed.all_reduce(tensor, op=op, group=group)
def _start_storage_threads(self):
"""Start storage prefetch/backup threads and their queues.
This is used by runtime attach, and also by reset when storage is enabled.
"""
assert self.enable_storage
assert not self.storage_stop_event.is_set()
self.prefetch_thread = threading.Thread(
target=self.prefetch_thread_func, daemon=True
)
self.prefetch_io_aux_thread = threading.Thread(
target=self.prefetch_io_aux_func, daemon=True
)
self.prefetch_sync_thread = threading.Thread(
target=self.prefetch_sync_thread_func, daemon=True
)
self.backup_thread = threading.Thread(
target=self.backup_thread_func, daemon=True
)
self.prefetch_queue = Queue()
self.backup_queue = Queue()
self.prefetch_buffer = Queue()
self.prefetch_sync_queue = Queue()
self.prefetch_hit_queue = Queue()
self.ack_prefetch_queue = Queue()
self.ack_backup_queue = Queue()
self.host_mem_release_queue = Queue()
self.prefetch_thread.start()
self.prefetch_io_aux_thread.start()
self.prefetch_sync_thread.start()
self.backup_thread.start()
def _stop_storage_threads(self):
"""Stop storage prefetch/backup threads and drain internal queues.
Caller should ensure no in-flight requests.
"""
# Always request stop. This is safe even when storage is already disabled,
# and makes detach truly idempotent (previous partial detach may have left
# threads alive).
# NOTE: do NOT clear storage_stop_event unless threads have fully stopped; otherwise
# a still-alive thread may resume and touch released state.
self.storage_stop_event.set()
# Best-effort wakeups so threads exit promptly even if blocked on queues.
try:
if hasattr(self, "prefetch_queue"):
self.prefetch_queue.put_nowait(None)
if hasattr(self, "backup_queue"):
self.backup_queue.put_nowait(None)
if hasattr(self, "prefetch_buffer"):
self.prefetch_buffer.put_nowait(None)
if hasattr(self, "prefetch_sync_queue"):
self.prefetch_sync_queue.put_nowait(None)
except Exception:
pass
# Best-effort joins (threads are daemon, but join keeps state clean).
threads = []
if hasattr(self, "prefetch_thread"):
threads.append(self.prefetch_thread)
if hasattr(self, "backup_thread"):
threads.append(self.backup_thread)
if hasattr(self, "prefetch_io_aux_thread"):
threads.append(self.prefetch_io_aux_thread)
if hasattr(self, "prefetch_sync_thread"):
threads.append(self.prefetch_sync_thread)
for t in threads:
try:
t.join(timeout=10)
except Exception:
pass
alive = [t for t in threads if getattr(t, "is_alive", lambda: False)()]
if alive:
logger.error(
"Failed to stop HiCache storage threads cleanly: %s",
[getattr(t, "name", repr(t)) for t in alive],
)
raise RuntimeError("Failed to stop HiCache storage threads cleanly.")
# A subclass whose attach continues registering host pools after super()
# returns fires on_host_pools_registered() itself, once.
_registers_host_pools_after_super_attach = False
def attach_storage_backend(
self,
storage_backend: str,
prefetch_threshold: int = 256,
model_name: Optional[str] = None,
storage_backend_extra_config: Optional[dict] = None,
):
"""Attach (enable) storage backend at runtime.
Requirement: no in-flight requests. This call is expected to run on the scheduler
thread (control path), not concurrently with prefetch/backup.
"""
if self.enable_storage:
raise RuntimeError("Storage backend already attached.")
if get_parallel().attn_dcp_size > 1:
if self.host_memory_mode == "buffer_only":
# Buffer-mode write staging still budgets physical host rows.
raise NotImplementedError(
"HiCache L3 with DCP does not support buffer_only host memory."
)
from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
if storage_backend not in ("file", "mooncake"):
raise NotImplementedError(
"HiCache L3 with DCP requires file or Mooncake storage."
)
if (
not isinstance(self.storage_host_pool, MLATokenToKVPoolHost)
or self.storage_host_pool.kv_buffer is None
):
raise NotImplementedError(
"HiCache L3 with DCP requires a materialized MLA host pool."
)
if self.mem_pool_host is not self.storage_host_pool and any(
entry.host_pool is not self.storage_host_pool
and (
entry.name != PoolName.MAMBA
or not isinstance(entry.host_pool, MambaPoolHost)
)
for entry in self.mem_pool_host.entries
):
raise NotImplementedError(
"HiCache L3 with DCP supports only MLA KV and optional KDA state."
)
if self.storage_host_pool.layout not in (
"layer_first",
"page_first",
"page_first_direct",
):
raise NotImplementedError(
"HiCache L3 with DCP requires a generic MLA page layout."
)
# Defensive: a previous partial detach may have flipped `enable_storage` but
# left background threads alive. Attaching on top of them is unsafe.
try:
self._stop_storage_threads()
except Exception as e:
raise RuntimeError(
"Cannot attach storage backend: previous detach did not stop storage threads cleanly."
) from e
# Rollback-safe init: if creation fails, keep controller state consistent
# for future attach attempts.
self.storage_backend_type = storage_backend
from sglang.srt.mem_cache.utils import get_hash_str
self.get_hash_str = get_hash_str
self.storage_config = self._generate_storage_config(
model_name, storage_backend_extra_config
)
# Use storage backend factory for dynamic backend creation
from sglang.srt.mem_cache.storage import StorageBackendFactory
try:
self.storage_backend = StorageBackendFactory.create_backend(
storage_backend, self.storage_config, self.storage_host_pool
)
self.storage_backend.register_mem_pool_host(self.storage_host_pool)
if not self._registers_host_pools_after_super_attach:
self.storage_backend.on_host_pools_registered()
self.enable_storage = True
# todo: threshold policy for prefetching
self.prefetch_threshold = max(prefetch_threshold, self.page_size)
if self.host_memory_mode == "buffer_only":
# The whole pool is transient staging; loads may fill it up
# to this fraction, and the tree's write flush gate yields
# to live fetch demand (the write fraction is a floor).
self.prefetch_capacity_limit = int(
HICACHE_LOAD_POOL_USAGE_FRACTION * self.mem_pool_host.size
)
else:
# Leave half the pool for write-back. Prefetch occupancy counts
# logical tokens, not physical rows on each DCP rank.
self.prefetch_capacity_limit = int(
0.5 * self.mem_pool_host.logical_size
)
# tracking the number of tokens locked in prefetching, updated by the main scheduler thread
self.prefetch_tokens_occupied = 0
# Use dedicated gloo groups so storage prefetch sync is isolated
# from other collectives and consistent across CPxTP participants.
self.prefetch_hits_sync_groups = self._create_sync_groups()
self.prefetch_completion_sync_groups = self._create_sync_groups()
# Select the get and set functions
self.page_get_func = self._generic_page_get
self.page_set_func = self._generic_page_set
if (
self.storage_backend_type
in [
"hf3fs",
"mooncake",
"npu_memcache",
"eic",
"nixl",
"simm",
"mori",
"tensorcast",
"seaweedfs",
]
) or (
self.storage_backend_type == "dynamic"
and bool(self.storage_config.extra_config.get("interface_v1", 0))
):
self.page_get_func = self._page_get_zero_copy
self.page_set_func = self._page_set_zero_copy
# Ensure stop_event is clear before starting threads.
self.storage_stop_event.clear()
self._start_storage_threads()
except Exception:
# Best-effort cleanup for partial init.
try:
self._stop_storage_threads()
except Exception:
pass
self._destroy_sync_groups(self.prefetch_hits_sync_groups)
self._destroy_sync_groups(self.prefetch_completion_sync_groups)
self.prefetch_hits_sync_groups = []
self.prefetch_completion_sync_groups = []
try:
if (
hasattr(self, "storage_backend")
and self.storage_backend is not None
):
if hasattr(self.storage_backend, "close"):
self.storage_backend.close()
except Exception:
pass
self.storage_backend = None
self.storage_backend_type = None
self.enable_storage = False
self.page_get_func = self._generic_page_get
self.page_set_func = self._generic_page_set
raise
def detach_storage_backend(self):
"""Detach (disable) storage backend at runtime.
Requirement: no in-flight requests. This will stop storage threads and release
the backend instance (best-effort close).
"""
# Async backup submissions own pinned index buffers and ACKs; stop them
# before any queue/pool teardown so no result is lost.
self._stop_write_submission_thread()
# Idempotent cleanup: even if `enable_storage` is already False,
# we may still have leftover resources (threads/backend/process group) from a
# previous partial detach. We attempt cleanup whenever possible.
try:
self._stop_storage_threads()
except Exception as e:
# Do not proceed tearing down backend/process group if threads are not
# fully stopped; otherwise still-alive threads may touch released state.
# Caller can retry detach.
logger.exception("Stop storage threads failed: %s", e)
# IMPORTANT: Do not silently succeed. Upper layers rely on exceptions here
# to avoid flipping `enable_storage` flags while threads are still alive.
raise RuntimeError("Stop storage threads failed; detach aborted.") from e
# Best-effort destroy process groups created for storage ops.
self._destroy_sync_groups(
self.prefetch_hits_sync_groups + self.prefetch_completion_sync_groups
)
self.prefetch_hits_sync_groups = []
self.prefetch_completion_sync_groups = []
# Best-effort close (some backends rely on GC/destructor).
try:
if (
hasattr(self, "storage_backend")
and self.storage_backend is not None
and hasattr(self.storage_backend, "close")
):
self.storage_backend.close()
except Exception:
logger.exception("Failed to close storage backend cleanly.")
self.storage_backend = None
self.storage_backend_type = None
self.enable_storage = False
self.page_get_func = self._generic_page_get
self.page_set_func = self._generic_page_set
# Now it's safe to clear the stop event for future re-attach.
self.storage_stop_event.clear()
@property
def backup_skip(self) -> bool:
return not self.storage_config.is_storage_writer
def _generate_storage_config(
self,
model_name: Optional[str] = None,
storage_backend_extra_config: Optional[dict] = None,
):
if storage_backend_extra_config is None:
storage_backend_extra_config = {}
parallel = get_parallel()
if is_dp_attention_enabled():
tp_rank = parallel.attn_tp_rank
tp_size = parallel.attn_tp_size
dp_rank = parallel.attn_dp_rank
else:
tp_rank = parallel.tp_rank
tp_size = parallel.tp_size
dp_rank = 0
# Currently, NPUMLATokenToKVPool is the subclass of MLATokenToKVPool.
# DeepSeekV4TokenToKVPool has compressed MLA-style rank-replicated cache
# data. storage only needs rank 0 to write it back.
from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool
is_mla_model = isinstance(self.mem_pool_device, MLATokenToKVPool)
is_compressed_mla_model = isinstance(
self.mem_pool_device, DeepSeekV4TokenToKVPool
)
is_rank_replicated = is_mla_model or is_compressed_mla_model
# Least Common Multiple among heterogeneous tp size
tp_lcm_size = storage_backend_extra_config.pop("tp_lcm_size", None)
should_split_heads = False
if tp_lcm_size:
assert tp_lcm_size % tp_size == 0, (
"tp_lcm_size must be divisible by tp_size."
)
should_split_heads = (
not is_rank_replicated
and self.mem_pool_host.layout == "page_head"
and tp_lcm_size > tp_size
)
attn_cp_rank, attn_cp_size = self.get_attn_cp_rank_and_size()
dcp_size = get_parallel().attn_dcp_size
model_name = storage_model_name(model_name, self.storage_host_pool)
return HiCacheStorageConfig(
tp_rank=tp_rank,
tp_size=tp_size,
pp_rank=parallel.pp_rank,
pp_size=parallel.pp_size,
attn_cp_rank=attn_cp_rank,
attn_cp_size=attn_cp_size,
# TODO(hzh): Rename is_mla_model to is_rank_replicated.
is_mla_model=is_rank_replicated,
enable_storage_metrics=self.enable_storage_metrics,
is_page_first_layout=self.mem_pool_host.layout == "page_first",
model_name=model_name,
tp_lcm_size=tp_lcm_size,
should_split_heads=should_split_heads,
dp_rank=dp_rank,
extra_config=storage_backend_extra_config,
dcp_size=dcp_size,
dcp_rank=get_parallel().attn_dcp_rank,
logical_page_size=self.page_size if dcp_size > 1 else None,
)
def reset(self):
# Reuse detach's queue wakeups and bounded joins, and fail if any
# storage thread remains alive before clearing shared state.
self._stop_write_submission_thread()
self._stop_storage_threads()
self.write_queue.clear()
self.load_queue.clear()
self.ack_write_queue.clear()
self.ack_load_queue.clear()
if self.enable_storage:
self.prefetch_queue.queue.clear()
self.backup_queue.queue.clear()
self.prefetch_buffer.queue.clear()
self.prefetch_sync_queue.queue.clear()
self.prefetch_hit_queue.queue.clear()
self.ack_prefetch_queue.queue.clear()
self.ack_backup_queue.queue.clear()
self.host_mem_release_queue.queue.clear()
self.prefetch_tokens_occupied = 0
self.storage_stop_event.clear()
if self.enable_storage:
self.prefetch_thread = threading.Thread(
target=self.prefetch_thread_func, daemon=True
)
self.prefetch_io_aux_thread = threading.Thread(
target=self.prefetch_io_aux_func, daemon=True
)
self.prefetch_sync_thread = threading.Thread(
target=self.prefetch_sync_thread_func, daemon=True
)
self.backup_thread = threading.Thread(
target=self.backup_thread_func, daemon=True
)
self.prefetch_thread.start()
self.prefetch_io_aux_thread.start()
self.prefetch_sync_thread.start()
self.backup_thread.start()
def has_inflight_device_transfers(self) -> bool:
"""Whether queued or unacknowledged L2 transfers still use device rows."""
with self._write_submit_state_lock:
async_pending = (
self._write_submit_active > 0
or self._async_pinned_bytes > 0
or not self._write_result_queue.empty()
)
return bool(
self.write_queue
or self.load_queue
or self.ack_write_queue
or self.ack_load_queue
or async_pending
)
# ---- async backup submission -----------------------------------------
def async_backup_enabled(self) -> bool:
return (
_ASYNC_BACKUP_SUBMIT
and self.io_backend == "direct"
and getattr(self.mem_pool_host, "layout", None) == "page_first_direct"
)
def _async_submit_applicable(self) -> bool:
return (
self.async_backup_enabled()
and len(self.write_queue) <= _ASYNC_BACKUP_MAX_OPS
and all(op.async_parts is not None for op in self.write_queue)
)
def _ensure_indices_stream(self):
if self._indices_stream is None:
self._indices_stream = device_module.Stream()
return self._indices_stream
def _needs_device_index_copy(self, host_pool, indices) -> bool:
if indices is None or not getattr(indices, "is_cuda", False):
return False
# move_indices() returns shared-layout ids untouched; converting them
# here would hand the transfer a different interface than the sync path.
return not uses_shared_host_layout(host_pool)
def _async_index_targets(self, op: CacheOperation):
targets = []
if self._needs_device_index_copy(
self.mem_pool_host.anchor_entry.host_pool, op.device_indices
):
targets.append(((None, None), op.device_indices))
for transfer in op.pool_transfers or []:
if transfer.device_indices is None:
continue
entry = self.mem_pool_host.entry_map.get(transfer.name)
if entry is None:
continue
if not self._needs_device_index_copy(
entry.host_pool, transfer.device_indices
):
continue
key = (transfer.name, transfer.indices_from_pool)
targets.append((key, transfer.device_indices))
return targets
def _start_async_index_copy(self, op: CacheOperation) -> None:
"""Start the pinned index D2H copy now, off the scheduler stream.
Runs in the scheduler thread at backup-op creation time. The copy runs
on a dedicated stream ordered after this instant, so it does not inherit
later fences (the WAR barrier of a later forward) and the scheduler
never blocks in ``.cpu()``.
"""
if not self.async_backup_enabled():
return
targets = self._async_index_targets(op)
if not targets:
return
nbytes = sum(int(t.numel()) * int(t.element_size()) for _, t in targets)
with self._write_submit_state_lock:
if self._async_pinned_bytes + nbytes > _ASYNC_BACKUP_PINNED_BYTES_MAX:
return
self._async_pinned_bytes += nbytes
try:
stream = self._ensure_indices_stream()
source_stream = device_module.current_stream()
copies: List[AsyncIndexCopy] = []
with device_module.StreamContext(stream):
stream.wait_stream(source_stream)
for key, src in targets:
# Keep the device rows alive until the copy has run, and pin
# the destination so copy_ is a real asynchronous D2H.
src.record_stream(stream)
dst = torch.empty(
src.shape,
dtype=src.dtype,
device="cpu",
pin_memory=True,
)
dst.copy_(src, non_blocking=True)
copies.append(
AsyncIndexCopy(
key=key,
src=src,
dst=dst,
nbytes=int(src.numel()) * int(src.element_size()),
)
)
ready_event = device_module.Event()
ready_event.record(stream)
except BaseException:
with self._write_submit_state_lock:
self._async_pinned_bytes -= nbytes
logger.exception("Async backup index copy failed; keeping sync path")
return
op.async_parts = AsyncIndexParts(
copies=copies, ready_event=ready_event, nbytes=nbytes
)
def _ensure_write_submit_thread(self) -> None:
if self._write_submit_queue is None:
self._write_submit_queue = Queue(maxsize=64)
thread = self._write_submit_thread
if thread is not None and thread.is_alive():
return
self._write_submit_stop.clear()
thread = threading.Thread(
target=self._write_submit_worker,
name="hicache-async-backup",
daemon=True,
)
self._write_submit_thread = thread
thread.start()
def _enqueue_async_write(self, ops, tracer) -> bool:
"""Enqueue one submission; False means "fall back to the sync path".
A full queue or a timeout is not fatal: the caller submits the same ops
synchronously. Before returning False we wait for the worker to become
idle so the fallback cannot race another submission on the shared
device_to_host_stream.
"""
self._ensure_write_submit_thread()
item = WriteSubmission(
ops=ops, tracer=tracer, enqueue_mono_ns=time.monotonic_ns()
)
with self._write_submit_cond:
self._write_submit_active += 1
try:
with batch_telemetry.backup_phase("enqueue"):
self._write_submit_queue.put(
item, timeout=_ASYNC_BACKUP_ENQUEUE_TIMEOUT
)
except Full:
with self._write_submit_cond:
self._write_submit_active -= 1
self._write_submit_cond.notify_all()
logger.warning(
"Async backup submit queue full after %.1fs; falling back to "
"the synchronous path",
_ASYNC_BACKUP_ENQUEUE_TIMEOUT,
)
self.wait_write_submissions_idle()
return False
return True
def _materialize_async_indices(self, ops) -> None:
"""Replace device index tensors with the pinned CPU copies (worker)."""
for op in ops:
parts = op.async_parts
if parts is None:
continue
mapping = {copy.key: copy.dst for copy in parts.copies}
anchor = mapping.get((None, None))
if anchor is not None:
op.device_indices = anchor
for transfer in op.pool_transfers or []:
dst = mapping.get((transfer.name, transfer.indices_from_pool))
if dst is not None:
transfer.device_indices = dst
def _write_submit_worker(self) -> None:
while True:
item = self._write_submit_queue.get()
if item is None:
break
tracer = item.tracer
previous = batch_telemetry.backup_flush_attach(tracer)
try:
with batch_telemetry.backup_phase("worker"):
for op in item.ops:
parts = op.async_parts
if parts is not None:
parts.ready_event.synchronize()
self._materialize_async_indices(item.ops)
merged = CacheOperation.merge_ops(item.ops)
ack = self._submit_write_operation(merged)
result = WriteResult(ack=ack, ops=item.ops, error=None)
except BaseException as exc: # noqa: BLE001 - fail closed
logger.exception("Async backup submission failed")
result = WriteResult(ack=None, ops=item.ops, error=exc)
finally:
batch_telemetry.backup_flush_detach(previous)
self._release_async_parts(item.ops)
try:
batch_telemetry.backup_flush_end(tracer)
except Exception:
pass
# Publish the result before marking the submission done, so an idle
# waiter cannot observe active==0 with the result still missing.
self._write_result_queue.put(result)
with self._write_submit_cond:
self._write_submit_active -= 1
self._write_submit_cond.notify_all()
def _release_async_parts(self, ops) -> None:
released = 0
for op in ops:
parts = op.async_parts
if parts is None:
continue
op.async_parts = None
parts.copies.clear()
released += parts.nbytes
if released:
with self._write_submit_state_lock:
self._async_pinned_bytes -= released
def _retire_async_parts(self, ops) -> None:
"""Release copies that will not be consumed (sync fallback)."""
for op in ops:
parts = op.async_parts
if parts is not None:
self._retire_parts.append(parts)
op.async_parts = None
self._reap_retired_parts()
def _reap_retired_parts(self) -> None:
if not self._retire_parts:
return
pending = []
for parts in self._retire_parts:
try:
done = parts.ready_event.query()
except Exception:
done = True
if done:
parts.copies.clear()
with self._write_submit_state_lock:
self._async_pinned_bytes -= parts.nbytes
else:
pending.append(parts)
self._retire_parts = pending
def drain_write_submission_results(self) -> None:
"""Move finished async ACKs into ack_write_queue (scheduler thread)."""
while True:
try:
result = self._write_result_queue.get_nowait()
except Empty:
break
if result.error is not None:
raise RuntimeError("async backup submission failed") from result.error
if result.ack is not None:
self.ack_write_queue.append(result.ack)
self._reap_retired_parts()
def wait_write_submissions_idle(
self, timeout: float = _ASYNC_BACKUP_IDLE_TIMEOUT
) -> None:
deadline = time.monotonic() + timeout
with self._write_submit_cond:
while self._write_submit_active > 0:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise RuntimeError(
"Timed out waiting for async backup submissions"
)
self._write_submit_cond.wait(remaining)
def _stop_write_submission_thread(self) -> None:
if self._write_submit_thread is None:
self.drain_write_submission_results()
self._reap_retired_parts()
if self._retire_parts:
raise RuntimeError("async backup index copies still in flight")
return
try:
self.wait_write_submissions_idle()
finally:
self._write_submit_stop.set()
queue = self._write_submit_queue
if queue is not None:
try:
queue.put(None, timeout=5.0)
except Exception:
logger.exception("Failed to wake the async backup worker")
thread = self._write_submit_thread
if thread is not None:
thread.join(timeout=30.0)
if thread.is_alive():
raise RuntimeError("async backup submit worker did not stop")
self.drain_write_submission_results()
if self._retire_parts:
raise RuntimeError("async backup index copies still in flight")
self._write_submit_thread = None
self._write_submit_queue = None
def write(
self,
device_indices: torch.Tensor,
priority: Optional[int] = None,
node_id: int = -1,
extra_pools: Optional[List[PoolTransfer]] = None,
) -> Optional[torch.Tensor]:
"""
Back up KV caches from device memory to host memory.
"""
if extra_pools:
raise ValueError("Side-pool transfers require HybridCacheController.")
host_indices = self.mem_pool_host.alloc(len(device_indices))
if host_indices is None:
return None
op = CacheOperation(host_indices, device_indices, node_id, priority)
self._start_async_index_copy(op)
self.write_queue.append(op)
self.start_writing()
return host_indices
def start_writing(self) -> None:
if len(self.write_queue) == 0:
# Empty flushes are the common case when write-through fires every
# batch: counted so a small `flush_backups` cannot be explained as
# "one big flush per loop" without noticing the empty ones.
batch_telemetry.detail_count("flush_calls")
batch_telemetry.detail_count("flush_empty")
return
ops = list(self.write_queue)
tracer = batch_telemetry.backup_flush_begin()
async_owned = False
try:
batch_telemetry.detail_count("flush_calls")
batch_telemetry.detail_count("flush_nonempty")
self._note_write_queue(ops, len(self.ack_write_queue))
if self._async_submit_applicable():
if self._enqueue_async_write(ops, tracer):
self.write_queue.clear()
batch_telemetry.flush_note(
"sglang.flush_submission_mode", "async"
)
# The worker owns the tracer from here on (it emits once).
async_owned = True
return
# Queue full / enqueue timeout: retire the pinned copies and
# submit synchronously on this thread. `_enqueue_async_write`
# waited for the worker to go idle first, so the two paths
# never submit to device_to_host_stream concurrently.
batch_telemetry.flush_note("sglang.flush_fallback_sync", 1)
self._retire_async_parts(ops)
# Any parts that were started for this queue are not consumed by the
# sync path. Retire them unconditionally (they are released once
# their events complete) so the pinned-byte budget never leaks.
batch_telemetry.flush_note(
"sglang.flush_async_skip_ops",
sum(1 for op in ops if op.async_parts is None),
)
self._retire_async_parts(ops)
batch_telemetry.flush_note("sglang.flush_submission_mode", "sync")
with batch_telemetry.backup_phase("merge_ops"):
op = CacheOperation.merge_ops(self.write_queue)
self.write_queue.clear()
ack = self._submit_write_operation(op)
with batch_telemetry.backup_phase("ack_enqueue"):
self.ack_write_queue.append(ack)
batch_telemetry.flush_note(
"sglang.flush_ack_queue_after", len(self.ack_write_queue)
)
finally:
if not async_owned:
batch_telemetry.backup_flush_end(tracer)
def _note_write_queue(self, ops: List[CacheOperation], ack_before: int) -> None:
"""Queue-level flush dimensions (same for the sync and async paths)."""
batch_telemetry.flush_note("sglang.flush_write_queue_ops", len(ops))
batch_telemetry.flush_note("sglang.flush_io_backend", self.io_backend)
batch_telemetry.flush_note(
"sglang.flush_layout",
str(getattr(self.mem_pool_host, "layout", None) or ""),
)
batch_telemetry.flush_note("sglang.flush_ack_queue_before", ack_before)
def _submit_write_operation(self, op: CacheOperation) -> HiCacheAck:
"""Move indices, submit the D2H copy and build the ACK.
Runs on the scheduler thread (sync path) or on the async submit worker
(async path, with CPU-resident indices). It must not touch
``write_queue`` or ``ack_write_queue``; the caller owns those.
"""
tokens = len(op.device_indices)
indices = op.device_indices
batch_telemetry.flush_note("sglang.flush_tokens", int(tokens))
batch_telemetry.flush_note("sglang.flush_nodes_merged", len(op.node_ids))
batch_telemetry.flush_note(
"sglang.flush_sidecar_pools", len(op.pool_transfers or [])
)
batch_telemetry.flush_note(
"sglang.flush_indices_device", str(indices.device.type)
)
batch_telemetry.flush_note("sglang.flush_indices_dtype", str(indices.dtype))
batch_telemetry.flush_note(
"sglang.flush_indices_numel", int(indices.numel())
)
with batch_telemetry.backup_phase("move_write"):
host_indices, device_indices, pool_transfers = (
self._move_write_operation(op)
)
batch_telemetry.flush_note(
"sglang.flush_host_indices_on_cpu",
bool(getattr(host_indices, "is_cuda", False) is False),
)
batch_telemetry.flush_note(
"sglang.flush_device_indices_on_cpu",
bool(getattr(device_indices, "is_cuda", False) is False),
)
if self.write_fence_stream is not None:
# The copy may read KV a forward is still writing: under overlap
# scheduling a finished request is cached while the next forward
# writes its last token, and an evicted SWA page can belong to an
# unfinished checkpoint. Later forwards still overlap with the copy.
with batch_telemetry.backup_phase("write_fence"):
self.l2_transfer_engine.device_to_host_stream.wait_stream(
self.write_fence_stream
)
with batch_telemetry.backup_phase("build_l2"):
transfers = self._l2_transfers(
host_indices, device_indices, pool_transfers
)
batch_telemetry.flush_note("sglang.flush_l2_transfers", len(transfers))
with batch_telemetry.backup_phase("submit_d2h"):
completion = self.l2_transfer_engine.submit_device_to_host(transfers)
with batch_telemetry.backup_phase("set_done_event"):
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "write"), completion.finish_event
)
return HiCacheAck(
start_event=completion.start_event,
finish_event=completion.finish_event,
node_ids=op.node_ids,
num_tokens=len(op.device_indices),
timing_enabled=completion.timing_enabled,
num_tokens_by_pool=self._num_tokens_by_pool(op),
num_bytes=self._transfer_num_bytes(op),
)
def _transfer_num_bytes(self, op: CacheOperation) -> int:
return len(op.device_indices) * self.mem_pool_host.size_per_token
def _num_tokens_by_pool(self, op: CacheOperation) -> dict[str, int]:
return {PoolName.KV.value: len(op.device_indices)}
def load(
self,
host_indices: torch.Tensor,
priority: Optional[int] = None,
node_id: int = -1,
) -> Optional[torch.Tensor]:
"""
Load KV caches from host memory to device memory.
"""
device_indices = self.mem_pool_device_allocator.alloc(len(host_indices))
if device_indices is None:
return None
self.load_queue.append(
CacheOperation(host_indices, device_indices, node_id, priority)
)
return device_indices
def move_indices(
self, host_indices: torch.Tensor, device_indices: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
# move indices to GPU if using kernels, to host if using direct indexing
if self.io_backend == "kernel":
if not host_indices.is_cuda:
with batch_telemetry.backup_phase("indices_h2d"):
host_indices = host_indices.to(self.device, non_blocking=True)
return host_indices, device_indices
elif self.io_backend == "direct":
if self.mem_pool_host.layout == "layer_first":
with batch_telemetry.backup_phase("indices_cpu"):
device_indices = device_indices.cpu()
with batch_telemetry.backup_phase("index_sort"):
host_indices, idx = host_indices.sort()
with batch_telemetry.backup_phase("index_select"):
return host_indices, device_indices.index_select(0, idx)
elif self.mem_pool_host.layout == "page_first_direct":
# Synchronous D2H of the token indices: the host wall time of
# this call is the scheduler blocking on the copy, not a
# device bandwidth measurement.
with batch_telemetry.backup_phase("indices_cpu"):
device_indices = device_indices.cpu()
return host_indices, device_indices
else:
raise ValueError(
f"Unsupported layout {self.mem_pool_host.layout!r} for io backend 'direct'"
)
elif self.io_backend == "kernel_ascend":
from sglang.srt.mem_cache.pool_host.npu_memfabric import (
ascendc_io_enabled,
to_device_no_sync,
)
if ascendc_io_enabled():
# The fused acc_offload kv_exchange kernel reads the token
# indices directly on the device; keeping them there avoids
# the D2H sync that would serialize the layer-group pipeline.
# (The legacy memcpy2d exchange op still wants CPU indices and
# converts them itself.)
# Upload through pinned memory: host_indices comes from the
# radix-tree match as a pageable CPU tensor, and a pageable
# .to(device) completes with a stream synchronize that drains
# all compute queued on the current (default) stream.
if host_indices.device != self.device:
host_indices = to_device_no_sync(host_indices, self.device)
return host_indices, device_indices
return host_indices, device_indices.cpu()
else:
raise ValueError(f"Unsupported io backend")
def _move_write_operation(
self, op: CacheOperation
) -> tuple[torch.Tensor, torch.Tensor, Optional[List[PoolTransfer]]]:
"""Keep CPU host indices only for page-first staged write-back."""
if (
self.io_backend == "kernel"
and self.mem_pool_host.layout == "page_first"
and getattr(self.mem_pool_host, "can_use_write_back_jit", False)
):
return op.host_indices, op.device_indices, op.pool_transfers
return self._move_op_indices(op)
def _move_op_indices(
self, op: CacheOperation
) -> tuple[torch.Tensor, torch.Tensor, Optional[List[PoolTransfer]]]:
return (*self.move_indices(op.host_indices, op.device_indices), None)
def _l2_transfers(
self,
host_indices: torch.Tensor,
device_indices: torch.Tensor,
pool_transfers: Optional[List[PoolTransfer]] = None,
) -> list[L2Transfer]:
transfers = [
L2Transfer(
host_pool=self.mem_pool_host,
device_pool=self.mem_pool_device,
host_indices=host_indices,
device_indices=device_indices,
)
]
return transfers
def _l2_load_transfers(
self,
host_indices: torch.Tensor,
device_indices: torch.Tensor,
pool_transfers: Optional[List[PoolTransfer]] = None,
) -> list[L2Transfer]:
return self._l2_transfers(host_indices, device_indices, pool_transfers)
def start_loading(self) -> int:
if len(self.load_queue) == 0:
return -1
producer_id = self.layer_done_counter.update_producer()
op = CacheOperation.merge_ops(self.load_queue)
host_indices, device_indices, pool_transfers = self._move_op_indices(op)
self.load_queue.clear()
producer_event = self.layer_done_counter.events[producer_id]
producer_event.start_event.record()
if self.load_fence_stream is not None:
# in overlap scheduling, reclaimed pages might still be written by the forward thread
# therefore a fence is needed for loading thread to prevent memory corruption
# todo: it's possible to use a finer-grained fence
self.l2_transfer_engine.host_to_device_stream.wait_stream(
self.load_fence_stream
)
completion = self.l2_transfer_engine.submit_host_to_device(
self._l2_load_transfers(host_indices, device_indices, pool_transfers),
start_event=producer_event.start_event,
on_layer_done=producer_event.complete,
transfer_layer_id_max=self.transfer_layer_id_max,
)
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "load"), completion.finish_event
)
self.ack_load_queue.append(
HiCacheAck(
start_event=completion.start_event,
finish_event=completion.finish_event,
node_ids=op.node_ids,
num_tokens=len(op.device_indices),
timing_enabled=completion.timing_enabled,
num_tokens_by_pool=self._num_tokens_by_pool(op),
num_bytes=self._transfer_num_bytes(op),
)
)
return producer_id
def evict_host(self, host_indices: torch.Tensor, backup_only: bool = True) -> int:
if not backup_only:
raise ValueError("Other eviction policies are not supported yet.")
self.mem_pool_host.free(host_indices)
return len(host_indices)
def prefetch(
self,
request_id: str,
new_input_tokens: List[int],
last_hash: Optional[str] = None,
prefix_keys: Optional[List[str]] = None,
) -> PrefetchOperation:
"""
Prefetch KV caches from storage backend to host memory.
"""
operation = PrefetchOperation(
request_id, new_input_tokens, last_hash, prefix_keys
)
self.prefetch_queue.put(operation)
return operation
def terminate_prefetch(self, operation):
"""
Request to terminate a prefetch operation.
Must be called in the scheduler thread.
Asynchronous prefetch tasks may be running in background threads. When all prefetch
tasks are terminated, a PrefetchAck with completed_req=True will be sent to ack_prefetch_queue.
"""
operation.mark_terminate()
return operation.completed_tokens, operation.hash_value
def append_host_mem_release(self, host_indices: torch.Tensor):
if host_indices.numel() == 0:
return
pages = host_indices.split(self.mem_pool_host.page_size)
for page in pages:
self.host_mem_release_queue.put(page)
def _page_get_zero_copy(
self, operation, hash_values, host_indices, extra_info=None
) -> int:
results = self.storage_backend.batch_get_v1(
hash_values, host_indices, extra_info
)
inc = 0
for i in range(len(hash_values)):
if not results[i]:
logger.warning(
f"Prefetch operation {operation.request_id} failed to retrieve page {hash_values[i]}."
)
break
inc += 1
return inc
# todo: deprecate
def _generic_page_get(
self, operation, hash_values, host_indices, extra_info=None
) -> int:
dummy_page_dst = [
self.storage_host_pool.get_dummy_flat_data_page() for _ in hash_values
]
page_data = self.storage_backend.batch_get(hash_values, dummy_page_dst)
if page_data is None:
return 0
count = 0
for i in range(len(hash_values)):
if page_data[i] is None:
logger.warning(
f"Prefetch operation {operation.request_id} failed to retrieve page {hash_values[i]}."
)
break
if operation.is_terminated():
break
self.storage_host_pool.set_from_flat_data_page(
host_indices[i * self.page_size],
page_data[i],
)
count += 1
return count
def _page_transfer(
self,
operation: PrefetchOperation,
ack_sink: Optional[Callable[[PrefetchAck], None]] = None,
) -> int:
# Transfer batch by batch
prefix_keys = operation.prefix_keys
kv_derived_transfers = [
transfer
for transfer in getattr(operation, "pool_transfers", None) or []
if transfer.indices_from_pool == PoolName.KV
]
all_success = True
completed_pages = 0
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
# When an error is occurred, we should keep looping and produce the same number of
# PrefetchAck as other ranks do, because prefetch_sync_thread (i.e. consumer of
# prefetch_sync_queue) perform reduce on the results. This is so tricky.
if all_success and operation.is_terminated():
all_success = False
if all_success:
batch_hashes = operation.hash_value[i : i + STORAGE_BATCH_SIZE]
batch_host_indices = operation.host_indices[
i * self.page_size : (i + len(batch_hashes)) * self.page_size
]
# Get one batch token, and update the completed_tokens if succeed
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
try:
hit_pages = self._page_transfer_kv_batch(
operation,
batch_hashes,
batch_host_indices,
extra_info,
kv_derived_transfers,
)
except Exception:
logger.exception(
"HiCache prefetch transfer failed for request %s",
operation.request_id,
)
hit_pages = 0
# Check termination
if hit_pages != len(batch_hashes):
all_success = False
if prefix_keys is not None:
prefix_keys = prefix_keys + batch_hashes
completed_pages += hit_pages
ack = PrefetchAck(
rid=operation.request_id,
completed_tokens=completed_pages * self.page_size,
operation=operation,
)
if ack_sink is None:
# Keep the direct-call behavior for existing users. The pipelined
# coordinator always supplies a sink so workers never share the queue.
self.prefetch_sync_queue.put(ack)
else:
ack_sink(ack)
return completed_pages
def _transfer_prefetch_and_collect_acks(
self, operation: PrefetchOperation
) -> List[PrefetchAck]:
"""Run one worker without publishing ACKs to the shared sync queue."""
acks = []
try:
self._page_transfer(operation, ack_sink=acks.append)
except Exception:
logger.exception(
"HiCache prefetch transfer failed for request %s",
operation.request_id,
)
return acks
def _page_transfer_kv_batch(
self,
operation: PrefetchOperation,
batch_hashes: List[str],
batch_host_indices: torch.Tensor,
extra_info: HiCacheStorageExtraInfo,
kv_derived_transfers: List[PoolTransfer],
) -> int:
"""Read a single batch from KV and KV-derived pools (e.g. indexer pool).
Return the number of hit pages. If the hits from KV and KV-derived pools differ,
clamp to the minimal number of hits.
Here, "batch" means a single unit of L3 read, not a "batch" in model forward.
"""
# Read from KV pool.
kv_hits = self.page_get_func(
operation, batch_hashes, batch_host_indices, extra_info
)
# Read from KV-derived sidecar pools, if any.
sidecar_hits: dict[str, int] = {}
if len(kv_derived_transfers) > 0:
current_kv_derived_transfers = [
PoolTransfer(
name=transfer.name,
host_indices=batch_host_indices,
keys=batch_hashes,
)
for transfer in kv_derived_transfers
]
sidecar_results = self.storage_backend.batch_get_v2(
current_kv_derived_transfers, extra_info=extra_info
)
sidecar_hits = count_pool_hits(sidecar_results)
# Clamp to minimal number of hits.
return min([kv_hits, *sidecar_hits.values()])
def prefetch_io_aux_func(self):
"""
Auxiliary function conducting IO operations for prefetching.
# [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_PREFETCH_IO_THREADS
# transfers in flight. Every ACK of one operation is published before any
# ACK of the next operation, keeping per-rank collective pairing unchanged.
"""
import time as _time
from collections import deque as _deque
from concurrent.futures import ThreadPoolExecutor as _TPE
K = max(1, _HC_PREFETCH_WORKERS)
timing = _HC_TIMING
ex = _TPE(max_workers=K, thread_name_prefix="hc-prefetch-io")
inflight = _deque()
lat = []
pages = 0
try:
while not self.storage_stop_event.is_set():
while len(inflight) < K:
try:
operation = self.prefetch_buffer.get_nowait()
except Empty:
break
if operation is None:
continue
inflight.append(
(
operation,
_time.perf_counter(),
ex.submit(
self._transfer_prefetch_and_collect_acks, operation
),
)
)
if not inflight:
try:
operation = self.prefetch_buffer.get(block=True, timeout=1)
except Empty:
continue
if operation is None:
continue
inflight.append(
(
operation,
_time.perf_counter(),
ex.submit(
self._transfer_prefetch_and_collect_acks, operation
),
)
)
operation, t0, fut = inflight[0]
try:
acks = fut.result()
except Exception:
logger.exception("HiCache prefetch transfer failed")
acks = []
finally:
inflight.popleft()
if timing:
lat.append((_time.perf_counter() - t0) * 1000.0)
pages += len(getattr(operation, "hash_value", []) or [])
for ack in acks:
self.prefetch_sync_queue.put(ack)
self.prefetch_sync_queue.put(
PrefetchAck(
rid=operation.request_id,
completed_req=True,
operation=operation,
)
)
if timing and len(lat) >= 100:
lat.sort()
print("[HC_TIMING] prefetch queue=%d inflight=%d n=%d transfer_ms p50=%.1f p95=%.1f pages=%d"
% (self.prefetch_buffer.qsize(), len(inflight), len(lat),
lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True)
lat = []
pages = 0
finally:
# Detach closes the backend after joining this outer thread. Do not
# let pool workers outlive that join and touch released state.
ex.shutdown(wait=True)
def prefetch_rate_limited(self) -> bool:
"""
Rate limit the prefetching operations to avoid overwhelming the storage backend.
"""
if self.host_memory_mode == "buffer_only":
# Gate on real pool usage: buffer mode allocates hit-sized, so
# prefetch_tokens_occupied's requested spans overstate it. Pool
# state mutates only at scheduler-thread lockstep points, so this
# stays TP-deterministic. Write staging is the write budget's
# usage; charging it here would park hits behind its storage drain.
used = self.mem_pool_host.size - self.mem_pool_host.available_size()
if self.host_write_staged_tokens_fn is not None:
used -= self.host_write_staged_tokens_fn()
return max(0, used) >= self.prefetch_capacity_limit
# cancel prefetch if too much memory is occupied
if self.prefetch_tokens_occupied >= self.prefetch_capacity_limit:
return True
# todo: more sophisticated rate limiting based on storage backend performance
return False
def alloc_prefetch_host_buffers(
self, operation: StorageOperation, need_size: int
) -> Optional[torch.Tensor]:
"""Allocate the host bounce for a storage hit."""
return self.mem_pool_host.alloc(need_size)
def can_fit_prefetch_host_buffers(
self, operation: StorageOperation, need_size: int
) -> bool:
"""Whether a prefetch bounce can fit when its host pools are empty."""
# need_size is in logical tokens; under DCP each row holds dcp_size.
return need_size <= self.mem_pool_host.logical_size
def free_prefetch_host_buffers(
self, operation: StorageOperation, host_indices: torch.Tensor
) -> None:
"""Roll back a hit-sized host bounce before transfer ownership moves."""
self.mem_pool_host.free(host_indices)
def _storage_hit_query(self, operation) -> tuple[list[str], int]:
last_hash = operation.last_hash
tokens_to_fetch = operation.token_ids
prefix_keys = operation.prefix_keys
storage_query_count = 0
hash_value = []
page_hashes = get_storage_hash_str(
tokens_to_fetch, last_hash, page_size=self.page_size
)
operation.all_hash_values = page_hashes
for start in range(0, len(page_hashes), STORAGE_BATCH_SIZE):
batch_hashes = page_hashes[start : start + STORAGE_BATCH_SIZE]
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
hit_page_num = self.storage_backend.batch_exists(batch_hashes, extra_info)
hash_value.extend(batch_hashes[:hit_page_num])
storage_query_count += hit_page_num * self.page_size
if hit_page_num < len(batch_hashes):
break
if prefix_keys is not None:
prefix_keys = prefix_keys + batch_hashes
return hash_value, storage_query_count
def prefetch_thread_func(self):
"""
Manage prefetching operations from storage backend to host memory.
"""
while (not self.storage_stop_event.is_set()) or not self.prefetch_queue.empty():
try:
operation = self.prefetch_queue.get(block=True, timeout=1)
if operation is None:
continue
try:
if operation.is_terminated():
hash_value, storage_hit_count = [], 0
else:
hash_value, storage_hit_count = self._storage_hit_query(
operation
)
except Exception:
logger.exception(
"HiCache storage query failed for request %s",
operation.request_id,
)
hash_value, storage_hit_count = [], 0
storage_hit_count = self._sync_prefetch_hit_query(
operation, storage_hit_count
)
# Record the TP-synced hit count; the scheduler thread decides
# at drain time whether to revoke (below threshold) or allocate.
operation.hash_value = hash_value[
: (storage_hit_count // self.page_size)
]
operation.storage_hit_count = storage_hit_count
self.prefetch_hit_queue.put(operation)
except Empty:
continue
def _sync_prefetch_hit_query(self, operation, storage_hit_count: int) -> int:
"""Rank-reduce a hit query (MIN) so every rank agrees on the usable
prefix. Runs for every operation, hit or miss."""
hit_count = torch.tensor(storage_hit_count, dtype=torch.int)
self._all_reduce(
hit_count,
torch.distributed.ReduceOp.MIN,
self.prefetch_hits_sync_groups,
)
return int(hit_count.item())
def extend_storage_prefix(self, prefix_keys: Any, hash_values: List[str]) -> Any:
"""Advance the backend's prefix context past omitted pages."""
return prefix_keys + hash_values if prefix_keys is not None else None
def write_storage(
self,
host_indices: torch.Tensor,
token_ids: List[int],
hash_value: Optional[List[str]] = None,
prefix_keys: Optional[List[str]] = None,
) -> int:
"""
Write KV caches from host memory to storage backend.
"""
operation = StorageOperation(
host_indices, token_ids, hash_value=hash_value, prefix_keys=prefix_keys
)
self.backup_queue.put(operation)
return operation.id
# todo: deprecate
def _generic_page_set(self, hash_values, host_indices, extra_info=None) -> bool:
data = [
self.storage_host_pool.get_data_page(host_indices[i * self.page_size])
for i in range(len(hash_values))
]
return self.storage_backend.batch_set(hash_values, data)
def _page_set_zero_copy(self, hash_values, host_indices, extra_info=None) -> bool:
return all(
self.storage_backend.batch_set_v1(hash_values, host_indices, extra_info)
)
# Backup batch by batch
def _page_backup(self, operation):
# Backup batch by batch
prefix_keys = operation.prefix_keys
for i in range(0, len(operation.hash_value), STORAGE_BATCH_SIZE):
batch_hashes = operation.hash_value[i : i + STORAGE_BATCH_SIZE]
batch_host_indices = operation.host_indices[
i * self.page_size : (i + len(batch_hashes)) * self.page_size
]
# Set one batch token, and record if success.
# todo: allow partial success
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
success = self.page_set_func(batch_hashes, batch_host_indices, extra_info)
if not success:
logger.warning(
f"Write page to storage: {len(batch_hashes)} pages failed."
)
break
if prefix_keys is not None:
prefix_keys = prefix_keys + batch_hashes
operation.completed_tokens += self.page_size * len(batch_hashes)
def _page_backup_or_log(self, operation):
try:
self._page_backup(operation)
except Exception:
logger.exception(
"HiCache backup failed after %d tokens", operation.completed_tokens
)
def _should_skip_backup(self) -> bool:
"""Whether this rank can ACK without doing any storage IO."""
return self.backup_skip
def backup_thread_func(self):
"""
Manage backup operations from host memory to storage backend.
# [HC_PARALLEL_PATCH] pipelined: up to SGLANG_HICACHE_BACKUP_IO_THREADS
# writes in flight, acked in FIFO order.
"""
import time as _time
from collections import deque as _deque
from concurrent.futures import ThreadPoolExecutor as _TPE
K = max(1, _HC_BACKUP_WORKERS)
timing = _HC_TIMING
ex = _TPE(max_workers=K, thread_name_prefix="hc-backup-io")
inflight = _deque()
lat = []
pages = 0
def submit(operation) -> bool:
if self._should_skip_backup():
self.ack_backup_queue.put(operation)
return False
inflight.append(
(
operation,
_time.perf_counter(),
ex.submit(self._page_backup_or_log, operation),
)
)
return True
try:
while not self.storage_stop_event.is_set():
while len(inflight) < K:
try:
operation = self.backup_queue.get_nowait()
except Empty:
break
if operation is None:
continue
submit(operation)
if not inflight:
try:
operation = self.backup_queue.get(block=True, timeout=1)
except Empty:
continue
if operation is None:
continue
if not submit(operation):
continue
operation, t0, fut = inflight[0]
try:
fut.result()
except Exception:
logger.exception("HiCache backup failed")
finally:
inflight.popleft()
if timing:
lat.append((_time.perf_counter() - t0) * 1000.0)
pages += len(getattr(operation, "hash_value", []) or [])
self.ack_backup_queue.put(operation)
if timing and len(lat) >= 100:
lat.sort()
print("[HC_TIMING] backup queue=%d inflight=%d n=%d write_ms p50=%.1f p95=%.1f pages=%d"
% (self.backup_queue.qsize(), len(inflight), len(lat),
lat[len(lat) // 2], lat[int(len(lat) * 0.95)], pages), flush=True)
lat = []
pages = 0
finally:
# The outer thread is joined before backend/process-group teardown.
# Wait for every submitted write before reporting that thread stopped.
ex.shutdown(wait=True)
def prefetch_sync_thread_func(self):
"""Synchronize prefetch results across all PP and TP ranks."""
while not self.storage_stop_event.is_set():
try:
ack = self.prefetch_sync_queue.get(block=True, timeout=1)
if ack is None:
continue
self._reduce_prefetch_ack(ack)
self.ack_prefetch_queue.put(ack)
except Empty:
continue
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
"""Synchronize all ranks to agree on a PrefetchAck."""
if ack.completed_tokens is not None:
# Determine the minimal successful prefix of tokens.
completed_tokens_tensor = torch.tensor(
ack.completed_tokens, dtype=torch.int
)
self._all_reduce(
completed_tokens_tensor,
torch.distributed.ReduceOp.MIN,
self.prefetch_completion_sync_groups,
)
ack.completed_tokens = completed_tokens_tensor.item()