Fix unified HiCache physical transfers (#39479)

Co-authored-by: yhzhuang <yhzhuang@fb.com>
Co-authored-by: Lianmin Zheng <lianminzheng@gmail.com>
Co-authored-by: Yonghao Zhuang <yhzhuang@users.noreply.github.com>
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
This commit is contained in:
Yonghao Zhuang
2026-10-04 00:38:56 -07:00
committed by GitHub
co-authored by yhzhuang Lianmin Zheng Yonghao Zhuang Cheng Wan metamergebot
parent 6cc661f223
commit 937c0a6cfe
41 changed files with 1670 additions and 356 deletions
+15 -8
View File
@@ -530,18 +530,10 @@ def handle_unified_memory_pool(server_args: Any) -> None:
)
if cfg.disaggregation_decode_retraction_backup == "host_pool":
model_config = model_config_of(server_args)
assert not cfg.disaggregation_decode_enable_radix_cache, (
"--enable-unified-memory host-pool decode retraction does not "
"support decode radix-cache H2D/D2H transfers yet."
)
assert mambaish_config(model_config) is None, (
"--enable-unified-memory host-pool decode retraction does not "
"support hybrid-Mamba models."
)
assert not model_config.is_hybrid_swa, (
"--enable-unified-memory host-pool decode retraction does not "
"support hybrid-SWA H2D/D2H transfers yet."
)
assert cfg.speculative_algorithm in (None, "DSPARK"), (
"--enable-unified-memory only supports --speculative-algorithm "
"DSPARK (chain draft); other speculative algorithms are not yet "
@@ -578,6 +570,21 @@ def handle_unified_memory_pool(server_args: Any) -> None:
"the LMCache offload path indexes the device buffers with the ids it "
"is handed, and under the unified pool those are VIRTUAL."
)
assert not cfg.enable_unified_cache_external_linker, (
"--enable-unified-memory does not support "
"--enable-unified-cache-external-linker: direct L3 transfers do not "
"preserve unified page-envelope indices and compaction lifetimes. "
"Use --enable-hierarchical-cache for supported L2/L3 transfers."
)
if cfg.enable_hierarchical_cache:
assert cfg.pp_size == 1, (
"--enable-unified-memory with hierarchical cache does not support "
"pipeline parallelism (--pp-size > 1)."
)
assert not envs.SGLANG_DISABLE_LAZY_COMPACTION.get(), (
"--enable-unified-memory with hierarchical cache requires lazy "
"compaction so pending H2D physical reservations remain stable."
)
if cfg.dcp_size > 1:
_validate_unified_memory_dcp(server_args)
# Prefill cuda-graph capture IS wired for the unified pool: the captured
+12 -17
View File
@@ -787,12 +787,19 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
Match a request against the decode-side radix cache, lock the matched
node to prevent eviction, and return the matched prefix information.
"""
max_prefix_len = None
if self._uses_swa_tail_prealloc():
fill_len = self._pre_alloc_fill_len(req)
max_prefix_len = fill_len - self._swa_tail_len(fill_len)
# Match and lock only reusable FULL KV. The entire SWA tail must be
# freshly allocated, including when the prefix comes from L2/L3.
result = match_prefix_for_req(
self.tree_cache,
req,
req.origin_input_ids,
cow_mamba=self.tree_cache.supports_mamba(),
include_req=True,
max_prefix_len=max_prefix_len,
)
# Keep aggregated scheduling semantics while preserving the SWA lock
# boundary needed for the matching dec_lock_ref; the full receipt
@@ -800,7 +807,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
req.lock_receipt = self.tree_cache.inc_lock_ref(
result.last_device_node
).to_dec_params()
return self._build_decode_prefix_match(req, result)
return self._build_decode_prefix_match(
req, result, max_prefix_len=max_prefix_len
)
def _resolve_prefill_dp_rank(self, req: Req) -> Optional[int]:
prefill_info = self.kv_manager.prefill_info_table.get(_bootstrap_addr(req))
@@ -1422,20 +1431,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
fill_len = self._pre_alloc_fill_len(decode_req.req)
# Cap full-attention prefix reuse at the sliding-window start so
# the SWA window lands entirely in the fresh delta, keeping
# alloc_extend_swa_tail's tail->full mapping in range. Costs reuse
# of only the last ~window_size full-attention tokens.
if uses_swa_tail_prealloc and prefix_len > 0:
swa_prefix_cap = fill_len - self._swa_tail_len(fill_len)
if prefix_len > swa_prefix_cap:
prefix_len = swa_prefix_cap
prefix_indices = prefix_indices[:prefix_len]
# Cap the prefill-committed prefix too: tokens past the
# cap are not device-resident, so prefill must transfer
# them.
total_prefix_len = prefix_len
# Decode transfers the SWA tail fresh, so retain only the
# full-attention prefix lock needed for reuse.
if (
@@ -2317,9 +2312,9 @@ def alloc_for_decode_prealloc(
# the live window tail.
kv_loc = allocator.alloc_extend_swa_tail(
prefix_lens=torch.tensor(
[prefix_len], dtype=torch.int64, device=device
[total_prefix_len], dtype=torch.int64, device=device
),
prefix_lens_cpu=torch.tensor([prefix_len], dtype=torch.int64),
prefix_lens_cpu=torch.tensor([total_prefix_len], dtype=torch.int64),
seq_lens=torch.tensor([fill_len], dtype=torch.int64, device=device),
seq_lens_cpu=torch.tensor([fill_len], dtype=torch.int64),
last_loc=last_loc,
@@ -61,7 +61,9 @@ class HiCacheRestoreResult(Enum):
class DecodeHiCachePreallocMixin:
"""HiCache hooks for ``DecodePreallocQueue``: issue prefetch + reserve tokens."""
def _build_decode_prefix_match(self, req: Req, result: Any) -> DecodePrefixMatch:
def _build_decode_prefix_match(
self, req: Req, result: Any, *, max_prefix_len: Optional[int] = None
) -> DecodePrefixMatch:
"""Convert a ``match_prefix_for_req`` result into ``DecodePrefixMatch``.
Performs the optional L3 storage hit length query when decode-side
@@ -79,7 +81,7 @@ class DecodeHiCachePreallocMixin:
last_host_node
):
matched_len = l1_prefix_len + l2_host_hit_length
suffix_tokens = req.origin_input_ids[matched_len:]
suffix_tokens = req.origin_input_ids[matched_len:max_prefix_len]
last_hash = self.tree_cache.get_last_hash_value(last_host_node)
prefix_keys = (
self.tree_cache.get_prefix_hash_values(last_host_node)
@@ -221,6 +223,7 @@ class DecodeHiCacheTransferMixin:
dr.req.origin_input_ids,
cow_mamba=False,
include_req=True,
max_prefix_len=pm.decode_prefix_len,
)
new_indices, restored_node = self.tree_cache.init_load_back(
InitLoadBackParams(
+64 -16
View File
@@ -43,7 +43,7 @@ from sglang.srt.layers.dp_attention import (
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
from sglang.srt.mem_cache.utils import get_storage_hash_str
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_memory, get_parallel
from sglang.srt.utils import get_device_module
logger = logging.getLogger(__name__)
@@ -842,6 +842,10 @@ class HiCacheController:
self._l2_transfers(host_indices, device_indices, pool_transfers)
)
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "write"), completion.finish_event
)
self.ack_write_queue.append(
HiCacheAck(
start_event=completion.start_event,
@@ -986,6 +990,10 @@ class HiCacheController:
transfer_layer_id_max=self.transfer_layer_id_max,
)
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "load"), completion.finish_event
)
self.ack_load_queue.append(
HiCacheAck(
start_event=completion.start_event,
@@ -1108,13 +1116,22 @@ class HiCacheController:
# Get one batch token, and update the completed_tokens if succeed
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
hit_pages = self._page_transfer_kv_batch(
operation,
batch_hashes,
batch_host_indices,
extra_info,
kv_derived_transfers,
)
try:
hit_pages = self._page_transfer_kv_batch(
operation,
batch_hashes,
batch_host_indices,
extra_info,
kv_derived_transfers,
)
except Exception:
if not get_memory().enable_unified_memory:
raise
logger.exception(
"HiCache prefetch transfer failed for request %s",
operation.request_id,
)
hit_pages = 0
# Check termination
if hit_pages != len(batch_hashes):
all_success = False
@@ -1175,10 +1192,13 @@ class HiCacheController:
while not self.storage_stop_event.is_set():
try:
operation = self.prefetch_buffer.get(block=True, timeout=1)
if operation is None:
continue
except Empty:
continue
if operation is None:
continue
try:
self._page_transfer(operation)
finally:
self.prefetch_sync_queue.put(
PrefetchAck(
rid=operation.request_id,
@@ -1186,8 +1206,6 @@ class HiCacheController:
operation=operation,
)
)
except Empty:
continue
def prefetch_rate_limited(self) -> bool:
"""
@@ -1209,6 +1227,24 @@ class HiCacheController:
# todo: more sophisticated rate limiting based on storage backend performance
return False
def alloc_prefetch_host_buffers(
self, operation: StorageOperation, need_size: int
) -> Optional[torch.Tensor]:
"""Allocate the host bounce for a storage hit."""
return self.mem_pool_host.alloc(need_size)
def can_fit_prefetch_host_buffers(
self, operation: StorageOperation, need_size: int
) -> bool:
"""Whether a prefetch bounce can fit when its host pools are empty."""
return need_size <= self.mem_pool_host.size
def free_prefetch_host_buffers(
self, operation: StorageOperation, host_indices: torch.Tensor
) -> None:
"""Roll back a hit-sized host bounce before transfer ownership moves."""
self.mem_pool_host.free(host_indices)
def _storage_hit_query(self, operation) -> tuple[list[str], int]:
last_hash = operation.last_hash
tokens_to_fetch = operation.token_ids
@@ -1243,10 +1279,22 @@ class HiCacheController:
operation = self.prefetch_queue.get(block=True, timeout=1)
if operation is None:
continue
if operation.is_terminated():
try:
if operation.is_terminated():
hash_value, storage_hit_count = [], 0
else:
hash_value, storage_hit_count = self._storage_hit_query(
operation
)
except Exception:
if not get_memory().enable_unified_memory:
raise
logger.exception(
"HiCache storage query failed for request %s",
operation.request_id,
)
hash_value, storage_hit_count = [], 0
else:
hash_value, storage_hit_count = self._storage_hit_query(operation)
storage_hit_count_tensor = torch.tensor(
storage_hit_count, dtype=torch.int
)
+74 -14
View File
@@ -33,7 +33,7 @@ logger = logging.getLogger(__name__)
import os
import random
from collections import Counter
from contextlib import contextmanager
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
from enum import Enum, auto
from functools import lru_cache
@@ -68,6 +68,11 @@ from sglang.srt.mem_cache.base_prefix_cache import (
zero_match_result,
)
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
from sglang.srt.mem_cache.unified_cache.components import (
CacheTransferPhase,
ComponentType,
)
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
@@ -161,6 +166,7 @@ def match_prefix_for_req(
*,
cow_mamba: bool = False,
include_req: bool = False,
max_prefix_len: Optional[int] = None,
):
if token_ids is None:
token_ids = req.origin_input_ids + req.output_ids
@@ -171,6 +177,10 @@ def match_prefix_for_req(
# this request's SWA ring. No-op for other layouts.
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
if max_prefix_len is not None:
key_limit = (
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
)
match_result = tree_cache.match_prefix(
MatchPrefixParams(
@@ -1188,14 +1198,23 @@ class PrefillAdder:
return req if truncated else None
@contextmanager
def _lock_node(self, last_node: TreeNode):
# Replay the acquire's receipt (SWA boundary uuid, mamba flag) so the
# release takes back exactly what this temporary lock took.
dec_lock_params = self.tree_cache.inc_lock_ref(last_node).to_dec_params()
def _lock_node(self, last_node: TreeNode, *, lock_host: bool = False):
host_lock_params = (
self.tree_cache.inc_host_lock_ref(last_node).to_dec_params()
if lock_host
else None
)
try:
yield None
# Replay the acquire's receipt (SWA boundary uuid, mamba flag) so the
# release takes back exactly what this temporary lock took.
dec_lock_params = self.tree_cache.inc_lock_ref(last_node).to_dec_params()
try:
yield None
finally:
self.tree_cache.dec_lock_ref(last_node, dec_lock_params)
finally:
self.tree_cache.dec_lock_ref(last_node, dec_lock_params)
if host_lock_params is not None:
self.tree_cache.dec_host_lock_ref(last_node, host_lock_params)
def add_one_req_ignore_eos(self, req: Req):
cand_extend_input_len = len(req.full_untruncated_fill_ids) - len(
@@ -1398,14 +1417,55 @@ class PrefillAdder:
return AddReqResult.OTHER
if req.needs_host_load_back():
promised_host_hit = req.host_hit_length
loaded = self.tree_cache.init_load_back(
InitLoadBackParams(
best_match_node=req.best_match_node,
host_hit_length=req.host_hit_length,
req=req,
load_max_new = min(max_new, admission.max_new_tokens)
# Reclaim can write back device victims and evict host leaves.
# Pin the selected host/aux match until load-back owns its locks.
with (
self._lock_node(req.best_match_node, lock_host=True)
if isinstance(self.tree_cache, UnifiedRadixCache)
else nullcontext()
):
full_load_tokens = req.host_hit_length
if (
isinstance(self.tree_cache, UnifiedRadixCache)
and self.tree_cache.buffer_pipeline is None
and not (
self.tree_cache.linker is not None
and self.tree_cache.linker.has_hit(req.rid)
)
):
# Host hits can include resident FULL behind host-only aux.
# Reuse the FULL transfer spec to count only new slots.
full_transfer = (
self.tree_cache.tree_core.build_hicache_transfers(
ComponentType.FULL,
req.best_match_node,
CacheTransferPhase.LOAD_BACK,
)[0]
)
full_load_tokens = len(full_transfer.host_indices)
if not self.memory_budget.prepare_load_back(
full_tokens=(
full_load_tokens
+ admission.extend_len
+ load_max_new
+ self.page_size
+ mamba_gap_reserve
),
extend_input_len=admission.extend_len,
max_new_tokens=load_max_new,
swa_host_hit_length=req.swa_host_hit_length,
chunk_limit=self.rem_chunk_tokens,
):
return AddReqResult.NO_TOKEN
promised_host_hit = req.host_hit_length
loaded = self.tree_cache.init_load_back(
InitLoadBackParams(
best_match_node=req.best_match_node,
host_hit_length=req.host_hit_length,
req=req,
)
)
)
if loaded is None:
return AddReqResult.OTHER
new_indices, req.last_node = loaded
@@ -16,7 +16,7 @@ limitations under the License.
from __future__ import annotations
import abc
from typing import TYPE_CHECKING, Protocol
from typing import TYPE_CHECKING, Hashable, Protocol
import torch
@@ -228,6 +228,10 @@ class BaseTokenToKVPoolAllocator(abc.ABC):
):
raise NotImplementedError()
def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
"""Record an asynchronous HiCache transfer completion event if needed."""
return
def alloc_extend(self, *args, **kwargs):
raise NotImplementedError("alloc_extend is only for paged allocator")
@@ -19,7 +19,7 @@ from __future__ import annotations
import logging
import math
from abc import abstractmethod
from typing import Callable, List, Optional, Sequence, Tuple
from typing import Callable, Hashable, List, Optional, Sequence, Tuple
import torch
from torch.profiler import record_function
@@ -502,11 +502,11 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
The static composite allocates the two sides independently and records
a full->swa index mapping. That is not representable here: the two
sides SHARE one virtual id space (a virtual page names a full-physical
page and, if bound, a swa-physical one), which is why
`set_full_to_swa_mapping` is a no-op on this allocator and
`translate_loc_from_full_to_swa` derives the swa id from the virtual id
instead of a table. Running the static body would call `alloc_extend`
on the swa sub-allocator, which asserts it is not the id owner.
page and, if bound, a swa-physical one). Load-back installs that binding
through `set_full_to_swa_mapping`, and `translate_loc_from_full_to_swa`
resolves the SWA kernel-facing id from the shared virtual id. Running the
static body would call `alloc_extend` on the swa sub-allocator, which
asserts it is not the id owner.
The tail is expressed by binding swa for the TAIL's virtual pages only.
A new page left unbound has no swa-physical page, which reads as the
@@ -678,8 +678,12 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
def set_full_to_swa_mapping(
self, full_indices: torch.Tensor, swa_indices: torch.Tensor
) -> None:
"""Binding load-back rows already updates the shared SWA v2p mapping."""
return
if full_indices.numel() == 0:
return
assert full_indices.numel() == swa_indices.numel()
full_pages = full_indices.to(torch.int64) // self.page_size
swa_pages = swa_indices.to(torch.int64) // self.page_size
self.swa_attn_allocator.bind(full_pages, swa_pages)
def clear_full_to_swa_mapping(self, full_indices: torch.Tensor) -> None:
# Paired with set_full_to_swa_mapping: shared mode has no mapping tensor.
@@ -797,6 +801,10 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
@abstractmethod
def _ask_float_for_room(self, need_tokens: int) -> None: ...
def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
self.full_attn_allocator.set_hicache_transfer_done_event(transfer_key, event)
self.swa_attn_allocator.set_hicache_transfer_done_event(transfer_key, event)
class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
"""Two-ended FULL/SWA allocator with asymmetric shared-byte reservations."""
@@ -1204,7 +1212,10 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
return
full_reclaim, swa_reclaim = reclaim_plan
if full_reclaim or swa_reclaim:
tree_cache.evict_for_alloc(
# The shared-byte plan returns cumulative eviction quotas.
# Per-component capacity targets can count the same shared bytes
# independently and stop before the joint allocation fits.
tree_cache.evict(
EvictParams(num_tokens=full_reclaim, swa_num_tokens=swa_reclaim)
)
# A zero-reclaim plan can still depend on compaction before allocation.
@@ -17,7 +17,7 @@ mamba-state end pools of one `UnifiedKVPool`."""
from __future__ import annotations
import logging
from typing import Callable, List, Optional, Sequence
from typing import Callable, Hashable, List, Optional, Sequence
import torch
from torch.profiler import record_function
@@ -443,3 +443,7 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
):
return 0
return fa.flush_opportunistic() + ma.flush_opportunistic()
def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
self.full_attn_allocator.set_hicache_transfer_done_event(transfer_key, event)
self.mamba_allocator.set_hicache_transfer_done_event(transfer_key, event)
@@ -29,6 +29,7 @@ from typing import (
Callable,
Dict,
Generic,
Hashable,
List,
Optional,
Sequence,
@@ -419,6 +420,12 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
# write-race check. Single slot: at most ONE forward in flight per call
# site; `_flush` materializes the write-set lazily, avoiding a sync here.
self._inflight_forward: Optional[Tuple[torch.cuda.Event, torch.Tensor]] = None
self._hicache_transfer_done_events: Dict[Hashable, torch.cuda.Event] = {}
# HiCache reserves SWA physical pages during load-back, then binds them
# into the tree before the H2D copy is submitted. Compaction must not
# move those pages in that interval because the queued PoolTransfer still
# carries the original physical ids.
self._pending_hicache_load_pages = 0
# Per-call move cap on NON-urgent `_flush`: bounds work per `on_idle()` so
# a large backlog doesn't block ZMQ IPC. Urgent retries are uncapped.
@@ -521,6 +528,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self.live_page_count = 0
self._inflight_forward = None
self._latest_forward_done_event = None
self._hicache_transfer_done_events.clear()
self._pending_hicache_load_pages = 0
def clear_inverse_history(self) -> None:
self._inverse_history.clear()
@@ -726,6 +735,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
def moves_blocked(self) -> bool:
"""Whether any installed gate currently forbids relocating pages."""
if self._pending_hicache_load_pages > 0:
# Physical H2D reservations must stay put even before queueing.
return True
for gate in (self.disagg_move_gate, self.host_transfer_move_gate):
if gate is not None and not gate():
return True
@@ -926,6 +938,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
with record_function("MultiEndedAlloc._alloc_bind_fast_or_slow"):
if N == 0:
return torch.empty(0, dtype=torch.int64, device=self.device)
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
self._wait_hicache_transfers()
# FAST PATH: eager, or lazy with no current holes.
if not self.lazy_compaction or self._free_phys_pages.numel() == 0:
@@ -1082,19 +1096,15 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
if not _relieve_for_alloc(self, need_size):
return None
num_pages = need_size // self.page_size
if num_pages > len(self.free_virtual_ids):
return None
v_pages = self.free_virtual_ids[:num_pages]
self.free_virtual_ids = self.free_virtual_ids[num_pages:]
phys_pages = self._alloc_bind_fast_or_slow(v_pages, num_pages)
if phys_pages is None:
self.free_virtual_ids = torch.cat([v_pages, self.free_virtual_ids])
return None
if self.page_size == 1:
return v_pages # v_pages already IS the token id list
# Expand page ids to token ids: (P, 1) * S + (S,) -> (P, S) -> (P*S,).
return (
v_pages[:, None] * self.page_size
+ torch.arange(self.page_size, device=self.device)
).reshape(-1)
return self._expand_pages_to_tokens(v_pages)
def alloc_with_virtual(self, virtual_pages: torch.Tensor) -> None:
"""Take physical PAGES for caller-supplied virtual PAGE ids (not token
@@ -1264,6 +1274,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self._free_lazy(free_index, pages=_pages)
return
# --- EAGER path ---
self._wait_hicache_transfers()
# Near-no-op in normal mode (sampling's CPU sync already drained
# forward_stream); in overlap mode it does the serializing.
if self.forward_stream is not None:
@@ -1713,6 +1724,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
if self.moves_blocked():
# Holes stay in the free list; the next flush picks them up.
return 0
self._wait_hicache_transfers()
self._stats_n_flush_calls += 1
with record_function("MultiEndedAlloc._flush"):
self._drain_pending_reuse(urgent=urgent)
@@ -1965,6 +1977,92 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self.size = self.max_slots
self.clear()
_pending_hicache_load_pages: _CapacityField[int] = _CapacityField()
def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
self._hicache_transfer_done_events[transfer_key] = event
if transfer_key == "load" or (
isinstance(transfer_key, tuple) and transfer_key[-1] == "load"
):
# The H2D copy is now ordered before this event. Future compaction
# may proceed after _wait_hicache_transfers() waits on it.
self._pending_hicache_load_pages = 0
def _wait_hicache_transfers(self) -> None:
if not self._hicache_transfer_done_events:
return
current_stream = torch.cuda.current_stream()
events = tuple(self._hicache_transfer_done_events.values())
self._hicache_transfer_done_events.clear()
for event in events:
current_stream.wait_event(event)
def _expand_pages_to_tokens(self, pages: torch.Tensor) -> torch.Tensor:
if self.page_size == 1:
return pages
return (
pages[:, None] * self.page_size
+ torch.arange(self.page_size, device=self.device)
).reshape(-1)
def alloc_physical(self, need_size: int) -> Optional[torch.Tensor]:
"""Reserve physical token slots without assigning virtual page ids."""
if need_size <= 0:
return torch.empty(0, dtype=torch.int64, device=self.device)
assert need_size % self.page_size == 0, (
f"MultiEndedAllocator({self.sub_pool_name!r}).alloc_physical: need_size="
f"{need_size} must be a multiple of page_size={self.page_size}"
)
if need_size > self.available_size():
if not _relieve_for_alloc(self, need_size):
return None
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
self._wait_hicache_transfers()
physical_pages = self.take_physical_pages(need_size // self.page_size)
if physical_pages is None:
return None
if self.lazy_compaction:
self._pending_hicache_load_pages += int(physical_pages.shape[0])
return self._expand_pages_to_tokens(physical_pages)
def cancel_physical_reservation(self, free_index: torch.Tensor) -> None:
"""Roll back a HiCache physical allocation before its H2D is submitted."""
if free_index is None or free_index.numel() == 0:
return
if self.lazy_compaction:
num_pages = free_index.numel() // self.page_size
assert num_pages <= self._pending_hicache_load_pages, (
f"MultiEndedAllocator({self.sub_pool_name!r}) released {num_pages} "
f"HiCache pages with only {self._pending_hicache_load_pages} pending"
)
self._pending_hicache_load_pages -= num_pages
self.free_physical(free_index)
def free_physical(self, free_index: torch.Tensor) -> None:
"""Release physical token slots reserved by :meth:`alloc_physical`."""
if free_index is None or free_index.numel() == 0:
return
assert free_index.numel() % self.page_size == 0, (
f"MultiEndedAllocator({self.sub_pool_name!r}).free_physical: "
f"{free_index.numel()} tokens must contain whole pages of "
f"size {self.page_size}"
)
physical_pages = (
free_index.detach().to(torch.int64)[:: self.page_size] // self.page_size
)
self._wait_hicache_transfers()
if self.forward_stream is not None and not self.lazy_compaction:
torch.cuda.current_stream().wait_stream(self.forward_stream)
virtual_pages = self.physical_to_virtual[physical_pages]
bound_mask = virtual_pages >= 0
self.virtual_to_physical.index_fill_(0, virtual_pages[bound_mask], -1)
self.physical_to_virtual.index_fill_(0, physical_pages, -1)
if self.lazy_compaction:
self._free_phys_pages = torch.cat([self._free_phys_pages, physical_pages])
self.live_page_count -= int(physical_pages.shape[0])
return
self._compact_pending(physical_pages)
def _chain_byte_accounting_violations(
chain: List[MultiEndedAllocator],
@@ -2258,6 +2356,23 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
self._holes_dirty = True
self._park_if_empty()
def free_physical(self, free_index: torch.Tensor) -> None:
"""Release physical reservations using the float's hole/span bookkeeping."""
if free_index is None or free_index.numel() == 0:
return
assert free_index.numel() % self.page_size == 0
physical_pages = (
free_index.detach().to(torch.int64)[:: self.page_size] // self.page_size
)
self._wait_hicache_transfers()
virtual_pages = self.physical_to_virtual[physical_pages]
bound_pages = virtual_pages[virtual_pages >= 0]
self.virtual_to_physical.index_fill_(0, bound_pages, -1)
self.physical_to_virtual.index_fill_(0, physical_pages, -1)
self._free_phys_pages = torch.cat([self._free_phys_pages, physical_pages])
self._holes_dirty = True
self._park_if_empty()
def _park_if_empty(self) -> bool:
"""Reset the span and go frontier-transparent once no live page
remains. Sync-free: `_live_pages()` is span minus hole COUNT, both
@@ -28,6 +28,7 @@ from __future__ import annotations
import logging
from array import array
from collections import deque
from dataclasses import replace
from typing import TYPE_CHECKING, Optional
import msgspec
@@ -49,6 +50,7 @@ from sglang.srt.mem_cache.hicache_storage import (
PoolTransfer,
SidecarPoolSpec,
)
from sglang.srt.mem_cache.pool_host.base import HostKVCache
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.storage_prefetch import StagedPrefetchPlan
from sglang.srt.mem_cache.unified_cache.cache_action import RebuildFullToSWAMapping
@@ -71,6 +73,13 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
def _minimum_full_transfer_tokens(
host_pool: HostKVCache, prefetch_threshold: int
) -> int:
page_size = host_pool.page_size
return max(page_size, -(-prefetch_threshold // page_size) * page_size)
class _UnifiedBackupIntent(msgspec.Struct):
"""Buffer-mode backup intent, unpinned while queued.
@@ -95,6 +104,7 @@ class _UnifiedBufferBackupEntry(msgspec.Struct):
host_indices: torch.Tensor
aux_xfers: list[PoolTransfer]
lock_params: DecLockRefParams
occupied_units: int
class _StagedPrefetch(msgspec.Struct):
@@ -159,6 +169,7 @@ def validate_buffer_only_stack(
sidecar_pool_specs: list[SidecarPoolSpec],
host_pool_group: HostPoolGroup,
swa_component: Optional[SWAComponent],
storage_prefetch_threshold: int = 256,
) -> None:
"""Post-assembly buffer-mode fences.
@@ -201,12 +212,40 @@ def validate_buffer_only_stack(
# window), so every window-carrying intent would be dropped as
# oversize and SWA storage coverage would silently be zero.
window_tokens = swa.full_window_pages * swa._swa_kv_pool_host.page_size
if swa._swa_kv_pool_host.size < 2 * window_tokens:
shared_domain = (
swa._swa_kv_pool_host.shared_allocation_domain
if isinstance(swa._swa_kv_pool_host, HostKVCache)
else None
)
if shared_domain is not None:
full_host_pool = (
swa.cache.cache_controller.mem_pool_host.anchor_entry.host_pool
)
min_full_tokens = _minimum_full_transfer_tokens(
full_host_pool, storage_prefetch_threshold
)
one_transfer_bytes = (
window_tokens * swa._swa_kv_pool_host.size_per_token
+ min_full_tokens * full_host_pool.size_per_token
)
one_transfer = [
(full_host_pool.pool_label, min_full_tokens),
(swa._swa_kv_pool_host.pool_label, window_tokens),
]
enough_capacity = shared_domain.can_fit_many_then(
one_transfer, one_transfer, empty=True
)
capacity = f"{shared_domain.capacity_bytes} shared bytes"
requirement = f"{2 * one_transfer_bytes} shared bytes"
else:
enough_capacity = swa._swa_kv_pool_host.size >= 2 * window_tokens
capacity = f"{swa._swa_kv_pool_host.size} SWA tokens"
requirement = f"{2 * window_tokens} SWA tokens"
if not enough_capacity:
raise ValueError(
"--hicache-host-memory-mode buffer_only requires an SWA "
f"host pool of at least two trailing windows "
f"({2 * window_tokens} tokens; got "
f"{swa._swa_kv_pool_host.size}): one staging a write "
"--hicache-host-memory-mode buffer_only requires a host arena "
"large enough for two minimum Full/SWA transfers "
f"({requirement}; got {capacity}): one staging a write "
"while one stays reserved for prefetch window allocs."
)
@@ -300,8 +339,89 @@ class BufferModePipeline:
self.anchor_locked_tokens_ = 0
self._anchor_lock_cap_skips = 0
def _shared_host_domain(self):
cc = self._cache.cache_controller
anchor = cc.mem_pool_host.anchor_entry.host_pool
if not isinstance(anchor, HostKVCache):
return None
return anchor.shared_allocation_domain
def _transfer_tokens(self, transfer: PoolTransfer) -> int:
if transfer.host_indices is not None:
return len(transfer.host_indices)
if transfer.keys is None:
return 0
entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name)
return 0 if entry is None else len(transfer.keys) * entry.host_pool.page_size
def host_allocation_units(
self,
host_indices: Optional[torch.Tensor],
aux_xfers: Optional[list[PoolTransfer]],
) -> int:
"""Host usage expressed in anchor-token units for scheduler accounting."""
if host_indices is None:
return 0
return self._host_request_units(len(host_indices), aux_xfers)
def _shared_host_requests(
self, kv_tokens: int, aux_xfers: Optional[list[PoolTransfer]]
) -> Optional[list[tuple[str, int]]]:
if self._shared_host_domain() is None:
return None
return [
(pool.pool_label, tokens)
for pool, tokens in self._host_staging_sizes(kv_tokens, aux_xfers)
]
def _host_staging_sizes(self, kv_tokens, aux_xfers):
cc = self._cache.cache_controller
yield cc.mem_pool_host.anchor_entry.host_pool, kv_tokens
for transfer in aux_xfers or ():
if transfer.indices_from_pool is not None:
continue
entry = cc.mem_pool_host.entry_map.get(transfer.name)
if entry is not None:
yield entry.host_pool, self._transfer_tokens(transfer)
def _host_request_units(
self, kv_tokens: int, aux_xfers: Optional[list[PoolTransfer]]
) -> int:
"""Host staging in anchor-token accounting units."""
cc = self._cache.cache_controller
anchor = cc.mem_pool_host.anchor_entry.host_pool
if self._shared_host_domain() is None:
return kv_tokens
num_bytes = sum(
tokens * pool.size_per_token
for pool, tokens in self._host_staging_sizes(kv_tokens, aux_xfers)
)
return (num_bytes + anchor.size_per_token - 1) // anchor.size_per_token
def _shared_backup_fits(self, requests, *, empty: bool = False) -> bool:
return self._shared_host_domain().can_fit_many_then(
requests, self._shared_load_reserve_requests(), empty=empty
)
def _shared_load_reserve_requests(self) -> list[tuple[str, int]]:
cc = self._cache.cache_controller
anchor = cc.mem_pool_host.anchor_entry.host_pool
swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA)
min_full_tokens = _minimum_full_transfer_tokens(
anchor, self._cache.prefetch_threshold
)
requests = [(anchor.pool_label, min_full_tokens)]
if swa_entry is not None and self._swa_window_pages:
requests.append(
(
swa_entry.host_pool.pool_label,
self._swa_window_pages * swa_entry.host_pool.page_size,
)
)
return requests
def is_idle(self) -> bool:
"""No pending or in-flight buffer-mode transfer work."""
"""No queued, staged, or in-flight buffer-mode work or anchor pins."""
return not (
self.pending_hit_allocs
or self.staged_prefetches
@@ -310,6 +430,7 @@ class BufferModePipeline:
or self.inflight_backup_node_ids
or self.ongoing_write_through
or self.ongoing_backup
or self.anchor_locks
)
def swa_transient_size(self) -> int:
@@ -446,11 +567,18 @@ class BufferModePipeline:
capacity (total for KV, total minus the loads-priority margin for aux
pools — matching ``_aux_budget_blocked``'s admission ceiling): such an
intent could never stage and would wedge the FIFO head."""
cc = self._cache.cache_controller
if intent_tokens > cc.mem_pool_host.size:
return True
if aux_xfers is None:
aux_xfers = self._build_aux_staging_transfers(node_id, hash_values)
cc = self._cache.cache_controller
shared_requests = self._shared_host_requests(intent_tokens, aux_xfers)
if shared_requests is not None:
pool_tokens = cc.mem_pool_host.size
max_write_units = pool_tokens - pool_tokens // 10
if self._host_request_units(intent_tokens, aux_xfers) > max_write_units:
return True
return not self._shared_backup_fits(shared_requests, empty=True)
if intent_tokens > cc.mem_pool_host.size:
return True
for t in aux_xfers or ():
entry = cc.mem_pool_host.entry_map.get(t.name)
if entry is not None and (
@@ -540,7 +668,7 @@ class BufferModePipeline:
self._log_backup_dropped(intent_tokens)
continue
if self.write_staged_tokens_ >= live_cap:
# Yield to live fetch demand; retry next round.
# Wait for current copies to free staging before rebuilding the head.
break
device_value, comp_xfers = self._cache.tree_core.build_backup_spec(
snapshot.node_id
@@ -560,6 +688,17 @@ class BufferModePipeline:
self.write_backlog_tokens_ -= intent_tokens
self._log_backup_dropped(intent_tokens)
continue
shared_domain = self._shared_host_domain()
staging_at_limit = (
self.write_staged_tokens_
+ self._host_request_units(intent_tokens, sizing_xfers)
> live_cap
if shared_domain is not None
else self.write_staged_tokens_ >= live_cap
)
if staging_at_limit:
# Yield to live fetch demand; retry next round.
break
if self._aux_budget_blocked(intent, sizing_xfers):
# An aux pool lacks staging headroom: yield at the gate
# instead of failing the alloc inside cc.write; acks free
@@ -609,13 +748,15 @@ class BufferModePipeline:
# NOTE: no commit_backup — the node must never appear
# host-resident in buffer mode; staging slots live in the entry.
lock_params = cache.inc_lock_ref(snapshot.node_id).to_dec_params()
occupied_units = self.host_allocation_units(host_indices, aux_xfers)
self.ongoing_write_through[snapshot.node_id] = _UnifiedBufferBackupEntry(
intent=intent,
host_indices=host_indices,
aux_xfers=aux_xfers,
lock_params=lock_params,
occupied_units=occupied_units,
)
self.write_staged_tokens_ += len(host_indices)
self.write_staged_tokens_ += occupied_units
self.write_backlog_tokens_ -= len(snapshot.hash_values) * cache.page_size
return True
@@ -636,9 +777,14 @@ class BufferModePipeline:
aux = self._build_aux_staging_transfers(
snapshot.node_id, snapshot.hash_values
)
cc = self._cache.cache_controller
shared_requests = self._shared_host_requests(
len(snapshot.hash_values) * self._cache.page_size, aux
)
if shared_requests is not None:
return not self._shared_backup_fits(shared_requests)
if not aux:
return False
cc = self._cache.cache_controller
for t in aux:
entry = cc.mem_pool_host.entry_map.get(t.name)
if entry is None:
@@ -740,7 +886,7 @@ class BufferModePipeline:
snapshot = intent.snapshot
self._cache.storage_existence_cache.add(PoolName.KV, snapshot.hash_values)
self._free_staging_now(entry.host_indices, entry.aux_xfers)
self.write_staged_tokens_ -= len(entry.host_indices)
self.write_staged_tokens_ -= entry.occupied_units
self.inflight_backup_node_ids.discard(snapshot.node_id)
_untrack_content_refs(self.inflight_backup_hashes, snapshot.hash_values)
@@ -968,12 +1114,6 @@ class BufferModePipeline:
f.request.rid, f.matched_len + f.num_tokens
)
@staticmethod
def _occupied_span(host_indices) -> int:
"""Occupancy units a buffer-mode prefetch holds: granted at
hit-alloc, sized to the allocation (0 while still querying)."""
return len(host_indices) if host_indices is not None else 0
def stage_completed_prefetch(
self,
request: CacheRequestHandle,
@@ -1001,6 +1141,9 @@ class BufferModePipeline:
for transfer in operation.pool_transfers or ()
if transfer.indices_from_pool is not None
)
occupied_tokens = operation.buffer_host_occupied_units
if occupied_tokens is None:
occupied_tokens = self.host_allocation_units(host_indices, aux_xfers)
has_aux = any(
t.host_indices is not None and t.host_indices.numel() > 0 for t in aux_xfers
@@ -1012,7 +1155,7 @@ class BufferModePipeline:
cc.append_host_mem_release(
host_indices[:num_tokens], extra_pools=aux_xfers or None
)
cc.prefetch_tokens_occupied -= self._occupied_span(host_indices)
cc.prefetch_tokens_occupied -= occupied_tokens
cache.prefetch_loaded_tokens_by_reqid[request] = 0
cache.prefetch_loaded_storage_start_by_reqid.pop(request, None)
return True
@@ -1024,8 +1167,6 @@ class BufferModePipeline:
# itself is the evidence, so feeding is sound even if this staged
# prefetch is later dropped unconsumed.
cache.storage_existence_cache.add(PoolName.KV, list(staged_hashes))
occupied_tokens = self._occupied_span(host_indices)
self.staged_prefetches[request] = _StagedPrefetch(
request=request,
key_tokens=array(
@@ -1150,6 +1291,52 @@ class BufferModePipeline:
),
0,
)
swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA)
binds_swa_to_full = (
swa_entry is not None
and swa_entry.device_indices_from_anchor_fn is not None
)
repair_ranges = []
if binds_swa_to_full and staged_swa:
window_start = span_end - staged_swa
repair_end = min(splice_base, span_end)
if window_start < repair_end:
repair_ranges = cache.tree_core.swa_tombstone_ranges(
key, window_start, repair_end
)
# Only missing SWA rows belong to this load. Existing bindings may
# be in use by another request and must survive allocation rollback.
anchor_parts = []
host_parts = []
for repair_start, repair_end_ in repair_ranges:
anchor_parts.append(req.prefix_indices[repair_start:repair_end_])
host_parts.append(
slice(repair_start - window_start, repair_end_ - window_start)
)
tail_start = max(splice_base, window_start)
if tail_start < span_end:
anchor_parts.append(
slice(tail_start - splice_base, span_end - splice_base)
)
host_parts.append(
slice(tail_start - window_start, span_end - window_start)
)
for i, transfer in enumerate(load_xfers):
if transfer.name != PoolName.SWA:
continue
# Keep the original complete host bounce for ack/drop release.
load_xfers[i] = replace(
transfer,
host_indices=(
torch.cat([transfer.host_indices[part] for part in host_parts])
if host_parts
else transfer.host_indices[:0]
),
anchor_index_parts=anchor_parts,
)
if not anchor_parts:
load_xfers = [t for t in load_xfers if t.name != PoolName.SWA]
device_indices = cc.load(
host_indices=f.host_indices[trim_tokens:],
node_id=load_back_id,
@@ -1177,7 +1364,18 @@ class BufferModePipeline:
None,
)
aux_device_releases: list[tuple[PoolName, torch.Tensor]] = []
if swa_dev is not None:
if swa_dev is not None and binds_swa_to_full:
# Binding already updated the allocator's virtual-to-physical table.
# The tree owns virtual FULL rows, not the H2D kernel-facing IDs.
for repair_start, repair_end_ in repair_ranges:
for action in cache.tree_core.attach_swa_window(
key,
repair_start,
repair_end_,
req.prefix_indices[repair_start:repair_end_],
):
cache._apply_cache_action(action)
elif swa_dev is not None:
# Register the window's FULL->SWA translation now (attention reads
# through it). Keep SWA slots another request may still hold; their
# redundant H2D destinations are reclaimed at the transfer ack.
@@ -1185,7 +1383,7 @@ class BufferModePipeline:
-len(swa_dev) :
]
allocator = cache.token_to_kv_pool_allocator
old_swa = allocator.full_to_swa_index_mapping[full_window.to(torch.int64)]
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
missing = old_swa <= 0
window_start = span_end - len(swa_dev)
repair_end = min(splice_base, span_end)
@@ -1287,8 +1485,19 @@ class BufferModePipeline:
self._free_staging_now(f.host_indices, f.aux_xfers)
for pool_name, device_indices in f.aux_device_releases:
entry = cc.mem_pool_host.entry_map[pool_name]
free_fn = entry.device_free_fn or entry.device_pool.free
free_fn(device_indices)
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWAAllocatorBase,
)
allocator = cache.token_to_kv_pool_allocator
if pool_name == PoolName.SWA and isinstance(
allocator, UnifiedSWAAllocatorBase
):
# H2D has completed; these redundant slots are no longer pending.
allocator.swa_attn_allocator.free_physical(device_indices)
else:
free_fn = entry.device_free_fn or entry.device_pool.free
free_fn(device_indices)
cc.prefetch_tokens_occupied -= f.occupied_tokens
logger.info(
@@ -39,7 +39,9 @@ from sglang.srt.mem_cache.l2_transfer import L2Transfer
from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry
from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.runtime_context import get_memory
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
@@ -163,6 +165,7 @@ class PrefetchOperation(StorageOperation):
pool_transfers=pool_transfers,
)
self.pool_transfers_done = not bool(pool_transfers)
self.buffer_host_occupied_units: Optional[int] = None
# The Python transfer worker leaves the unfinished tail to the ACK drain;
# a controller that releases it itself must set this False.
self.ack_releases_incomplete_host_indices = True
@@ -277,6 +280,11 @@ class HybridCacheController(BaseHiCacheController):
self._stop_pp_prefetch_thread()
super()._stop_storage_threads()
@staticmethod
def supports_page_envelope_host(storage_backend: str | None) -> bool:
"""Only opt in backends whose FULL object is the complete shared page."""
return storage_backend is None or storage_backend == "mori"
def attach_storage_backend(
self,
storage_backend: str,
@@ -285,6 +293,14 @@ class HybridCacheController(BaseHiCacheController):
storage_backend_extra_config: Optional[dict] = None,
host_pools: Optional[list[PoolEntry]] = None,
):
if isinstance(
self.storage_host_pool, UnifiedPageEnvelopeHostPool
) and not self.supports_page_envelope_host(storage_backend):
# A runtime backend switch cannot replace the existing host arena.
raise ValueError(
f"Storage backend {storage_backend!r} requires separate host pools. "
"Restart with this backend selected to create compatible host pools."
)
enable_pp_ticket = (
self.host_memory_mode == "buffer_only"
and storage_backend == "mooncake"
@@ -572,6 +588,9 @@ class HybridCacheController(BaseHiCacheController):
for entry, (_, need_size) in zip(entries, requests, strict=True)
]
# Scheduler drains commit the same logical host allocations and frees
# across attention ranks. Layout compaction waits for local I/O before
# allocating; transfer readiness does not change the capacity decision.
allocated = domain.alloc_many(domain_requests)
if allocated is not None:
return allocated
@@ -615,7 +634,8 @@ class HybridCacheController(BaseHiCacheController):
// pool.page_size
* pool.page_size
)
if entry.host_evict_fn(tokens) <= 0:
evicted = entry.host_evict_fn(tokens)
if evicted <= 0:
continue
made_progress = True
allocated = domain.alloc_many(domain_requests)
@@ -665,6 +685,134 @@ class HybridCacheController(BaseHiCacheController):
return None
return host_indices, resolved
def _shared_prefetch_requests(
self, operation: PrefetchOperation, need_size: int
) -> tuple[list[tuple[PoolName, int]], list[PoolTransfer]]:
anchor = self.mem_pool_host.anchor_entry
pool_transfers = operation.pool_transfers or []
requests = [(anchor.name, need_size)]
sidecar_hit_pages = (
operation.sidecar_hit_pages
if operation.sidecar_hash_values is not None
else need_size // self.page_size
)
independent_transfers = []
for transfer in pool_transfers:
if transfer.indices_from_pool is not None:
continue
if transfer.host_indices is not None:
raise AssertionError(
"Shared prefetch pools must be allocated atomically."
)
entry = self.mem_pool_host.entry_map.get(transfer.name)
if entry is None:
continue
num_pages = len(transfer.keys or [])
if transfer.hit_policy == PoolHitPolicy.TRAILING_PAGES:
num_pages = min(num_pages or 1, sidecar_hit_pages)
requests.append((transfer.name, num_pages * entry.host_pool.page_size))
independent_transfers.append(transfer)
return requests, independent_transfers
def can_fit_prefetch_host_buffers(
self, operation: StorageOperation, need_size: int, *, empty: bool = True
) -> bool:
anchor = self.mem_pool_host.anchor_entry
if not uses_shared_host_layout(anchor.host_pool):
return super().can_fit_prefetch_host_buffers(operation, need_size)
domain = anchor.host_pool.shared_allocation_domain
requests, _ = self._shared_prefetch_requests(operation, need_size)
domain_requests = [
(self.mem_pool_host.entry_map[name].host_pool.pool_label, size)
for name, size in requests
]
return domain.can_fit_many_then(domain_requests, (), empty=empty)
def prefetch_rate_limited(self) -> bool:
if self.host_memory_mode == "buffer_only" and uses_shared_host_layout(
self.mem_pool_host.anchor_entry.host_pool
):
# Shared-arena occupancy includes the Full and SWA byte footprint.
return self.prefetch_tokens_occupied >= self.prefetch_capacity_limit
return super().prefetch_rate_limited()
def alloc_prefetch_host_buffers(
self, operation: StorageOperation, need_size: int
) -> Optional[torch.Tensor]:
"""Atomically allocate every prefetch slice from a shared arena."""
anchor = self.mem_pool_host.anchor_entry
if not uses_shared_host_layout(anchor.host_pool):
return super().alloc_prefetch_host_buffers(operation, need_size)
requests, independent_transfers = self._shared_prefetch_requests(
operation, need_size
)
allocated = self._alloc_shared_host_requests_with_reclaim(requests)
if allocated is None:
return None
self._sync_trailing_keys(
operation.pool_transfers or [],
operation.sidecar_hash_values or operation.hash_value,
(
operation.sidecar_hit_pages
if operation.sidecar_hash_values is not None
else need_size // self.page_size
),
)
for transfer, indices in zip(independent_transfers, allocated[1:], strict=True):
transfer.host_indices = indices
return allocated[0]
def allocate_storage_hit(
self,
operation: StorageOperation,
hit_tokens: int,
*,
allow_partial: bool,
min_tokens: int,
evict_host: Callable[[int], int],
) -> tuple[Optional[torch.Tensor], int]:
host_indices = self.alloc_prefetch_host_buffers(operation, hit_tokens)
shared = uses_shared_host_layout(self.mem_pool_host.anchor_entry.host_pool)
if host_indices is None and not shared:
evict_host(hit_tokens)
host_indices = self.alloc_prefetch_host_buffers(operation, hit_tokens)
if host_indices is not None or not allow_partial:
return host_indices, hit_tokens
if shared:
low, high = 0, hit_tokens // self.page_size
while low < high:
mid = (low + high + 1) // 2
if self.can_fit_prefetch_host_buffers(
operation, mid * self.page_size, empty=False
):
low = mid
else:
high = mid - 1
alloc_len = low * self.page_size
else:
available = self.mem_pool_host.available_size()
alloc_len = min(hit_tokens, available - available % self.page_size)
if alloc_len >= min_tokens:
host_indices = self.alloc_prefetch_host_buffers(operation, alloc_len)
return host_indices, alloc_len
def free_prefetch_host_buffers(
self, operation: StorageOperation, host_indices: torch.Tensor
) -> None:
"""Roll back an unsubmitted prefetch's anchor and independent pools."""
pool_transfers = operation.pool_transfers or []
self.mem_pool_host.free(host_indices)
for transfer in pool_transfers:
if transfer.indices_from_pool is not None or transfer.host_indices is None:
continue
entry = self.mem_pool_host.entry_map.get(transfer.name)
if entry is not None:
entry.host_pool.free(transfer.host_indices)
transfer.host_indices = None
def _move_op_indices(
self, op: CacheOperation
) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]:
@@ -1308,16 +1456,26 @@ class HybridCacheController(BaseHiCacheController):
)
self._sync_trailing_keys(transfers_nonkv, sidecar_hashes, sidecar_hit_pages)
self._resolve_sidecar_nonkv_derived_pool_transfers(operation)
results = {}
for extra_info, transfers in _trailing_chain_groups(
operation.prefix_keys,
sidecar_hashes,
transfers_nonkv,
):
results.update(
self.storage_backend.batch_get_v2(transfers, extra_info=extra_info)
try:
results = {}
for extra_info, transfers in _trailing_chain_groups(
operation.prefix_keys,
sidecar_hashes,
transfers_nonkv,
):
results.update(
self.storage_backend.batch_get_v2(
transfers, extra_info=extra_info
)
)
pool_hits = count_pool_hits(results)
except Exception:
if not get_memory().enable_unified_memory:
raise
logger.exception(
"HiCache sidecar prefetch failed for request %s",
operation.request_id,
)
pool_hits = count_pool_hits(results)
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
# to perform all_reduce.
@@ -1521,7 +1679,7 @@ class HybridCacheController(BaseHiCacheController):
anchor_transfers = []
def rollback_allocated() -> None:
for prev_pool, prev_free_fn, prev_indices in newly_allocated:
for prev_pool, prev_free_fn, prev_indices in reversed(newly_allocated):
prev_free_fn(prev_indices)
prev_pool.device_indices = None
@@ -273,6 +273,13 @@ def build_kv_only_group(
def _swa_allocation_callbacks(allocator, bind=None, free_bound=None) -> dict:
"""Keep allocation and rollback in the same ID space for every SWA stack."""
from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator
if isinstance(allocator, MultiEndedAllocator):
return dict(
device_alloc_fn=allocator.alloc_physical,
device_free_fn=allocator.cancel_physical_reservation,
)
if bind is not None:
assert free_bound is not None
return dict(
@@ -300,7 +307,9 @@ def _uses_unified_page_envelope_host(
for pool in (full_kv_pool, swa_kv_pool)
)
and {full_kv_pool.grow_direction, swa_kv_pool.grow_direction} == {"up", "down"}
and get_memory().hicache_host_memory_mode != "buffer_only"
and HybridCacheController.supports_page_envelope_host(
get_memory().hicache_storage_backend
)
)
@@ -176,19 +176,6 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
backend = disagg.disaggregation_decode_retraction_backup
unified_hybrid_swa = memory.enable_unified_memory and tp_worker.is_hybrid_swa
unified_decode_radix = (
memory.enable_unified_memory and disagg.disaggregation_decode_enable_radix_cache
)
if backend == "host_pool" and unified_decode_radix:
raise ValueError(
"Unified-memory H2D/D2H does not support host-pool decode "
"retraction with decode radix cache yet."
)
if backend == "host_pool" and unified_hybrid_swa:
raise ValueError(
"Unified-memory hybrid-SWA H2D/D2H does not support host-pool "
"decode retraction yet."
)
if backend is None:
kv_cache = tp_worker.get_memory_pool()[1].get_kvcache()
full_tokens_per_layer = (
@@ -2174,6 +2174,7 @@ class KVCacheConfigurator:
swa_allocator = token_to_kv_pool_allocator.logical_attn_allocator
else:
swa_allocator = token_to_kv_pool_allocator
assert isinstance(swa_allocator, SWATokenToKVPoolAllocator)
uses_unified_virtual_ids = isinstance(
swa_allocator, UnifiedSWAAllocatorBase
)
@@ -2194,7 +2195,6 @@ class KVCacheConfigurator:
identity_mapping[-1] = -1
token_to_kv_pool.register_mapping(identity_mapping)
elif not uses_unified_virtual_ids:
assert isinstance(swa_allocator, SWATokenToKVPoolAllocator)
token_to_kv_pool.register_mapping(
swa_allocator.full_to_swa_index_mapping
)
@@ -64,6 +64,7 @@ class LogicalHostPool:
"""
shared_allocation_domain = None
stores_page_envelope = False
storage_format_tag = None
def __init__(self, size: int, page_size: int, layout: str = "layer_first"):
@@ -164,6 +164,7 @@ class HostKVCache(abc.ABC):
dcp_size = 1
dcp_rank = 0
shared_allocation_domain = None
stores_page_envelope = False
# Names this pool's page byte format in storage keys when it has one of its
# own, so pages persisted in another format miss instead of loading.
storage_format_tag: Optional[str] = None
@@ -291,6 +291,10 @@ class HostPoolGroup:
def size_per_token(self):
return self.anchor_entry.host_pool.size_per_token
@property
def stores_page_envelope(self) -> bool:
return self.anchor_entry.host_pool.stores_page_envelope
def clear(self) -> None:
for entry in self.entries:
entry.host_pool.clear()
@@ -303,15 +303,30 @@ class _SharedPageEnvelopeHostBacking:
extension_bytes += extension_pages * side.page_bytes
return extension_bytes <= self._current_gap_bytes()
def _can_fit_packed(self, page_counts: dict[str, int]) -> bool:
def _can_fit_state(
self,
live_page_counts: dict[str, int],
free_logical_page_counts: dict[str, int],
request_page_counts: dict[str, int],
) -> bool:
used_bytes = 0
for name, page_count in page_counts.items():
side = self.sides[name]
if page_count > self._logical_free_page_count(side):
for name, page_count in request_page_counts.items():
if page_count > free_logical_page_counts[name]:
return False
used_bytes += (side.live_page_count + page_count) * side.page_bytes
side = self.sides[name]
used_bytes += (live_page_counts[name] + page_count) * side.page_bytes
return used_bytes <= self._allocatable_bytes
def _can_fit_packed(self, page_counts: dict[str, int]) -> bool:
return self._can_fit_state(
{name: side.live_page_count for name, side in self.sides.items()},
{
name: self._logical_free_page_count(side)
for name, side in self.sides.items()
},
page_counts,
)
def _begin_compaction(self) -> list:
if self._layout_lease_state.depth:
raise RuntimeError(
@@ -543,6 +558,48 @@ class _SharedPageEnvelopeHostBacking:
self.sides[name].free_logical_extents = extents
return results
def can_fit_many_then(
self,
requests: Sequence[tuple[str, int]],
following_requests: Sequence[tuple[str, int]],
*,
empty: bool = False,
) -> bool:
"""Whether both allocation groups fit in order without mutating state."""
with self.lock:
_, first_page_counts = self._request_page_counts(requests)
_, following_page_counts = self._request_page_counts(following_requests)
if empty:
live_page_counts = {name: 0 for name in self.sides}
free_logical_page_counts = {
name: side.page_num for name, side in self.sides.items()
}
else:
live_page_counts = {
name: side.live_page_count for name, side in self.sides.items()
}
free_logical_page_counts = {
name: self._logical_free_page_count(side)
for name, side in self.sides.items()
}
if not self._can_fit_state(
live_page_counts, free_logical_page_counts, first_page_counts
):
return False
live_after_first = {
name: live_page_counts[name] + first_page_counts[name]
for name in self.sides
}
free_after_first = {
name: free_logical_page_counts[name] - first_page_counts[name]
for name in self.sides
}
return self._can_fit_state(
live_after_first,
free_after_first,
following_page_counts,
)
def free(self, name: str, indices: torch.Tensor) -> int:
with self.lock:
side = self.sides[name]
@@ -610,6 +667,7 @@ class _SharedPageEnvelopeHostBacking:
class UnifiedPageEnvelopeHostPool(HostKVCache):
"""Host mirror that transfers complete unified-memory page envelopes."""
stores_page_envelope = True
storage_format_tag = "unified-token-major"
def __init__(
+44 -1
View File
@@ -15,7 +15,8 @@
The scheduler supplies token demand and its chunk/decode limits. These objects
account for admitted but not yet allocated work and query live cache capacity:
locking a prefix or preempting a request must affect the next admission check.
They neither select requests nor mutate the prefix cache or allocator.
Selection checks do not mutate cache state. Shared-pool load preparation
realizes the selected reservation before a host transfer pins device rows.
"""
from typing import Optional
@@ -72,6 +73,18 @@ class PrefillBudget:
def ceil_paged_tokens(self, tokens: int) -> int:
return -(-tokens // self.page_size) * self.page_size
def prepare_load_back(
self,
*,
full_tokens: int,
extend_input_len: int,
max_new_tokens: int,
swa_host_hit_length: int,
chunk_limit: int | None,
) -> bool:
"""Prepare pools whose admission depends on movable shared space."""
return True
def _available_and_evictable(self):
evictable = (
self.tree_cache.full_evictable_size()
@@ -322,6 +335,36 @@ class SharedSWAPrefillBudget(SWAPrefillBudget):
require_token_slack=empty_pool,
)
def prepare_load_back(
self,
*,
full_tokens: int,
extend_input_len: int,
max_new_tokens: int,
swa_host_hit_length: int,
chunk_limit: int | None,
) -> bool:
# H2D pins both bands. Realize the selected joint budget while their
# holes can still be compacted, including pending prefill/decode demand.
full_tokens = int(self.ceil_paged_tokens(full_tokens + self.total_offset))
swa_tokens = int(
self.ceil_paged_tokens(
self.swa_tokens(
extend_input_len,
max_new_tokens,
chunk_limit=chunk_limit,
swa_host_hit_length=swa_host_hit_length,
)
+ self.swa_offset
)
)
return (
self.allocator.reclaim_for_prealloc(
self.tree_cache, full_tokens, swa_tokens
)
is None
)
def _joint_chunk_cap(self, *, max_chunk_tokens, chunk_limit, swa_host_hit_length=0):
lo, hi = 0, max(0, max_chunk_tokens) // self.page_size
while lo < hi:
@@ -5,7 +5,7 @@ from __future__ import annotations
import sys
from array import array
from typing import TYPE_CHECKING, Optional, Sequence
from typing import TYPE_CHECKING, Callable, Optional, Sequence
import torch
@@ -371,6 +371,19 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
)
self._page_size = params.page_size
self._swa_backup_index_mapper: Optional[
Callable[[torch.Tensor], torch.Tensor]
] = None
allocator = params.token_to_kv_pool_allocator
if allocator is not None and ComponentType.SWA in self.tree_components:
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWAAllocatorBase,
)
if isinstance(allocator, UnifiedSWAAllocatorBase):
self._swa_backup_index_mapper = (
allocator.translate_swa_indices_for_transfer
)
self.is_eagle = (
params.is_eagle and ComponentType.MAMBA not in self.tree_components
)
@@ -555,6 +568,11 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
result = EvictDeviceNextNodeResult(
node_id=binding_result.node_id,
made_progress=binding_result.made_progress,
backup_kv=(
_cache_action_from_tagged(binding_result.backup_kv)
if binding_result.backup_kv is not None
else None
),
unbacked_tokens=binding_result.unbacked_tokens,
mamba_backup_node_id=binding_result.mamba_backup_node_id,
swa_backup_node_id=binding_result.swa_backup_node_id,
@@ -770,6 +788,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def set_hicache_enabled(self) -> None:
self._binding.set_hicache_enabled()
def enable_swa_write_back_eviction_barrier(self) -> None:
self._binding.enable_swa_write_back_eviction_barrier()
def set_host_memory_buffer_only(self) -> None:
self._binding.set_host_memory_buffer_only()
@@ -868,11 +889,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
def _refresh_swa_backup_indices(self, transfers: Sequence[PoolTransfer]) -> None:
if not transfers:
return
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWAAllocatorBase,
)
if not isinstance(self._allocator, UnifiedSWAAllocatorBase):
if self._swa_backup_index_mapper is None:
return
for transfer in transfers:
if transfer.name != PoolName.SWA or not transfer.nodes_to_load:
@@ -883,7 +900,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
self.get_component_device_value(node_id, ComponentType.FULL)
for node_id in transfer.nodes_to_load
]
transfer.device_indices = self._allocator.translate_loc_from_full_to_swa(
transfer.device_indices = self._swa_backup_index_mapper(
torch.cat(full_values)
).to(torch.int64)
@@ -972,6 +989,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
) -> tuple[Optional[str], Optional[str]]:
return self._binding.prefetch_anchor_info(node_id)
def is_write_through_compatible(self) -> bool:
return self._binding.is_write_through_compatible()
def is_backuped(self, node_id: NodeId) -> bool:
return self._binding.node_backuped(node_id)
@@ -917,6 +917,7 @@ class UMBPStore(HiCacheStorage):
# Initialize before the optional constructor-time pool registration.
self.registered_pools: dict = {}
self._kv_anchor_is_logical = False
self._kv_anchor_is_page_envelope = False
self._registered_regions: set = set()
self.client = UMBPClient(cfg)
@@ -1001,6 +1002,7 @@ class UMBPStore(HiCacheStorage):
# A logical anchor owns indices; side pools carry the data.
self._kv_anchor_is_logical = self.mem_pool_host.kv_buffer is None
self._kv_anchor_is_page_envelope = self.mem_pool_host.stores_page_envelope
self._zero_copy_registered = False
# Side-pool registration needs the mode even for a logical anchor.
@@ -1163,68 +1165,61 @@ class UMBPStore(HiCacheStorage):
# ------------------------------------------------------------------
# Key suffix generation — mirrors MooncakeStore
# ------------------------------------------------------------------
def _get_mha_buffer_meta(self, keys, indices):
ptr_list, element_size_list = self.mem_pool_host.get_page_buffer_meta(indices)
key_list = []
for key_ in keys:
key_list.append(f"{key_}_{self.mha_suffix}_k")
key_list.append(f"{key_}_{self.mha_suffix}_v")
assert len(key_list) == len(ptr_list)
return key_list, ptr_list, element_size_list
def _get_mha_split_heads_buffer_meta(self, keys, indices):
ptr_list, element_size_list = (
self.mem_pool_host.get_split_heads_page_buffer_meta(
indices, self.split_factor
)
def _anchor_key_suffixes(self) -> tuple[str, ...]:
if self._kv_anchor_is_page_envelope:
return (f"_{self.mha_suffix}_kv",)
if self.is_mla_backend:
return (f"_{self.mla_suffix}_k",)
ranks = (
self.mha_suffix
if self.storage_config and self.storage_config.should_split_heads
else (self.mha_suffix,)
)
return tuple(
f"_{rank}_{component}" for rank in ranks for component in ("k", "v")
)
key_list = []
for key_ in keys:
for suffix in self.mha_suffix:
key_list.append(f"{key_}_{suffix}_k")
key_list.append(f"{key_}_{suffix}_v")
assert len(key_list) == len(ptr_list)
return key_list, ptr_list, element_size_list
def _get_mla_buffer_meta(self, keys, indices):
ptr_list, element_size_list = self.mem_pool_host.get_page_buffer_meta(indices)
key_list = []
for key_ in keys:
key_list.append(f"{key_}_{self.mla_suffix}_k")
assert len(key_list) == len(ptr_list)
return key_list, ptr_list, element_size_list
def _anchor_keys(self, keys) -> tuple[list[str], int]:
suffixes = self._anchor_key_suffixes()
return [f"{key}{suffix}" for key in keys for suffix in suffixes], len(suffixes)
def _batch_preprocess(self, keys, host_indices):
assert len(keys) > 0
assert len(keys) == len(host_indices) // self.mem_pool_host.page_size
if self.is_mla_backend:
return self._get_mla_buffer_meta(keys, host_indices)
key_list, _ = self._anchor_keys(keys)
if (
not self._kv_anchor_is_page_envelope
and not self.is_mla_backend
and self.storage_config
and self.storage_config.should_split_heads
):
ptr_list, element_size_list = (
self.mem_pool_host.get_split_heads_page_buffer_meta(
host_indices, self.split_factor
)
)
else:
if self.storage_config and self.storage_config.should_split_heads:
return self._get_mha_split_heads_buffer_meta(keys, host_indices)
else:
return self._get_mha_buffer_meta(keys, host_indices)
ptr_list, element_size_list = self.mem_pool_host.get_page_buffer_meta(
host_indices
)
assert len(key_list) == len(ptr_list)
return key_list, ptr_list, element_size_list
def _batch_postprocess(self, results: List[bool], is_set_operate=False):
"""Convert per-key-component results to per-page results.
For MHA: each page has K+V → group pairs.
For MLA: each page has K only.
Unified page envelopes and MLA have one object per page. Ordinary MHA
has K+V pairs, or one pair per split rank.
"""
if self.is_mla_backend:
group_size = len(self._anchor_key_suffixes())
if group_size == 1:
return list(results)
else:
if self.storage_config and self.storage_config.should_split_heads:
group_size = self.split_factor * 2
groups = [
results[i : i + group_size]
for i in range(0, len(results), group_size)
]
return [all(g) for g in groups]
else:
# Group K/V pairs
kv_pairs = zip(results[::2], results[1::2])
return [k and v for k, v in kv_pairs]
result_count = len(results)
if not (self.storage_config and self.storage_config.should_split_heads):
result_count -= result_count % group_size
return [
all(results[i : i + group_size]) for i in range(0, result_count, group_size)
]
# ------------------------------------------------------------------
# Zero-copy v1 interface
@@ -1282,19 +1277,8 @@ class UMBPStore(HiCacheStorage):
depths_per_page = [prefix_len + i for i in range(len(keys))]
# Expand to match the key_strs layout produced by _batch_preprocess.
expanded = []
for d in depths_per_page:
if self.is_mla_backend:
expanded.append(d) # MLA: 1 key per page
elif self.storage_config and self.storage_config.should_split_heads:
# split heads: 2 keys per split rank, split_factor ranks per page
for _ in range(self.split_factor):
expanded.append(d)
expanded.append(d)
else:
expanded.append(d) # K
expanded.append(d) # V
return expanded
objects_per_page = len(self._anchor_key_suffixes())
return [depth for depth in depths_per_page for _ in range(objects_per_page)]
def batch_set_v1(
self,
@@ -1352,23 +1336,7 @@ class UMBPStore(HiCacheStorage):
self, keys: List[str], extra_info: Optional[HiCacheStorageExtraInfo] = None
) -> int:
"""Return count of consecutive existing keys from start."""
if self.is_mla_backend:
query_keys = [f"{key}_{self.mla_suffix}_k" for key in keys]
key_multiplier = 1
else:
query_keys = []
if self.storage_config and self.storage_config.should_split_heads:
for key in keys:
for suffix in self.mha_suffix:
query_keys.append(f"{key}_{suffix}_k")
query_keys.append(f"{key}_{suffix}_v")
key_multiplier = 2 * self.split_factor
else:
for key in keys:
query_keys.append(f"{key}_{self.mha_suffix}_k")
query_keys.append(f"{key}_{self.mha_suffix}_v")
key_multiplier = 2
query_keys, key_multiplier = self._anchor_keys(keys)
hit_count = self.client.batch_exists_consecutive(query_keys)
return hit_count // key_multiplier
@@ -221,9 +221,14 @@ class SWAComponent(TreeComponent):
)
def _translate_full_to_swa(self, full_indices: torch.Tensor) -> torch.Tensor:
return self.cache.token_to_kv_pool_allocator.translate_loc_from_full_to_swa(
full_indices
swa_indices = (
self.cache.token_to_kv_pool_allocator.translate_swa_indices_for_transfer(
full_indices
)
)
# Tree component values use int64 indices; normalize transfer ids at the
# tree boundary.
return swa_indices.to(torch.int64)
def _unified_allocator(self):
"""The unified SWA composite, or None when running on the static pool."""
@@ -14,9 +14,11 @@ from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Optional
from sglang.srt.mem_cache.buffer_mode.pipeline import validate_buffer_only_stack
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.observability.metrics_collector import (
STAT_LOGGER_ROLE_STORAGE,
StorageMetricsCollector,
@@ -73,6 +75,23 @@ class StorageAttachment:
"launch with --enable-hierarchical-cache to attach a backend.",
)
# Write-back can retain auxiliary host rows without FULL, or backed
# descendants below an unbacked FULL ancestor. Those states are invalid
# for write-through insertion and eviction. Check before changing any
# policy, including on the same-backend update and re-attach paths.
if (
cache.is_write_back
and cache.host_memory_mode != "buffer_only"
and hicache_write_policy in ("write_through", "write_through_selective")
and not cache.tree_core.is_write_through_compatible()
):
return (
False,
"Cannot switch HiCache from write_back to write_through while "
"the cache has host data without its FULL prefix. "
"Flush the cache before changing the write policy.",
)
if cache.enable_storage:
current_backend = controller.storage_backend_type
if current_backend != storage_backend:
@@ -90,10 +109,6 @@ class StorageAttachment:
"policies updated.",
)
# Apply policies before the controller attach, so the storage threads
# observe the new values as soon as they start.
self._apply_policies(hicache_storage_prefetch_policy, hicache_write_policy)
logger.info(f"Attaching HiCache storage backend: {storage_backend}")
try:
(
@@ -113,8 +128,23 @@ class StorageAttachment:
f"'{storage_backend_extra_config_json}': {e}",
)
original_policies = (
cache.prefetch_stop_policy,
controller.write_policy,
cache.write_through_threshold,
cache.is_write_back,
)
try:
prefetch_threshold = self.resolve_prefetch_threshold(prefetch_threshold)
if cache.host_memory_mode == "buffer_only":
validate_buffer_only_stack(
sidecar_pool_specs=cache.sidecar_pool_specs,
host_pool_group=cache.host_pool_group,
swa_component=cache.components.get(ComponentType.SWA),
storage_prefetch_threshold=prefetch_threshold,
)
# New workers must see the requested policy from their first operation.
self._apply_policies(hicache_storage_prefetch_policy, hicache_write_policy)
controller.attach_storage_backend(
storage_backend=storage_backend,
prefetch_threshold=prefetch_threshold,
@@ -123,6 +153,12 @@ class StorageAttachment:
host_pools=controller.mem_pool_host.entries,
)
except Exception as e:
(
cache.prefetch_stop_policy,
controller.write_policy,
cache.write_through_threshold,
cache.is_write_back,
) = original_policies
logger.exception(
f"Failed to attach storage backend '{storage_backend}': {e}"
)
@@ -395,15 +431,19 @@ class StorageAttachment:
cache.buffer_pipeline.release_anchor_lock(handle)
controller.append_host_mem_release(
host_indices=info.host_indices[:completed_tokens],
extra_pools=[
x for xfers in info.comp_xfers.values() for x in xfers
],
extra_pools=(
[x for xfers in info.comp_xfers.values() for x in xfers]
if info.operation.pool_transfers_done
else None
),
)
controller.prefetch_tokens_occupied = max(
0,
controller.prefetch_tokens_occupied
- cache._prefetch_occupied_span(
info.prefetch_key, info.host_indices
info.prefetch_key,
info.host_indices,
operation=info.operation,
),
)
except Exception:
@@ -544,6 +544,22 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
"""
return self._node_arena[node_id]
def is_write_through_compatible(self) -> bool:
for node in self._node_arena.values():
if node is self.root_node:
continue
full_host = node.component_data[BASE_COMPONENT_TYPE].host_value
if full_host is None:
if any(
node.component_data[ct].host_value is not None
for ct in self.component_types
if ct != BASE_COMPONENT_TYPE
):
return False
elif node.parent is not self.root_node and not node.parent.backuped:
return False
return True
def is_backuped(self, node_id: NodeId) -> bool:
"""Whether the node's KV is already backed up to host."""
return self._node_arena[node_id].backuped
@@ -1756,14 +1772,20 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
# A failed backup never issues the D->H copy, so the subtree root has
# no host state and no in-flight DMA reading its device slots.
assert not node.backuped and node.write_through_pending_id is None
if any(cd.host_lock_ref > 0 for cd in node.component_data):
if node.load_back_pending_id is not None or any(
cd.lock_ref > 0 or cd.host_lock_ref > 0 for cd in node.component_data
):
return result
descendants: list[UnifiedTreeNode] = []
stack = list(node.children.values())
while stack:
cur = stack.pop()
if any(
cd.lock_ref > 0 or cd.host_lock_ref > 0 for cd in cur.component_data
if (
cur.write_through_pending_id is not None
or cur.load_back_pending_id is not None
or any(
cd.lock_ref > 0 or cd.host_lock_ref > 0 for cd in cur.component_data
)
):
return result
descendants.append(cur)
@@ -2275,12 +2297,17 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
child_key = key.child_key(self.page_size)
matched_length = 0
host_prefix_len = 0
refilled_host_node = None
cache_actions: list[CacheAction | ComponentAction] = []
while len(key) > 0 and child_key in node.children:
node = node.children[child_key]
self._touch_node(node)
prefix_len = node.key.match(key, page_size=self.page_size)
matched_host_value = host_value[:prefix_len]
matched_hash_value = hash_value[: prefix_len // self.page_size]
key = key[prefix_len:]
host_value = host_value[prefix_len:]
hash_value = hash_value[prefix_len // self.page_size :]
@@ -2291,18 +2318,37 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
if action is not None:
cache_actions.append(action)
if not self.is_write_back:
full_data = node.component_data[BASE_COMPONENT_TYPE]
if full_data.host_value is None:
full_data.host_value = matched_host_value.clone()
if node.hash_value is None:
node.hash_value = list(matched_hash_value)
self.kv_events.record_store(node, medium=StorageMedium.CPU)
self._update_evictable_leaf_sets(node)
if node.parent is not None:
self._update_evictable_leaf_sets(node.parent)
self._update_duplicate_tracking(node)
refilled_host_node = node
else:
assert refilled_host_node is None, (
"write-through host-prefix invariant broken: encountered "
f"backed node {node.id} below a refilled host tombstone"
)
host_prefix_len += prefix_len
if len(key):
child_key = key.child_key(self.page_size)
result = InsertResult(
prefix_len=matched_length,
prefix_len=(matched_length if self.is_write_back else host_prefix_len),
total_len=total_len,
cache_actions=cache_actions,
)
if len(key) == 0:
if (
node is not self.root_node
and node.component_data[BASE_COMPONENT_TYPE].host_value is not None
if node is not self.root_node and (
refilled_host_node is not None
or node.component_data[BASE_COMPONENT_TYPE].host_value is not None
):
result.inserted_host_node = node.id
return result
@@ -52,6 +52,7 @@ class EvictDeviceNextNodeResult(BaseEvictionResult):
"""
node_id: Optional[NodeId] = None
backup_kv: Optional[BackupKV] = None
made_progress: bool = False
unbacked_tokens: int = 0
mamba_backup_node_id: Optional[NodeId] = None
@@ -183,6 +184,14 @@ class UnifiedTreeCoreInterface(ABC):
"""
...
@abstractmethod
def is_write_through_compatible(self) -> bool:
"""Whether current host ownership satisfies write-through invariants.
Read-only; the caller must have drained in-flight cache operations.
"""
...
@abstractmethod
def is_backuped(self, node_id: NodeId) -> bool:
"""Whether the node's KV is already backed up to host."""
@@ -510,6 +519,13 @@ class UnifiedTreeCoreInterface(ABC):
"""Mark the host tier (HiCache) as wired."""
...
def enable_swa_write_back_eviction_barrier(self) -> None:
"""Enable a backend-managed barrier when needed.
The Python core demotes through SWAComponent directly. Native cores
may return a backup action to the cache executor before eviction.
"""
@abstractmethod
def set_host_memory_buffer_only(self) -> None:
"""Mark the host tier as buffer-only: one node staged per backup intent."""
@@ -287,6 +287,7 @@ class UnifiedRadixCache(BasePrefixCache):
"declined_rate_limited": 0,
"declined_anchor_lost": 0,
"declined_device_covered": 0,
"declined_host_oversize": 0,
"revoked_insufficient": 0,
"revoked_full_miss": 0,
"l3_demand_requests": 0,
@@ -470,6 +471,7 @@ class UnifiedRadixCache(BasePrefixCache):
sidecar_pool_specs=self.sidecar_pool_specs,
host_pool_group=self.host_pool_group,
swa_component=swa,
storage_prefetch_threshold=storage_prefetch_threshold,
)
self.buffer_pipeline = BufferModePipeline(
cache=self,
@@ -485,11 +487,11 @@ class UnifiedRadixCache(BasePrefixCache):
# content would drop-newest and punch storage holes.
write_backlog_cap=2 * self.token_to_kv_pool_allocator.size_full,
)
# State initialization
if self.buffer_pipeline is not None:
self.cache_controller.host_write_staged_tokens_fn = lambda: (
self.buffer_pipeline.write_staged_tokens_
)
# State initialization
self.write_through_threshold = (
1 if get_memory().hicache_write_policy == "write_through" else 2
)
@@ -497,6 +499,13 @@ class UnifiedRadixCache(BasePrefixCache):
self.cache_controller is not None
and self.cache_controller.write_policy == "write_back"
)
# Preserve the SWA host window before device eviction makes it unrecoverable.
if (
get_memory().enable_unified_memory
and self.host_memory_mode == "cache"
and self.tree_core.has_swa_host_pool
):
self.tree_core.enable_swa_write_back_eviction_barrier()
# Pre-seed the logical dropped-tokens series.
if self.metrics_collector is not None and self.cache_controller is not None:
reasons = ["host_pressure"]
@@ -749,45 +758,74 @@ class UnifiedRadixCache(BasePrefixCache):
def _evict_device_next_node(
self, component_type: ComponentType, tracker: dict[ComponentType, int]
) -> tuple[Optional[NodeId], bool]:
"""Advance the eviction walk one node, consuming its step result."""
result = self.tree_core.evict_device_next_node(component_type, tracker)
if result.mamba_backup_node_id is not None:
assert component_type == ComponentType.MAMBA and result.node_id is None
assert (
not result.device_frees and not result.host_frees and not result.tracker
"""Advance the walk, completing pending host-backup barriers."""
while True:
result = self.tree_core.evict_device_next_node(component_type, tracker)
if result.mamba_backup_node_id is not None:
assert component_type == ComponentType.MAMBA and result.node_id is None
assert (
not result.device_frees
and not result.host_frees
and not result.tracker
)
# Reserve a host state slot and wait for the backup acknowledgment
# before freeing device state. If allocation fails, eviction still
# proceeds to make room on the device.
node_id = result.mamba_backup_node_id
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
self.evict_host(1, ComponentType.MAMBA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_mamba_state_eviction(node_id)
elif result.swa_backup_node_id is not None:
assert component_type == ComponentType.SWA and result.node_id is None
assert (
not result.device_frees
and not result.host_frees
and not result.tracker
)
# The backup can cover several unbacked SWA segments. Reserve host
# space for the whole window before copying it, then resume eviction
# even if host allocation fails.
node_id = result.swa_backup_node_id
needed = result.swa_backup_num_tokens
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
if (
swa_host_pool is not None
and swa_host_pool.available_size() < needed
):
self.evict_host(needed, ComponentType.SWA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_swa_state_eviction(node_id)
self._free_values(result.device_frees, result.host_frees)
if self._tracks_write_through_unbacked_evictions():
self._record_dropped_tokens(
result.unbacked_tokens,
reason="write_through_unbacked_eviction",
)
self._accumulate_tracker(tracker, result.tracker)
if result.backup_kv is None:
return result.node_id, result.made_progress
assert result.node_id is None
assert self.buffer_pipeline is None, (
"SWA write-back eviction barriers are cache-mode only"
)
# Reserve a host state slot and wait for the backup acknowledgment
# before freeing device state. If allocation fails, eviction still
# proceeds to make room on the device.
node_id = result.mamba_backup_node_id
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
self.evict_host(1, ComponentType.MAMBA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_mamba_state_eviction(node_id)
elif result.swa_backup_node_id is not None:
assert component_type == ComponentType.SWA and result.node_id is None
assert (
not result.device_frees and not result.host_frees and not result.tracker
written = self._execute_and_commit_kv_backup(
result.backup_kv, write_back=True
)
# The backup can cover several unbacked SWA segments. Reserve host
# space for the whole window before copying it, then resume eviction
# even if host allocation fails.
node_id = result.swa_backup_node_id
needed = result.swa_backup_num_tokens
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
if swa_host_pool is not None and swa_host_pool.available_size() < needed:
self.evict_host(needed, ComponentType.SWA)
self.backup_node_for_write_back(node_id)
result = self.tree_core.finish_swa_state_eviction(node_id)
self._free_values(result.device_frees, result.host_frees)
if self._tracks_write_through_unbacked_evictions():
self._record_dropped_tokens(
result.unbacked_tokens,
reason="write_through_unbacked_eviction",
)
self._accumulate_tracker(tracker, result.tracker)
return result.node_id, result.made_progress
if written <= 0:
node_id = result.backup_kv.node_ids[0]
logger.warning(
"write_back: auxiliary backup failed under host pressure "
"(component=%s, node=%d); dropping only the component",
component_type.name,
node_id,
)
# Match the Python SWA demotion: preserve FULL and descendants;
# the resumed native walk tombstones this component after one try.
continue
self.writing_check(write_back=True)
def _evict_device_leaf(
self, node_id: NodeId, tracker: dict[ComponentType, int]
@@ -1366,14 +1404,17 @@ class UnifiedRadixCache(BasePrefixCache):
component_transfers: dict[ComponentType, list[PoolTransfer]] = {}
if self.supports_swa():
kv_cache = self.token_to_kv_pool_allocator.get_kvcache()
assert self.sliding_window_size is not None
window_start = max(0, num_tokens - self.sliding_window_size)
window_start = window_start // self.page_size * self.page_size
window_indices = self.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, window_start:num_tokens
].to(torch.int64)
swa_indices = kv_cache.translate_loc_from_full_to_swa(window_indices)
swa_indices = (
self.token_to_kv_pool_allocator.translate_swa_indices_for_transfer(
window_indices
)
)
assert bool((swa_indices > 0).all()), (
f"unmapped SWA window positions for request {req.rid}"
)
@@ -1392,13 +1433,12 @@ class UnifiedRadixCache(BasePrefixCache):
for transfers in component_transfers.values()
for transfer in transfers
]
extra_transfers.extend(
self._build_sidecar_transfers(
CacheTransferPhase.BACKUP_HOST,
kv_transfer,
component_transfers,
)
sidecar_transfers = self._build_sidecar_transfers(
CacheTransferPhase.BACKUP_HOST,
kv_transfer,
component_transfers,
)
extra_transfers.extend(sidecar_transfers)
return full_indices, extra_transfers
def _reclaim_retraction_host(self, num_tokens: int) -> int:
@@ -2397,7 +2437,9 @@ class UnifiedRadixCache(BasePrefixCache):
self.buffer_pipeline.release_anchor_lock(request)
del self.ongoing_prefetch[request]
self.cache_controller.prefetch_tokens_occupied -= (
self._prefetch_occupied_span(prefetch_key, host_indices)
self._prefetch_occupied_span(
prefetch_key, host_indices, operation=operation
)
)
self.prefetch_loaded_tokens_by_reqid[request] = 0
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
@@ -2584,7 +2626,7 @@ class UnifiedRadixCache(BasePrefixCache):
# Buffer mode granted occupancy at hit-alloc, sized to the bounce;
# cache mode reserved the requested span at enqueue.
self.cache_controller.prefetch_tokens_occupied -= self._prefetch_occupied_span(
prefetch_key, host_indices
prefetch_key, host_indices, operation=operation
)
def _invalidate_absent_from_hit_query(self, operation) -> None:
@@ -2621,11 +2663,18 @@ class UnifiedRadixCache(BasePrefixCache):
def prefetch_outcome_stats_snapshot(self) -> dict:
return self._prefetch_outcome_stats.copy()
def _prefetch_occupied_span(self, prefetch_key, host_indices) -> int:
def _prefetch_occupied_span(
self, prefetch_key, host_indices, *, operation=None
) -> int:
"""Occupancy units held by a prefetch: cache mode reserves the
requested span at enqueue; buffer mode grants at hit-alloc, sized
to the allocation (0 while still querying / parked)."""
if self.host_memory_mode == "buffer_only":
if (
operation is not None
and operation.buffer_host_occupied_units is not None
):
return operation.buffer_host_occupied_units
return len(host_indices) if host_indices is not None else 0
return len(prefetch_key)
@@ -2742,7 +2791,9 @@ class UnifiedRadixCache(BasePrefixCache):
cc.prefetch_tokens_occupied = max(
0,
cc.prefetch_tokens_occupied
- self._prefetch_occupied_span(prefetch_key, _host_indices),
- self._prefetch_occupied_span(
prefetch_key, _host_indices, operation=operation
),
)
def _drain_storage_control_queues_impl(
@@ -2827,20 +2878,24 @@ class UnifiedRadixCache(BasePrefixCache):
else:
aux_hit_tokens = hit_tokens
alloc_len = hit_tokens
host_indices = cc.mem_pool_host.alloc(alloc_len)
if host_indices is None:
self.evict_host(alloc_len)
host_indices = cc.mem_pool_host.alloc(alloc_len)
if host_indices is None and not buffer_mode:
# Memory-pressure fallback: a shorter page-aligned prefix.
# (Cache mode only — buffer mode parks for the full hit.)
available_size = cc.mem_pool_host.available_size()
alloc_len = min(
hit_tokens,
available_size - (available_size % self.page_size),
if buffer_mode and not cc.can_fit_prefetch_host_buffers(
operation, alloc_len
):
self._prefetch_outcome_stats["declined_host_oversize"] += 1
logger.warning(
"HiCache buffer prefetch declined req=%s: the Full/sidecar "
"bounce cannot fit in an empty host arena",
request,
)
if alloc_len >= self.prefetch_threshold:
host_indices = cc.mem_pool_host.alloc(alloc_len)
self.revoke_pending_prefetch(request)
return True
host_indices, alloc_len = cc.allocate_storage_hit(
operation,
hit_tokens,
allow_partial=not buffer_mode,
min_tokens=self.prefetch_threshold,
evict_host=self.evict_host,
)
if host_indices is None:
if buffer_mode:
# Parked ops hold no pin: release and re-take at the next
@@ -2890,7 +2945,12 @@ class UnifiedRadixCache(BasePrefixCache):
operation.host_indices = host_indices
self.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
if buffer_mode:
cc.prefetch_tokens_occupied += alloc_len
operation.buffer_host_occupied_units = (
self.buffer_pipeline.host_allocation_units(
host_indices, operation.pool_transfers
)
)
cc.prefetch_tokens_occupied += operation.buffer_host_occupied_units
cc.prefetch_buffer.put(operation)
return True