[HiCache] Block-group pool hits, caller-owned load destinations, allocation-target FULL eviction (#43868)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-13 11:39:48 -07:00
committed by GitHub
co-authored by Hanming Lu xiezhq-hermann
parent 8bf145d119
commit a8381d73be
6 changed files with 137 additions and 5 deletions
@@ -93,10 +93,15 @@ class PoolHitPolicy(str, Enum):
ALL_PAGES : every page in [0, kv_hit) must exist (e.g. DSA).
TRAILING_PAGES : only the last N pages must exist (e.g. Mamba/SWA states).
BLOCK_GROUPS : whole key groups at block boundaries from the span start
(a second form of the FULL pages, e.g. compressed blocks).
The pool reports the pages its leading complete groups
cover as its own verdict; it never gates the KV hit.
"""
ALL_PAGES = "all_pages"
TRAILING_PAGES = "trailing_pages"
BLOCK_GROUPS = "block_groups"
@dataclass
@@ -115,6 +120,9 @@ class PoolTransfer:
hit_policy: PoolHitPolicy = PoolHitPolicy.ALL_PAGES
nodes_to_load: Optional[List[Any]] = None
indices_from_pool: Optional[PoolName] = None
# BLOCK_GROUPS geometry: FULL pages per block and keys (pages) per group.
block_pages: int = 0
group_pages: int = 0
@dataclass(frozen=True)
@@ -156,6 +156,10 @@ class PrefetchOperation(StorageOperation):
self._lock = threading.Lock()
self._terminated_flag = False
self.storage_hit_count = 0
# KV pages the hit-time plan skips in front of the KV transfer: the
# device trim, then the span head a component's block groups serve.
self.prefetch_kv_page_offset = 0
self.group_skip_pages = 0
self.start_time = time.monotonic()
# Take the whole span as present instead of querying for it; the read
# itself is fail-soft, so a wrong guess shortens the fetch.
@@ -963,15 +967,25 @@ class HybridCacheController(BaseHiCacheController):
priority: Optional[int] = None,
node_id: int = -1,
extra_pools: Optional[list[PoolTransfer]] = None,
*,
device_indices: Optional[torch.Tensor] = None,
anchor_indices: Optional[torch.Tensor] = None,
) -> Optional[torch.Tensor]:
"""Queue an H2D of ``host_indices`` into freshly allocated FULL rows,
or into caller-owned ``device_indices`` (kept on failure); side pools
bind their device slots against ``anchor_indices`` (default: the KV
destination rows)."""
need_load_kv = host_indices.numel() > 0
caller_owned = device_indices is not None
full_allocator = getattr(
self.mem_pool_device_allocator,
"full_attn_allocator",
self.mem_pool_device_allocator,
)
if not need_load_kv:
if caller_owned:
assert len(device_indices) == len(host_indices)
elif not need_load_kv:
device_indices = torch.empty((0,), dtype=torch.int64, device=self.device)
else:
device_indices = full_allocator.alloc(len(host_indices))
@@ -980,11 +994,13 @@ class HybridCacheController(BaseHiCacheController):
pool_transfers = self._resolve_device_transfers(
extra_pools,
kv_device_indices=device_indices,
kv_device_indices=(
anchor_indices if anchor_indices is not None else device_indices
),
kv_host_indices=host_indices,
)
if pool_transfers is None and extra_pools:
if need_load_kv:
if need_load_kv and not caller_owned:
full_allocator.free(device_indices)
return None
@@ -1543,6 +1559,7 @@ class HybridCacheController(BaseHiCacheController):
operation.sidecar_hash_values = original_hashes
operation.sidecar_hit_pages = len(original_hashes)
operation.hash_value = original_hashes[trim_pages:]
operation.prefetch_kv_page_offset += trim_pages
operation.storage_hit_count -= trim_tokens
operation.storage_start += trim_tokens
@@ -912,11 +912,29 @@ class UnifiedRadixCache(BasePrefixCache):
for ct in self.tree_components:
request_cnt = request_by_type[ct]
# An allocation-driven FULL request names the allocator's
# shortfall, not what the walk must free: an allocator can hide
# headroom behind a clamp (a reserve it refills before raw
# allocation sees a page again) or page rounding, so freeing
# exactly the shortfall can leave its target unmet. Such a walk
# runs to the target; the check after every victim still stops it
# at the first sufficient one.
to_target = (
ct == ComponentType.FULL
and request_cnt > 0
and available_size_targets is not None
and ct in available_size_targets
)
# A preceding component may have cascade-evicted this component or,
# on a shared pool, released enough bytes to satisfy its allocation.
if tracker[ct] >= request_cnt or target_reached(ct):
if target_reached(ct) or (not to_target and tracker[ct] >= request_cnt):
continue
self.tree_core.evict_device_start(ct, request_cnt)
walk_quota = request_cnt
if to_target:
walk_quota = max(
request_cnt, tracker[ct] + self.tree_core.full_evictable_size()
)
self.tree_core.evict_device_start(ct, walk_quota)
try:
while True:
node_id, made_progress = self._evict_device_next_node(ct, tracker)