[HiCache] Block-group pool hits, caller-owned load destinations, allocation-target FULL eviction (#43868)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
xiezhq-hermann
parent
8bf145d119
commit
a8381d73be
@@ -93,10 +93,15 @@ class PoolHitPolicy(str, Enum):
|
||||
|
||||
ALL_PAGES : every page in [0, kv_hit) must exist (e.g. DSA).
|
||||
TRAILING_PAGES : only the last N pages must exist (e.g. Mamba/SWA states).
|
||||
BLOCK_GROUPS : whole key groups at block boundaries from the span start
|
||||
(a second form of the FULL pages, e.g. compressed blocks).
|
||||
The pool reports the pages its leading complete groups
|
||||
cover as its own verdict; it never gates the KV hit.
|
||||
"""
|
||||
|
||||
ALL_PAGES = "all_pages"
|
||||
TRAILING_PAGES = "trailing_pages"
|
||||
BLOCK_GROUPS = "block_groups"
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -115,6 +120,9 @@ class PoolTransfer:
|
||||
hit_policy: PoolHitPolicy = PoolHitPolicy.ALL_PAGES
|
||||
nodes_to_load: Optional[List[Any]] = None
|
||||
indices_from_pool: Optional[PoolName] = None
|
||||
# BLOCK_GROUPS geometry: FULL pages per block and keys (pages) per group.
|
||||
block_pages: int = 0
|
||||
group_pages: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
@@ -156,6 +156,10 @@ class PrefetchOperation(StorageOperation):
|
||||
self._lock = threading.Lock()
|
||||
self._terminated_flag = False
|
||||
self.storage_hit_count = 0
|
||||
# KV pages the hit-time plan skips in front of the KV transfer: the
|
||||
# device trim, then the span head a component's block groups serve.
|
||||
self.prefetch_kv_page_offset = 0
|
||||
self.group_skip_pages = 0
|
||||
self.start_time = time.monotonic()
|
||||
# Take the whole span as present instead of querying for it; the read
|
||||
# itself is fail-soft, so a wrong guess shortens the fetch.
|
||||
@@ -963,15 +967,25 @@ class HybridCacheController(BaseHiCacheController):
|
||||
priority: Optional[int] = None,
|
||||
node_id: int = -1,
|
||||
extra_pools: Optional[list[PoolTransfer]] = None,
|
||||
*,
|
||||
device_indices: Optional[torch.Tensor] = None,
|
||||
anchor_indices: Optional[torch.Tensor] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Queue an H2D of ``host_indices`` into freshly allocated FULL rows,
|
||||
or into caller-owned ``device_indices`` (kept on failure); side pools
|
||||
bind their device slots against ``anchor_indices`` (default: the KV
|
||||
destination rows)."""
|
||||
need_load_kv = host_indices.numel() > 0
|
||||
caller_owned = device_indices is not None
|
||||
|
||||
full_allocator = getattr(
|
||||
self.mem_pool_device_allocator,
|
||||
"full_attn_allocator",
|
||||
self.mem_pool_device_allocator,
|
||||
)
|
||||
if not need_load_kv:
|
||||
if caller_owned:
|
||||
assert len(device_indices) == len(host_indices)
|
||||
elif not need_load_kv:
|
||||
device_indices = torch.empty((0,), dtype=torch.int64, device=self.device)
|
||||
else:
|
||||
device_indices = full_allocator.alloc(len(host_indices))
|
||||
@@ -980,11 +994,13 @@ class HybridCacheController(BaseHiCacheController):
|
||||
|
||||
pool_transfers = self._resolve_device_transfers(
|
||||
extra_pools,
|
||||
kv_device_indices=device_indices,
|
||||
kv_device_indices=(
|
||||
anchor_indices if anchor_indices is not None else device_indices
|
||||
),
|
||||
kv_host_indices=host_indices,
|
||||
)
|
||||
if pool_transfers is None and extra_pools:
|
||||
if need_load_kv:
|
||||
if need_load_kv and not caller_owned:
|
||||
full_allocator.free(device_indices)
|
||||
return None
|
||||
|
||||
@@ -1543,6 +1559,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
operation.sidecar_hash_values = original_hashes
|
||||
operation.sidecar_hit_pages = len(original_hashes)
|
||||
operation.hash_value = original_hashes[trim_pages:]
|
||||
operation.prefetch_kv_page_offset += trim_pages
|
||||
operation.storage_hit_count -= trim_tokens
|
||||
operation.storage_start += trim_tokens
|
||||
|
||||
|
||||
@@ -912,11 +912,29 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
for ct in self.tree_components:
|
||||
request_cnt = request_by_type[ct]
|
||||
# An allocation-driven FULL request names the allocator's
|
||||
# shortfall, not what the walk must free: an allocator can hide
|
||||
# headroom behind a clamp (a reserve it refills before raw
|
||||
# allocation sees a page again) or page rounding, so freeing
|
||||
# exactly the shortfall can leave its target unmet. Such a walk
|
||||
# runs to the target; the check after every victim still stops it
|
||||
# at the first sufficient one.
|
||||
to_target = (
|
||||
ct == ComponentType.FULL
|
||||
and request_cnt > 0
|
||||
and available_size_targets is not None
|
||||
and ct in available_size_targets
|
||||
)
|
||||
# A preceding component may have cascade-evicted this component or,
|
||||
# on a shared pool, released enough bytes to satisfy its allocation.
|
||||
if tracker[ct] >= request_cnt or target_reached(ct):
|
||||
if target_reached(ct) or (not to_target and tracker[ct] >= request_cnt):
|
||||
continue
|
||||
self.tree_core.evict_device_start(ct, request_cnt)
|
||||
walk_quota = request_cnt
|
||||
if to_target:
|
||||
walk_quota = max(
|
||||
request_cnt, tracker[ct] + self.tree_core.full_evictable_size()
|
||||
)
|
||||
self.tree_core.evict_device_start(ct, walk_quota)
|
||||
try:
|
||||
while True:
|
||||
node_id, made_progress = self._evict_device_next_node(ct, tracker)
|
||||
|
||||
Reference in New Issue
Block a user