Fix unified HiCache physical transfers (#39479)
Co-authored-by: yhzhuang <yhzhuang@fb.com> Co-authored-by: Lianmin Zheng <lianminzheng@gmail.com> Co-authored-by: Yonghao Zhuang <yhzhuang@users.noreply.github.com> Co-authored-by: Cheng Wan <cheng.wan@radixark.ai> Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
This commit is contained in:
co-authored by
yhzhuang
Lianmin Zheng
Yonghao Zhuang
Cheng Wan
metamergebot
parent
6cc661f223
commit
937c0a6cfe
@@ -530,18 +530,10 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
)
|
||||
if cfg.disaggregation_decode_retraction_backup == "host_pool":
|
||||
model_config = model_config_of(server_args)
|
||||
assert not cfg.disaggregation_decode_enable_radix_cache, (
|
||||
"--enable-unified-memory host-pool decode retraction does not "
|
||||
"support decode radix-cache H2D/D2H transfers yet."
|
||||
)
|
||||
assert mambaish_config(model_config) is None, (
|
||||
"--enable-unified-memory host-pool decode retraction does not "
|
||||
"support hybrid-Mamba models."
|
||||
)
|
||||
assert not model_config.is_hybrid_swa, (
|
||||
"--enable-unified-memory host-pool decode retraction does not "
|
||||
"support hybrid-SWA H2D/D2H transfers yet."
|
||||
)
|
||||
assert cfg.speculative_algorithm in (None, "DSPARK"), (
|
||||
"--enable-unified-memory only supports --speculative-algorithm "
|
||||
"DSPARK (chain draft); other speculative algorithms are not yet "
|
||||
@@ -578,6 +570,21 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
"the LMCache offload path indexes the device buffers with the ids it "
|
||||
"is handed, and under the unified pool those are VIRTUAL."
|
||||
)
|
||||
assert not cfg.enable_unified_cache_external_linker, (
|
||||
"--enable-unified-memory does not support "
|
||||
"--enable-unified-cache-external-linker: direct L3 transfers do not "
|
||||
"preserve unified page-envelope indices and compaction lifetimes. "
|
||||
"Use --enable-hierarchical-cache for supported L2/L3 transfers."
|
||||
)
|
||||
if cfg.enable_hierarchical_cache:
|
||||
assert cfg.pp_size == 1, (
|
||||
"--enable-unified-memory with hierarchical cache does not support "
|
||||
"pipeline parallelism (--pp-size > 1)."
|
||||
)
|
||||
assert not envs.SGLANG_DISABLE_LAZY_COMPACTION.get(), (
|
||||
"--enable-unified-memory with hierarchical cache requires lazy "
|
||||
"compaction so pending H2D physical reservations remain stable."
|
||||
)
|
||||
if cfg.dcp_size > 1:
|
||||
_validate_unified_memory_dcp(server_args)
|
||||
# Prefill cuda-graph capture IS wired for the unified pool: the captured
|
||||
|
||||
@@ -787,12 +787,19 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
Match a request against the decode-side radix cache, lock the matched
|
||||
node to prevent eviction, and return the matched prefix information.
|
||||
"""
|
||||
max_prefix_len = None
|
||||
if self._uses_swa_tail_prealloc():
|
||||
fill_len = self._pre_alloc_fill_len(req)
|
||||
max_prefix_len = fill_len - self._swa_tail_len(fill_len)
|
||||
# Match and lock only reusable FULL KV. The entire SWA tail must be
|
||||
# freshly allocated, including when the prefix comes from L2/L3.
|
||||
result = match_prefix_for_req(
|
||||
self.tree_cache,
|
||||
req,
|
||||
req.origin_input_ids,
|
||||
cow_mamba=self.tree_cache.supports_mamba(),
|
||||
include_req=True,
|
||||
max_prefix_len=max_prefix_len,
|
||||
)
|
||||
# Keep aggregated scheduling semantics while preserving the SWA lock
|
||||
# boundary needed for the matching dec_lock_ref; the full receipt
|
||||
@@ -800,7 +807,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
req.lock_receipt = self.tree_cache.inc_lock_ref(
|
||||
result.last_device_node
|
||||
).to_dec_params()
|
||||
return self._build_decode_prefix_match(req, result)
|
||||
return self._build_decode_prefix_match(
|
||||
req, result, max_prefix_len=max_prefix_len
|
||||
)
|
||||
|
||||
def _resolve_prefill_dp_rank(self, req: Req) -> Optional[int]:
|
||||
prefill_info = self.kv_manager.prefill_info_table.get(_bootstrap_addr(req))
|
||||
@@ -1422,20 +1431,6 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
|
||||
fill_len = self._pre_alloc_fill_len(decode_req.req)
|
||||
|
||||
# Cap full-attention prefix reuse at the sliding-window start so
|
||||
# the SWA window lands entirely in the fresh delta, keeping
|
||||
# alloc_extend_swa_tail's tail->full mapping in range. Costs reuse
|
||||
# of only the last ~window_size full-attention tokens.
|
||||
if uses_swa_tail_prealloc and prefix_len > 0:
|
||||
swa_prefix_cap = fill_len - self._swa_tail_len(fill_len)
|
||||
if prefix_len > swa_prefix_cap:
|
||||
prefix_len = swa_prefix_cap
|
||||
prefix_indices = prefix_indices[:prefix_len]
|
||||
# Cap the prefill-committed prefix too: tokens past the
|
||||
# cap are not device-resident, so prefill must transfer
|
||||
# them.
|
||||
total_prefix_len = prefix_len
|
||||
|
||||
# Decode transfers the SWA tail fresh, so retain only the
|
||||
# full-attention prefix lock needed for reuse.
|
||||
if (
|
||||
@@ -2317,9 +2312,9 @@ def alloc_for_decode_prealloc(
|
||||
# the live window tail.
|
||||
kv_loc = allocator.alloc_extend_swa_tail(
|
||||
prefix_lens=torch.tensor(
|
||||
[prefix_len], dtype=torch.int64, device=device
|
||||
[total_prefix_len], dtype=torch.int64, device=device
|
||||
),
|
||||
prefix_lens_cpu=torch.tensor([prefix_len], dtype=torch.int64),
|
||||
prefix_lens_cpu=torch.tensor([total_prefix_len], dtype=torch.int64),
|
||||
seq_lens=torch.tensor([fill_len], dtype=torch.int64, device=device),
|
||||
seq_lens_cpu=torch.tensor([fill_len], dtype=torch.int64),
|
||||
last_loc=last_loc,
|
||||
|
||||
@@ -61,7 +61,9 @@ class HiCacheRestoreResult(Enum):
|
||||
class DecodeHiCachePreallocMixin:
|
||||
"""HiCache hooks for ``DecodePreallocQueue``: issue prefetch + reserve tokens."""
|
||||
|
||||
def _build_decode_prefix_match(self, req: Req, result: Any) -> DecodePrefixMatch:
|
||||
def _build_decode_prefix_match(
|
||||
self, req: Req, result: Any, *, max_prefix_len: Optional[int] = None
|
||||
) -> DecodePrefixMatch:
|
||||
"""Convert a ``match_prefix_for_req`` result into ``DecodePrefixMatch``.
|
||||
|
||||
Performs the optional L3 storage hit length query when decode-side
|
||||
@@ -79,7 +81,7 @@ class DecodeHiCachePreallocMixin:
|
||||
last_host_node
|
||||
):
|
||||
matched_len = l1_prefix_len + l2_host_hit_length
|
||||
suffix_tokens = req.origin_input_ids[matched_len:]
|
||||
suffix_tokens = req.origin_input_ids[matched_len:max_prefix_len]
|
||||
last_hash = self.tree_cache.get_last_hash_value(last_host_node)
|
||||
prefix_keys = (
|
||||
self.tree_cache.get_prefix_hash_values(last_host_node)
|
||||
@@ -221,6 +223,7 @@ class DecodeHiCacheTransferMixin:
|
||||
dr.req.origin_input_ids,
|
||||
cow_mamba=False,
|
||||
include_req=True,
|
||||
max_prefix_len=pm.decode_prefix_len,
|
||||
)
|
||||
new_indices, restored_node = self.tree_cache.init_load_back(
|
||||
InitLoadBackParams(
|
||||
|
||||
@@ -43,7 +43,7 @@ from sglang.srt.layers.dp_attention import (
|
||||
from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine
|
||||
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
||||
from sglang.srt.mem_cache.utils import get_storage_hash_str
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_memory, get_parallel
|
||||
from sglang.srt.utils import get_device_module
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -842,6 +842,10 @@ class HiCacheController:
|
||||
self._l2_transfers(host_indices, device_indices, pool_transfers)
|
||||
)
|
||||
|
||||
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
|
||||
(id(self), "write"), completion.finish_event
|
||||
)
|
||||
|
||||
self.ack_write_queue.append(
|
||||
HiCacheAck(
|
||||
start_event=completion.start_event,
|
||||
@@ -986,6 +990,10 @@ class HiCacheController:
|
||||
transfer_layer_id_max=self.transfer_layer_id_max,
|
||||
)
|
||||
|
||||
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
|
||||
(id(self), "load"), completion.finish_event
|
||||
)
|
||||
|
||||
self.ack_load_queue.append(
|
||||
HiCacheAck(
|
||||
start_event=completion.start_event,
|
||||
@@ -1108,13 +1116,22 @@ class HiCacheController:
|
||||
# Get one batch token, and update the completed_tokens if succeed
|
||||
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)
|
||||
|
||||
hit_pages = self._page_transfer_kv_batch(
|
||||
operation,
|
||||
batch_hashes,
|
||||
batch_host_indices,
|
||||
extra_info,
|
||||
kv_derived_transfers,
|
||||
)
|
||||
try:
|
||||
hit_pages = self._page_transfer_kv_batch(
|
||||
operation,
|
||||
batch_hashes,
|
||||
batch_host_indices,
|
||||
extra_info,
|
||||
kv_derived_transfers,
|
||||
)
|
||||
except Exception:
|
||||
if not get_memory().enable_unified_memory:
|
||||
raise
|
||||
logger.exception(
|
||||
"HiCache prefetch transfer failed for request %s",
|
||||
operation.request_id,
|
||||
)
|
||||
hit_pages = 0
|
||||
# Check termination
|
||||
if hit_pages != len(batch_hashes):
|
||||
all_success = False
|
||||
@@ -1175,10 +1192,13 @@ class HiCacheController:
|
||||
while not self.storage_stop_event.is_set():
|
||||
try:
|
||||
operation = self.prefetch_buffer.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
except Empty:
|
||||
continue
|
||||
if operation is None:
|
||||
continue
|
||||
try:
|
||||
self._page_transfer(operation)
|
||||
|
||||
finally:
|
||||
self.prefetch_sync_queue.put(
|
||||
PrefetchAck(
|
||||
rid=operation.request_id,
|
||||
@@ -1186,8 +1206,6 @@ class HiCacheController:
|
||||
operation=operation,
|
||||
)
|
||||
)
|
||||
except Empty:
|
||||
continue
|
||||
|
||||
def prefetch_rate_limited(self) -> bool:
|
||||
"""
|
||||
@@ -1209,6 +1227,24 @@ class HiCacheController:
|
||||
# todo: more sophisticated rate limiting based on storage backend performance
|
||||
return False
|
||||
|
||||
def alloc_prefetch_host_buffers(
|
||||
self, operation: StorageOperation, need_size: int
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Allocate the host bounce for a storage hit."""
|
||||
return self.mem_pool_host.alloc(need_size)
|
||||
|
||||
def can_fit_prefetch_host_buffers(
|
||||
self, operation: StorageOperation, need_size: int
|
||||
) -> bool:
|
||||
"""Whether a prefetch bounce can fit when its host pools are empty."""
|
||||
return need_size <= self.mem_pool_host.size
|
||||
|
||||
def free_prefetch_host_buffers(
|
||||
self, operation: StorageOperation, host_indices: torch.Tensor
|
||||
) -> None:
|
||||
"""Roll back a hit-sized host bounce before transfer ownership moves."""
|
||||
self.mem_pool_host.free(host_indices)
|
||||
|
||||
def _storage_hit_query(self, operation) -> tuple[list[str], int]:
|
||||
last_hash = operation.last_hash
|
||||
tokens_to_fetch = operation.token_ids
|
||||
@@ -1243,10 +1279,22 @@ class HiCacheController:
|
||||
operation = self.prefetch_queue.get(block=True, timeout=1)
|
||||
if operation is None:
|
||||
continue
|
||||
if operation.is_terminated():
|
||||
try:
|
||||
if operation.is_terminated():
|
||||
hash_value, storage_hit_count = [], 0
|
||||
else:
|
||||
hash_value, storage_hit_count = self._storage_hit_query(
|
||||
operation
|
||||
)
|
||||
except Exception:
|
||||
if not get_memory().enable_unified_memory:
|
||||
raise
|
||||
logger.exception(
|
||||
"HiCache storage query failed for request %s",
|
||||
operation.request_id,
|
||||
)
|
||||
hash_value, storage_hit_count = [], 0
|
||||
else:
|
||||
hash_value, storage_hit_count = self._storage_hit_query(operation)
|
||||
|
||||
storage_hit_count_tensor = torch.tensor(
|
||||
storage_hit_count, dtype=torch.int
|
||||
)
|
||||
|
||||
@@ -33,7 +33,7 @@ logger = logging.getLogger(__name__)
|
||||
import os
|
||||
import random
|
||||
from collections import Counter
|
||||
from contextlib import contextmanager
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from functools import lru_cache
|
||||
@@ -68,6 +68,11 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
zero_match_result,
|
||||
)
|
||||
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
|
||||
from sglang.srt.mem_cache.unified_cache.components import (
|
||||
CacheTransferPhase,
|
||||
ComponentType,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
@@ -161,6 +166,7 @@ def match_prefix_for_req(
|
||||
*,
|
||||
cow_mamba: bool = False,
|
||||
include_req: bool = False,
|
||||
max_prefix_len: Optional[int] = None,
|
||||
):
|
||||
if token_ids is None:
|
||||
token_ids = req.origin_input_ids + req.output_ids
|
||||
@@ -171,6 +177,10 @@ def match_prefix_for_req(
|
||||
# this request's SWA ring. No-op for other layouts.
|
||||
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
|
||||
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
|
||||
if max_prefix_len is not None:
|
||||
key_limit = (
|
||||
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
|
||||
)
|
||||
|
||||
match_result = tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
@@ -1188,14 +1198,23 @@ class PrefillAdder:
|
||||
return req if truncated else None
|
||||
|
||||
@contextmanager
|
||||
def _lock_node(self, last_node: TreeNode):
|
||||
# Replay the acquire's receipt (SWA boundary uuid, mamba flag) so the
|
||||
# release takes back exactly what this temporary lock took.
|
||||
dec_lock_params = self.tree_cache.inc_lock_ref(last_node).to_dec_params()
|
||||
def _lock_node(self, last_node: TreeNode, *, lock_host: bool = False):
|
||||
host_lock_params = (
|
||||
self.tree_cache.inc_host_lock_ref(last_node).to_dec_params()
|
||||
if lock_host
|
||||
else None
|
||||
)
|
||||
try:
|
||||
yield None
|
||||
# Replay the acquire's receipt (SWA boundary uuid, mamba flag) so the
|
||||
# release takes back exactly what this temporary lock took.
|
||||
dec_lock_params = self.tree_cache.inc_lock_ref(last_node).to_dec_params()
|
||||
try:
|
||||
yield None
|
||||
finally:
|
||||
self.tree_cache.dec_lock_ref(last_node, dec_lock_params)
|
||||
finally:
|
||||
self.tree_cache.dec_lock_ref(last_node, dec_lock_params)
|
||||
if host_lock_params is not None:
|
||||
self.tree_cache.dec_host_lock_ref(last_node, host_lock_params)
|
||||
|
||||
def add_one_req_ignore_eos(self, req: Req):
|
||||
cand_extend_input_len = len(req.full_untruncated_fill_ids) - len(
|
||||
@@ -1398,14 +1417,55 @@ class PrefillAdder:
|
||||
return AddReqResult.OTHER
|
||||
|
||||
if req.needs_host_load_back():
|
||||
promised_host_hit = req.host_hit_length
|
||||
loaded = self.tree_cache.init_load_back(
|
||||
InitLoadBackParams(
|
||||
best_match_node=req.best_match_node,
|
||||
host_hit_length=req.host_hit_length,
|
||||
req=req,
|
||||
load_max_new = min(max_new, admission.max_new_tokens)
|
||||
# Reclaim can write back device victims and evict host leaves.
|
||||
# Pin the selected host/aux match until load-back owns its locks.
|
||||
with (
|
||||
self._lock_node(req.best_match_node, lock_host=True)
|
||||
if isinstance(self.tree_cache, UnifiedRadixCache)
|
||||
else nullcontext()
|
||||
):
|
||||
full_load_tokens = req.host_hit_length
|
||||
if (
|
||||
isinstance(self.tree_cache, UnifiedRadixCache)
|
||||
and self.tree_cache.buffer_pipeline is None
|
||||
and not (
|
||||
self.tree_cache.linker is not None
|
||||
and self.tree_cache.linker.has_hit(req.rid)
|
||||
)
|
||||
):
|
||||
# Host hits can include resident FULL behind host-only aux.
|
||||
# Reuse the FULL transfer spec to count only new slots.
|
||||
full_transfer = (
|
||||
self.tree_cache.tree_core.build_hicache_transfers(
|
||||
ComponentType.FULL,
|
||||
req.best_match_node,
|
||||
CacheTransferPhase.LOAD_BACK,
|
||||
)[0]
|
||||
)
|
||||
full_load_tokens = len(full_transfer.host_indices)
|
||||
if not self.memory_budget.prepare_load_back(
|
||||
full_tokens=(
|
||||
full_load_tokens
|
||||
+ admission.extend_len
|
||||
+ load_max_new
|
||||
+ self.page_size
|
||||
+ mamba_gap_reserve
|
||||
),
|
||||
extend_input_len=admission.extend_len,
|
||||
max_new_tokens=load_max_new,
|
||||
swa_host_hit_length=req.swa_host_hit_length,
|
||||
chunk_limit=self.rem_chunk_tokens,
|
||||
):
|
||||
return AddReqResult.NO_TOKEN
|
||||
promised_host_hit = req.host_hit_length
|
||||
loaded = self.tree_cache.init_load_back(
|
||||
InitLoadBackParams(
|
||||
best_match_node=req.best_match_node,
|
||||
host_hit_length=req.host_hit_length,
|
||||
req=req,
|
||||
)
|
||||
)
|
||||
)
|
||||
if loaded is None:
|
||||
return AddReqResult.OTHER
|
||||
new_indices, req.last_node = loaded
|
||||
|
||||
@@ -16,7 +16,7 @@ limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import abc
|
||||
from typing import TYPE_CHECKING, Protocol
|
||||
from typing import TYPE_CHECKING, Hashable, Protocol
|
||||
|
||||
import torch
|
||||
|
||||
@@ -228,6 +228,10 @@ class BaseTokenToKVPoolAllocator(abc.ABC):
|
||||
):
|
||||
raise NotImplementedError()
|
||||
|
||||
def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
|
||||
"""Record an asynchronous HiCache transfer completion event if needed."""
|
||||
return
|
||||
|
||||
def alloc_extend(self, *args, **kwargs):
|
||||
raise NotImplementedError("alloc_extend is only for paged allocator")
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import math
|
||||
from abc import abstractmethod
|
||||
from typing import Callable, List, Optional, Sequence, Tuple
|
||||
from typing import Callable, Hashable, List, Optional, Sequence, Tuple
|
||||
|
||||
import torch
|
||||
from torch.profiler import record_function
|
||||
@@ -502,11 +502,11 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
The static composite allocates the two sides independently and records
|
||||
a full->swa index mapping. That is not representable here: the two
|
||||
sides SHARE one virtual id space (a virtual page names a full-physical
|
||||
page and, if bound, a swa-physical one), which is why
|
||||
`set_full_to_swa_mapping` is a no-op on this allocator and
|
||||
`translate_loc_from_full_to_swa` derives the swa id from the virtual id
|
||||
instead of a table. Running the static body would call `alloc_extend`
|
||||
on the swa sub-allocator, which asserts it is not the id owner.
|
||||
page and, if bound, a swa-physical one). Load-back installs that binding
|
||||
through `set_full_to_swa_mapping`, and `translate_loc_from_full_to_swa`
|
||||
resolves the SWA kernel-facing id from the shared virtual id. Running the
|
||||
static body would call `alloc_extend` on the swa sub-allocator, which
|
||||
asserts it is not the id owner.
|
||||
|
||||
The tail is expressed by binding swa for the TAIL's virtual pages only.
|
||||
A new page left unbound has no swa-physical page, which reads as the
|
||||
@@ -678,8 +678,12 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
def set_full_to_swa_mapping(
|
||||
self, full_indices: torch.Tensor, swa_indices: torch.Tensor
|
||||
) -> None:
|
||||
"""Binding load-back rows already updates the shared SWA v2p mapping."""
|
||||
return
|
||||
if full_indices.numel() == 0:
|
||||
return
|
||||
assert full_indices.numel() == swa_indices.numel()
|
||||
full_pages = full_indices.to(torch.int64) // self.page_size
|
||||
swa_pages = swa_indices.to(torch.int64) // self.page_size
|
||||
self.swa_attn_allocator.bind(full_pages, swa_pages)
|
||||
|
||||
def clear_full_to_swa_mapping(self, full_indices: torch.Tensor) -> None:
|
||||
# Paired with set_full_to_swa_mapping: shared mode has no mapping tensor.
|
||||
@@ -797,6 +801,10 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
@abstractmethod
|
||||
def _ask_float_for_room(self, need_tokens: int) -> None: ...
|
||||
|
||||
def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
|
||||
self.full_attn_allocator.set_hicache_transfer_done_event(transfer_key, event)
|
||||
self.swa_attn_allocator.set_hicache_transfer_done_event(transfer_key, event)
|
||||
|
||||
|
||||
class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
"""Two-ended FULL/SWA allocator with asymmetric shared-byte reservations."""
|
||||
@@ -1204,7 +1212,10 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
return
|
||||
full_reclaim, swa_reclaim = reclaim_plan
|
||||
if full_reclaim or swa_reclaim:
|
||||
tree_cache.evict_for_alloc(
|
||||
# The shared-byte plan returns cumulative eviction quotas.
|
||||
# Per-component capacity targets can count the same shared bytes
|
||||
# independently and stop before the joint allocation fits.
|
||||
tree_cache.evict(
|
||||
EvictParams(num_tokens=full_reclaim, swa_num_tokens=swa_reclaim)
|
||||
)
|
||||
# A zero-reclaim plan can still depend on compaction before allocation.
|
||||
|
||||
@@ -17,7 +17,7 @@ mamba-state end pools of one `UnifiedKVPool`."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Callable, List, Optional, Sequence
|
||||
from typing import Callable, Hashable, List, Optional, Sequence
|
||||
|
||||
import torch
|
||||
from torch.profiler import record_function
|
||||
@@ -443,3 +443,7 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
):
|
||||
return 0
|
||||
return fa.flush_opportunistic() + ma.flush_opportunistic()
|
||||
|
||||
def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
|
||||
self.full_attn_allocator.set_hicache_transfer_done_event(transfer_key, event)
|
||||
self.mamba_allocator.set_hicache_transfer_done_event(transfer_key, event)
|
||||
|
||||
@@ -29,6 +29,7 @@ from typing import (
|
||||
Callable,
|
||||
Dict,
|
||||
Generic,
|
||||
Hashable,
|
||||
List,
|
||||
Optional,
|
||||
Sequence,
|
||||
@@ -419,6 +420,12 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
# write-race check. Single slot: at most ONE forward in flight per call
|
||||
# site; `_flush` materializes the write-set lazily, avoiding a sync here.
|
||||
self._inflight_forward: Optional[Tuple[torch.cuda.Event, torch.Tensor]] = None
|
||||
self._hicache_transfer_done_events: Dict[Hashable, torch.cuda.Event] = {}
|
||||
# HiCache reserves SWA physical pages during load-back, then binds them
|
||||
# into the tree before the H2D copy is submitted. Compaction must not
|
||||
# move those pages in that interval because the queued PoolTransfer still
|
||||
# carries the original physical ids.
|
||||
self._pending_hicache_load_pages = 0
|
||||
|
||||
# Per-call move cap on NON-urgent `_flush`: bounds work per `on_idle()` so
|
||||
# a large backlog doesn't block ZMQ IPC. Urgent retries are uncapped.
|
||||
@@ -521,6 +528,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self.live_page_count = 0
|
||||
self._inflight_forward = None
|
||||
self._latest_forward_done_event = None
|
||||
self._hicache_transfer_done_events.clear()
|
||||
self._pending_hicache_load_pages = 0
|
||||
|
||||
def clear_inverse_history(self) -> None:
|
||||
self._inverse_history.clear()
|
||||
@@ -726,6 +735,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
|
||||
def moves_blocked(self) -> bool:
|
||||
"""Whether any installed gate currently forbids relocating pages."""
|
||||
if self._pending_hicache_load_pages > 0:
|
||||
# Physical H2D reservations must stay put even before queueing.
|
||||
return True
|
||||
for gate in (self.disagg_move_gate, self.host_transfer_move_gate):
|
||||
if gate is not None and not gate():
|
||||
return True
|
||||
@@ -926,6 +938,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
with record_function("MultiEndedAlloc._alloc_bind_fast_or_slow"):
|
||||
if N == 0:
|
||||
return torch.empty(0, dtype=torch.int64, device=self.device)
|
||||
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
|
||||
self._wait_hicache_transfers()
|
||||
|
||||
# FAST PATH: eager, or lazy with no current holes.
|
||||
if not self.lazy_compaction or self._free_phys_pages.numel() == 0:
|
||||
@@ -1082,19 +1096,15 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
if not _relieve_for_alloc(self, need_size):
|
||||
return None
|
||||
num_pages = need_size // self.page_size
|
||||
if num_pages > len(self.free_virtual_ids):
|
||||
return None
|
||||
v_pages = self.free_virtual_ids[:num_pages]
|
||||
self.free_virtual_ids = self.free_virtual_ids[num_pages:]
|
||||
phys_pages = self._alloc_bind_fast_or_slow(v_pages, num_pages)
|
||||
if phys_pages is None:
|
||||
self.free_virtual_ids = torch.cat([v_pages, self.free_virtual_ids])
|
||||
return None
|
||||
if self.page_size == 1:
|
||||
return v_pages # v_pages already IS the token id list
|
||||
# Expand page ids to token ids: (P, 1) * S + (S,) -> (P, S) -> (P*S,).
|
||||
return (
|
||||
v_pages[:, None] * self.page_size
|
||||
+ torch.arange(self.page_size, device=self.device)
|
||||
).reshape(-1)
|
||||
return self._expand_pages_to_tokens(v_pages)
|
||||
|
||||
def alloc_with_virtual(self, virtual_pages: torch.Tensor) -> None:
|
||||
"""Take physical PAGES for caller-supplied virtual PAGE ids (not token
|
||||
@@ -1264,6 +1274,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self._free_lazy(free_index, pages=_pages)
|
||||
return
|
||||
# --- EAGER path ---
|
||||
self._wait_hicache_transfers()
|
||||
# Near-no-op in normal mode (sampling's CPU sync already drained
|
||||
# forward_stream); in overlap mode it does the serializing.
|
||||
if self.forward_stream is not None:
|
||||
@@ -1713,6 +1724,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
if self.moves_blocked():
|
||||
# Holes stay in the free list; the next flush picks them up.
|
||||
return 0
|
||||
self._wait_hicache_transfers()
|
||||
self._stats_n_flush_calls += 1
|
||||
with record_function("MultiEndedAlloc._flush"):
|
||||
self._drain_pending_reuse(urgent=urgent)
|
||||
@@ -1965,6 +1977,92 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self.size = self.max_slots
|
||||
self.clear()
|
||||
|
||||
_pending_hicache_load_pages: _CapacityField[int] = _CapacityField()
|
||||
|
||||
def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
|
||||
self._hicache_transfer_done_events[transfer_key] = event
|
||||
if transfer_key == "load" or (
|
||||
isinstance(transfer_key, tuple) and transfer_key[-1] == "load"
|
||||
):
|
||||
# The H2D copy is now ordered before this event. Future compaction
|
||||
# may proceed after _wait_hicache_transfers() waits on it.
|
||||
self._pending_hicache_load_pages = 0
|
||||
|
||||
def _wait_hicache_transfers(self) -> None:
|
||||
if not self._hicache_transfer_done_events:
|
||||
return
|
||||
current_stream = torch.cuda.current_stream()
|
||||
events = tuple(self._hicache_transfer_done_events.values())
|
||||
self._hicache_transfer_done_events.clear()
|
||||
for event in events:
|
||||
current_stream.wait_event(event)
|
||||
|
||||
def _expand_pages_to_tokens(self, pages: torch.Tensor) -> torch.Tensor:
|
||||
if self.page_size == 1:
|
||||
return pages
|
||||
return (
|
||||
pages[:, None] * self.page_size
|
||||
+ torch.arange(self.page_size, device=self.device)
|
||||
).reshape(-1)
|
||||
|
||||
def alloc_physical(self, need_size: int) -> Optional[torch.Tensor]:
|
||||
"""Reserve physical token slots without assigning virtual page ids."""
|
||||
if need_size <= 0:
|
||||
return torch.empty(0, dtype=torch.int64, device=self.device)
|
||||
assert need_size % self.page_size == 0, (
|
||||
f"MultiEndedAllocator({self.sub_pool_name!r}).alloc_physical: need_size="
|
||||
f"{need_size} must be a multiple of page_size={self.page_size}"
|
||||
)
|
||||
if need_size > self.available_size():
|
||||
if not _relieve_for_alloc(self, need_size):
|
||||
return None
|
||||
if self.lazy_compaction and self._free_phys_pages.numel() > 0:
|
||||
self._wait_hicache_transfers()
|
||||
physical_pages = self.take_physical_pages(need_size // self.page_size)
|
||||
if physical_pages is None:
|
||||
return None
|
||||
if self.lazy_compaction:
|
||||
self._pending_hicache_load_pages += int(physical_pages.shape[0])
|
||||
return self._expand_pages_to_tokens(physical_pages)
|
||||
|
||||
def cancel_physical_reservation(self, free_index: torch.Tensor) -> None:
|
||||
"""Roll back a HiCache physical allocation before its H2D is submitted."""
|
||||
if free_index is None or free_index.numel() == 0:
|
||||
return
|
||||
if self.lazy_compaction:
|
||||
num_pages = free_index.numel() // self.page_size
|
||||
assert num_pages <= self._pending_hicache_load_pages, (
|
||||
f"MultiEndedAllocator({self.sub_pool_name!r}) released {num_pages} "
|
||||
f"HiCache pages with only {self._pending_hicache_load_pages} pending"
|
||||
)
|
||||
self._pending_hicache_load_pages -= num_pages
|
||||
self.free_physical(free_index)
|
||||
|
||||
def free_physical(self, free_index: torch.Tensor) -> None:
|
||||
"""Release physical token slots reserved by :meth:`alloc_physical`."""
|
||||
if free_index is None or free_index.numel() == 0:
|
||||
return
|
||||
assert free_index.numel() % self.page_size == 0, (
|
||||
f"MultiEndedAllocator({self.sub_pool_name!r}).free_physical: "
|
||||
f"{free_index.numel()} tokens must contain whole pages of "
|
||||
f"size {self.page_size}"
|
||||
)
|
||||
physical_pages = (
|
||||
free_index.detach().to(torch.int64)[:: self.page_size] // self.page_size
|
||||
)
|
||||
self._wait_hicache_transfers()
|
||||
if self.forward_stream is not None and not self.lazy_compaction:
|
||||
torch.cuda.current_stream().wait_stream(self.forward_stream)
|
||||
virtual_pages = self.physical_to_virtual[physical_pages]
|
||||
bound_mask = virtual_pages >= 0
|
||||
self.virtual_to_physical.index_fill_(0, virtual_pages[bound_mask], -1)
|
||||
self.physical_to_virtual.index_fill_(0, physical_pages, -1)
|
||||
if self.lazy_compaction:
|
||||
self._free_phys_pages = torch.cat([self._free_phys_pages, physical_pages])
|
||||
self.live_page_count -= int(physical_pages.shape[0])
|
||||
return
|
||||
self._compact_pending(physical_pages)
|
||||
|
||||
|
||||
def _chain_byte_accounting_violations(
|
||||
chain: List[MultiEndedAllocator],
|
||||
@@ -2258,6 +2356,23 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
|
||||
self._holes_dirty = True
|
||||
self._park_if_empty()
|
||||
|
||||
def free_physical(self, free_index: torch.Tensor) -> None:
|
||||
"""Release physical reservations using the float's hole/span bookkeeping."""
|
||||
if free_index is None or free_index.numel() == 0:
|
||||
return
|
||||
assert free_index.numel() % self.page_size == 0
|
||||
physical_pages = (
|
||||
free_index.detach().to(torch.int64)[:: self.page_size] // self.page_size
|
||||
)
|
||||
self._wait_hicache_transfers()
|
||||
virtual_pages = self.physical_to_virtual[physical_pages]
|
||||
bound_pages = virtual_pages[virtual_pages >= 0]
|
||||
self.virtual_to_physical.index_fill_(0, bound_pages, -1)
|
||||
self.physical_to_virtual.index_fill_(0, physical_pages, -1)
|
||||
self._free_phys_pages = torch.cat([self._free_phys_pages, physical_pages])
|
||||
self._holes_dirty = True
|
||||
self._park_if_empty()
|
||||
|
||||
def _park_if_empty(self) -> bool:
|
||||
"""Reset the span and go frontier-transparent once no live page
|
||||
remains. Sync-free: `_live_pages()` is span minus hole COUNT, both
|
||||
|
||||
@@ -28,6 +28,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
from array import array
|
||||
from collections import deque
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import msgspec
|
||||
@@ -49,6 +50,7 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolTransfer,
|
||||
SidecarPoolSpec,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.base import HostKVCache
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.storage_prefetch import StagedPrefetchPlan
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import RebuildFullToSWAMapping
|
||||
@@ -71,6 +73,13 @@ if TYPE_CHECKING:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _minimum_full_transfer_tokens(
|
||||
host_pool: HostKVCache, prefetch_threshold: int
|
||||
) -> int:
|
||||
page_size = host_pool.page_size
|
||||
return max(page_size, -(-prefetch_threshold // page_size) * page_size)
|
||||
|
||||
|
||||
class _UnifiedBackupIntent(msgspec.Struct):
|
||||
"""Buffer-mode backup intent, unpinned while queued.
|
||||
|
||||
@@ -95,6 +104,7 @@ class _UnifiedBufferBackupEntry(msgspec.Struct):
|
||||
host_indices: torch.Tensor
|
||||
aux_xfers: list[PoolTransfer]
|
||||
lock_params: DecLockRefParams
|
||||
occupied_units: int
|
||||
|
||||
|
||||
class _StagedPrefetch(msgspec.Struct):
|
||||
@@ -159,6 +169,7 @@ def validate_buffer_only_stack(
|
||||
sidecar_pool_specs: list[SidecarPoolSpec],
|
||||
host_pool_group: HostPoolGroup,
|
||||
swa_component: Optional[SWAComponent],
|
||||
storage_prefetch_threshold: int = 256,
|
||||
) -> None:
|
||||
"""Post-assembly buffer-mode fences.
|
||||
|
||||
@@ -201,12 +212,40 @@ def validate_buffer_only_stack(
|
||||
# window), so every window-carrying intent would be dropped as
|
||||
# oversize and SWA storage coverage would silently be zero.
|
||||
window_tokens = swa.full_window_pages * swa._swa_kv_pool_host.page_size
|
||||
if swa._swa_kv_pool_host.size < 2 * window_tokens:
|
||||
shared_domain = (
|
||||
swa._swa_kv_pool_host.shared_allocation_domain
|
||||
if isinstance(swa._swa_kv_pool_host, HostKVCache)
|
||||
else None
|
||||
)
|
||||
if shared_domain is not None:
|
||||
full_host_pool = (
|
||||
swa.cache.cache_controller.mem_pool_host.anchor_entry.host_pool
|
||||
)
|
||||
min_full_tokens = _minimum_full_transfer_tokens(
|
||||
full_host_pool, storage_prefetch_threshold
|
||||
)
|
||||
one_transfer_bytes = (
|
||||
window_tokens * swa._swa_kv_pool_host.size_per_token
|
||||
+ min_full_tokens * full_host_pool.size_per_token
|
||||
)
|
||||
one_transfer = [
|
||||
(full_host_pool.pool_label, min_full_tokens),
|
||||
(swa._swa_kv_pool_host.pool_label, window_tokens),
|
||||
]
|
||||
enough_capacity = shared_domain.can_fit_many_then(
|
||||
one_transfer, one_transfer, empty=True
|
||||
)
|
||||
capacity = f"{shared_domain.capacity_bytes} shared bytes"
|
||||
requirement = f"{2 * one_transfer_bytes} shared bytes"
|
||||
else:
|
||||
enough_capacity = swa._swa_kv_pool_host.size >= 2 * window_tokens
|
||||
capacity = f"{swa._swa_kv_pool_host.size} SWA tokens"
|
||||
requirement = f"{2 * window_tokens} SWA tokens"
|
||||
if not enough_capacity:
|
||||
raise ValueError(
|
||||
"--hicache-host-memory-mode buffer_only requires an SWA "
|
||||
f"host pool of at least two trailing windows "
|
||||
f"({2 * window_tokens} tokens; got "
|
||||
f"{swa._swa_kv_pool_host.size}): one staging a write "
|
||||
"--hicache-host-memory-mode buffer_only requires a host arena "
|
||||
"large enough for two minimum Full/SWA transfers "
|
||||
f"({requirement}; got {capacity}): one staging a write "
|
||||
"while one stays reserved for prefetch window allocs."
|
||||
)
|
||||
|
||||
@@ -300,8 +339,89 @@ class BufferModePipeline:
|
||||
self.anchor_locked_tokens_ = 0
|
||||
self._anchor_lock_cap_skips = 0
|
||||
|
||||
def _shared_host_domain(self):
|
||||
cc = self._cache.cache_controller
|
||||
anchor = cc.mem_pool_host.anchor_entry.host_pool
|
||||
if not isinstance(anchor, HostKVCache):
|
||||
return None
|
||||
return anchor.shared_allocation_domain
|
||||
|
||||
def _transfer_tokens(self, transfer: PoolTransfer) -> int:
|
||||
if transfer.host_indices is not None:
|
||||
return len(transfer.host_indices)
|
||||
if transfer.keys is None:
|
||||
return 0
|
||||
entry = self._cache.cache_controller.mem_pool_host.entry_map.get(transfer.name)
|
||||
return 0 if entry is None else len(transfer.keys) * entry.host_pool.page_size
|
||||
|
||||
def host_allocation_units(
|
||||
self,
|
||||
host_indices: Optional[torch.Tensor],
|
||||
aux_xfers: Optional[list[PoolTransfer]],
|
||||
) -> int:
|
||||
"""Host usage expressed in anchor-token units for scheduler accounting."""
|
||||
if host_indices is None:
|
||||
return 0
|
||||
return self._host_request_units(len(host_indices), aux_xfers)
|
||||
|
||||
def _shared_host_requests(
|
||||
self, kv_tokens: int, aux_xfers: Optional[list[PoolTransfer]]
|
||||
) -> Optional[list[tuple[str, int]]]:
|
||||
if self._shared_host_domain() is None:
|
||||
return None
|
||||
return [
|
||||
(pool.pool_label, tokens)
|
||||
for pool, tokens in self._host_staging_sizes(kv_tokens, aux_xfers)
|
||||
]
|
||||
|
||||
def _host_staging_sizes(self, kv_tokens, aux_xfers):
|
||||
cc = self._cache.cache_controller
|
||||
yield cc.mem_pool_host.anchor_entry.host_pool, kv_tokens
|
||||
for transfer in aux_xfers or ():
|
||||
if transfer.indices_from_pool is not None:
|
||||
continue
|
||||
entry = cc.mem_pool_host.entry_map.get(transfer.name)
|
||||
if entry is not None:
|
||||
yield entry.host_pool, self._transfer_tokens(transfer)
|
||||
|
||||
def _host_request_units(
|
||||
self, kv_tokens: int, aux_xfers: Optional[list[PoolTransfer]]
|
||||
) -> int:
|
||||
"""Host staging in anchor-token accounting units."""
|
||||
cc = self._cache.cache_controller
|
||||
anchor = cc.mem_pool_host.anchor_entry.host_pool
|
||||
if self._shared_host_domain() is None:
|
||||
return kv_tokens
|
||||
num_bytes = sum(
|
||||
tokens * pool.size_per_token
|
||||
for pool, tokens in self._host_staging_sizes(kv_tokens, aux_xfers)
|
||||
)
|
||||
return (num_bytes + anchor.size_per_token - 1) // anchor.size_per_token
|
||||
|
||||
def _shared_backup_fits(self, requests, *, empty: bool = False) -> bool:
|
||||
return self._shared_host_domain().can_fit_many_then(
|
||||
requests, self._shared_load_reserve_requests(), empty=empty
|
||||
)
|
||||
|
||||
def _shared_load_reserve_requests(self) -> list[tuple[str, int]]:
|
||||
cc = self._cache.cache_controller
|
||||
anchor = cc.mem_pool_host.anchor_entry.host_pool
|
||||
swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA)
|
||||
min_full_tokens = _minimum_full_transfer_tokens(
|
||||
anchor, self._cache.prefetch_threshold
|
||||
)
|
||||
requests = [(anchor.pool_label, min_full_tokens)]
|
||||
if swa_entry is not None and self._swa_window_pages:
|
||||
requests.append(
|
||||
(
|
||||
swa_entry.host_pool.pool_label,
|
||||
self._swa_window_pages * swa_entry.host_pool.page_size,
|
||||
)
|
||||
)
|
||||
return requests
|
||||
|
||||
def is_idle(self) -> bool:
|
||||
"""No pending or in-flight buffer-mode transfer work."""
|
||||
"""No queued, staged, or in-flight buffer-mode work or anchor pins."""
|
||||
return not (
|
||||
self.pending_hit_allocs
|
||||
or self.staged_prefetches
|
||||
@@ -310,6 +430,7 @@ class BufferModePipeline:
|
||||
or self.inflight_backup_node_ids
|
||||
or self.ongoing_write_through
|
||||
or self.ongoing_backup
|
||||
or self.anchor_locks
|
||||
)
|
||||
|
||||
def swa_transient_size(self) -> int:
|
||||
@@ -446,11 +567,18 @@ class BufferModePipeline:
|
||||
capacity (total for KV, total minus the loads-priority margin for aux
|
||||
pools — matching ``_aux_budget_blocked``'s admission ceiling): such an
|
||||
intent could never stage and would wedge the FIFO head."""
|
||||
cc = self._cache.cache_controller
|
||||
if intent_tokens > cc.mem_pool_host.size:
|
||||
return True
|
||||
if aux_xfers is None:
|
||||
aux_xfers = self._build_aux_staging_transfers(node_id, hash_values)
|
||||
cc = self._cache.cache_controller
|
||||
shared_requests = self._shared_host_requests(intent_tokens, aux_xfers)
|
||||
if shared_requests is not None:
|
||||
pool_tokens = cc.mem_pool_host.size
|
||||
max_write_units = pool_tokens - pool_tokens // 10
|
||||
if self._host_request_units(intent_tokens, aux_xfers) > max_write_units:
|
||||
return True
|
||||
return not self._shared_backup_fits(shared_requests, empty=True)
|
||||
if intent_tokens > cc.mem_pool_host.size:
|
||||
return True
|
||||
for t in aux_xfers or ():
|
||||
entry = cc.mem_pool_host.entry_map.get(t.name)
|
||||
if entry is not None and (
|
||||
@@ -540,7 +668,7 @@ class BufferModePipeline:
|
||||
self._log_backup_dropped(intent_tokens)
|
||||
continue
|
||||
if self.write_staged_tokens_ >= live_cap:
|
||||
# Yield to live fetch demand; retry next round.
|
||||
# Wait for current copies to free staging before rebuilding the head.
|
||||
break
|
||||
device_value, comp_xfers = self._cache.tree_core.build_backup_spec(
|
||||
snapshot.node_id
|
||||
@@ -560,6 +688,17 @@ class BufferModePipeline:
|
||||
self.write_backlog_tokens_ -= intent_tokens
|
||||
self._log_backup_dropped(intent_tokens)
|
||||
continue
|
||||
shared_domain = self._shared_host_domain()
|
||||
staging_at_limit = (
|
||||
self.write_staged_tokens_
|
||||
+ self._host_request_units(intent_tokens, sizing_xfers)
|
||||
> live_cap
|
||||
if shared_domain is not None
|
||||
else self.write_staged_tokens_ >= live_cap
|
||||
)
|
||||
if staging_at_limit:
|
||||
# Yield to live fetch demand; retry next round.
|
||||
break
|
||||
if self._aux_budget_blocked(intent, sizing_xfers):
|
||||
# An aux pool lacks staging headroom: yield at the gate
|
||||
# instead of failing the alloc inside cc.write; acks free
|
||||
@@ -609,13 +748,15 @@ class BufferModePipeline:
|
||||
# NOTE: no commit_backup — the node must never appear
|
||||
# host-resident in buffer mode; staging slots live in the entry.
|
||||
lock_params = cache.inc_lock_ref(snapshot.node_id).to_dec_params()
|
||||
occupied_units = self.host_allocation_units(host_indices, aux_xfers)
|
||||
self.ongoing_write_through[snapshot.node_id] = _UnifiedBufferBackupEntry(
|
||||
intent=intent,
|
||||
host_indices=host_indices,
|
||||
aux_xfers=aux_xfers,
|
||||
lock_params=lock_params,
|
||||
occupied_units=occupied_units,
|
||||
)
|
||||
self.write_staged_tokens_ += len(host_indices)
|
||||
self.write_staged_tokens_ += occupied_units
|
||||
self.write_backlog_tokens_ -= len(snapshot.hash_values) * cache.page_size
|
||||
return True
|
||||
|
||||
@@ -636,9 +777,14 @@ class BufferModePipeline:
|
||||
aux = self._build_aux_staging_transfers(
|
||||
snapshot.node_id, snapshot.hash_values
|
||||
)
|
||||
cc = self._cache.cache_controller
|
||||
shared_requests = self._shared_host_requests(
|
||||
len(snapshot.hash_values) * self._cache.page_size, aux
|
||||
)
|
||||
if shared_requests is not None:
|
||||
return not self._shared_backup_fits(shared_requests)
|
||||
if not aux:
|
||||
return False
|
||||
cc = self._cache.cache_controller
|
||||
for t in aux:
|
||||
entry = cc.mem_pool_host.entry_map.get(t.name)
|
||||
if entry is None:
|
||||
@@ -740,7 +886,7 @@ class BufferModePipeline:
|
||||
snapshot = intent.snapshot
|
||||
self._cache.storage_existence_cache.add(PoolName.KV, snapshot.hash_values)
|
||||
self._free_staging_now(entry.host_indices, entry.aux_xfers)
|
||||
self.write_staged_tokens_ -= len(entry.host_indices)
|
||||
self.write_staged_tokens_ -= entry.occupied_units
|
||||
self.inflight_backup_node_ids.discard(snapshot.node_id)
|
||||
_untrack_content_refs(self.inflight_backup_hashes, snapshot.hash_values)
|
||||
|
||||
@@ -968,12 +1114,6 @@ class BufferModePipeline:
|
||||
f.request.rid, f.matched_len + f.num_tokens
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _occupied_span(host_indices) -> int:
|
||||
"""Occupancy units a buffer-mode prefetch holds: granted at
|
||||
hit-alloc, sized to the allocation (0 while still querying)."""
|
||||
return len(host_indices) if host_indices is not None else 0
|
||||
|
||||
def stage_completed_prefetch(
|
||||
self,
|
||||
request: CacheRequestHandle,
|
||||
@@ -1001,6 +1141,9 @@ class BufferModePipeline:
|
||||
for transfer in operation.pool_transfers or ()
|
||||
if transfer.indices_from_pool is not None
|
||||
)
|
||||
occupied_tokens = operation.buffer_host_occupied_units
|
||||
if occupied_tokens is None:
|
||||
occupied_tokens = self.host_allocation_units(host_indices, aux_xfers)
|
||||
|
||||
has_aux = any(
|
||||
t.host_indices is not None and t.host_indices.numel() > 0 for t in aux_xfers
|
||||
@@ -1012,7 +1155,7 @@ class BufferModePipeline:
|
||||
cc.append_host_mem_release(
|
||||
host_indices[:num_tokens], extra_pools=aux_xfers or None
|
||||
)
|
||||
cc.prefetch_tokens_occupied -= self._occupied_span(host_indices)
|
||||
cc.prefetch_tokens_occupied -= occupied_tokens
|
||||
cache.prefetch_loaded_tokens_by_reqid[request] = 0
|
||||
cache.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
return True
|
||||
@@ -1024,8 +1167,6 @@ class BufferModePipeline:
|
||||
# itself is the evidence, so feeding is sound even if this staged
|
||||
# prefetch is later dropped unconsumed.
|
||||
cache.storage_existence_cache.add(PoolName.KV, list(staged_hashes))
|
||||
occupied_tokens = self._occupied_span(host_indices)
|
||||
|
||||
self.staged_prefetches[request] = _StagedPrefetch(
|
||||
request=request,
|
||||
key_tokens=array(
|
||||
@@ -1150,6 +1291,52 @@ class BufferModePipeline:
|
||||
),
|
||||
0,
|
||||
)
|
||||
swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA)
|
||||
binds_swa_to_full = (
|
||||
swa_entry is not None
|
||||
and swa_entry.device_indices_from_anchor_fn is not None
|
||||
)
|
||||
repair_ranges = []
|
||||
if binds_swa_to_full and staged_swa:
|
||||
window_start = span_end - staged_swa
|
||||
repair_end = min(splice_base, span_end)
|
||||
if window_start < repair_end:
|
||||
repair_ranges = cache.tree_core.swa_tombstone_ranges(
|
||||
key, window_start, repair_end
|
||||
)
|
||||
# Only missing SWA rows belong to this load. Existing bindings may
|
||||
# be in use by another request and must survive allocation rollback.
|
||||
anchor_parts = []
|
||||
host_parts = []
|
||||
for repair_start, repair_end_ in repair_ranges:
|
||||
anchor_parts.append(req.prefix_indices[repair_start:repair_end_])
|
||||
host_parts.append(
|
||||
slice(repair_start - window_start, repair_end_ - window_start)
|
||||
)
|
||||
tail_start = max(splice_base, window_start)
|
||||
if tail_start < span_end:
|
||||
anchor_parts.append(
|
||||
slice(tail_start - splice_base, span_end - splice_base)
|
||||
)
|
||||
host_parts.append(
|
||||
slice(tail_start - window_start, span_end - window_start)
|
||||
)
|
||||
for i, transfer in enumerate(load_xfers):
|
||||
if transfer.name != PoolName.SWA:
|
||||
continue
|
||||
# Keep the original complete host bounce for ack/drop release.
|
||||
load_xfers[i] = replace(
|
||||
transfer,
|
||||
host_indices=(
|
||||
torch.cat([transfer.host_indices[part] for part in host_parts])
|
||||
if host_parts
|
||||
else transfer.host_indices[:0]
|
||||
),
|
||||
anchor_index_parts=anchor_parts,
|
||||
)
|
||||
if not anchor_parts:
|
||||
load_xfers = [t for t in load_xfers if t.name != PoolName.SWA]
|
||||
|
||||
device_indices = cc.load(
|
||||
host_indices=f.host_indices[trim_tokens:],
|
||||
node_id=load_back_id,
|
||||
@@ -1177,7 +1364,18 @@ class BufferModePipeline:
|
||||
None,
|
||||
)
|
||||
aux_device_releases: list[tuple[PoolName, torch.Tensor]] = []
|
||||
if swa_dev is not None:
|
||||
if swa_dev is not None and binds_swa_to_full:
|
||||
# Binding already updated the allocator's virtual-to-physical table.
|
||||
# The tree owns virtual FULL rows, not the H2D kernel-facing IDs.
|
||||
for repair_start, repair_end_ in repair_ranges:
|
||||
for action in cache.tree_core.attach_swa_window(
|
||||
key,
|
||||
repair_start,
|
||||
repair_end_,
|
||||
req.prefix_indices[repair_start:repair_end_],
|
||||
):
|
||||
cache._apply_cache_action(action)
|
||||
elif swa_dev is not None:
|
||||
# Register the window's FULL->SWA translation now (attention reads
|
||||
# through it). Keep SWA slots another request may still hold; their
|
||||
# redundant H2D destinations are reclaimed at the transfer ack.
|
||||
@@ -1185,7 +1383,7 @@ class BufferModePipeline:
|
||||
-len(swa_dev) :
|
||||
]
|
||||
allocator = cache.token_to_kv_pool_allocator
|
||||
old_swa = allocator.full_to_swa_index_mapping[full_window.to(torch.int64)]
|
||||
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
|
||||
missing = old_swa <= 0
|
||||
window_start = span_end - len(swa_dev)
|
||||
repair_end = min(splice_base, span_end)
|
||||
@@ -1287,8 +1485,19 @@ class BufferModePipeline:
|
||||
self._free_staging_now(f.host_indices, f.aux_xfers)
|
||||
for pool_name, device_indices in f.aux_device_releases:
|
||||
entry = cc.mem_pool_host.entry_map[pool_name]
|
||||
free_fn = entry.device_free_fn or entry.device_pool.free
|
||||
free_fn(device_indices)
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWAAllocatorBase,
|
||||
)
|
||||
|
||||
allocator = cache.token_to_kv_pool_allocator
|
||||
if pool_name == PoolName.SWA and isinstance(
|
||||
allocator, UnifiedSWAAllocatorBase
|
||||
):
|
||||
# H2D has completed; these redundant slots are no longer pending.
|
||||
allocator.swa_attn_allocator.free_physical(device_indices)
|
||||
else:
|
||||
free_fn = entry.device_free_fn or entry.device_pool.free
|
||||
free_fn(device_indices)
|
||||
|
||||
cc.prefetch_tokens_occupied -= f.occupied_tokens
|
||||
logger.info(
|
||||
|
||||
@@ -39,7 +39,9 @@ from sglang.srt.mem_cache.l2_transfer import L2Transfer
|
||||
from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry
|
||||
from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
|
||||
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.runtime_context import get_memory
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
@@ -163,6 +165,7 @@ class PrefetchOperation(StorageOperation):
|
||||
pool_transfers=pool_transfers,
|
||||
)
|
||||
self.pool_transfers_done = not bool(pool_transfers)
|
||||
self.buffer_host_occupied_units: Optional[int] = None
|
||||
# The Python transfer worker leaves the unfinished tail to the ACK drain;
|
||||
# a controller that releases it itself must set this False.
|
||||
self.ack_releases_incomplete_host_indices = True
|
||||
@@ -277,6 +280,11 @@ class HybridCacheController(BaseHiCacheController):
|
||||
self._stop_pp_prefetch_thread()
|
||||
super()._stop_storage_threads()
|
||||
|
||||
@staticmethod
|
||||
def supports_page_envelope_host(storage_backend: str | None) -> bool:
|
||||
"""Only opt in backends whose FULL object is the complete shared page."""
|
||||
return storage_backend is None or storage_backend == "mori"
|
||||
|
||||
def attach_storage_backend(
|
||||
self,
|
||||
storage_backend: str,
|
||||
@@ -285,6 +293,14 @@ class HybridCacheController(BaseHiCacheController):
|
||||
storage_backend_extra_config: Optional[dict] = None,
|
||||
host_pools: Optional[list[PoolEntry]] = None,
|
||||
):
|
||||
if isinstance(
|
||||
self.storage_host_pool, UnifiedPageEnvelopeHostPool
|
||||
) and not self.supports_page_envelope_host(storage_backend):
|
||||
# A runtime backend switch cannot replace the existing host arena.
|
||||
raise ValueError(
|
||||
f"Storage backend {storage_backend!r} requires separate host pools. "
|
||||
"Restart with this backend selected to create compatible host pools."
|
||||
)
|
||||
enable_pp_ticket = (
|
||||
self.host_memory_mode == "buffer_only"
|
||||
and storage_backend == "mooncake"
|
||||
@@ -572,6 +588,9 @@ class HybridCacheController(BaseHiCacheController):
|
||||
for entry, (_, need_size) in zip(entries, requests, strict=True)
|
||||
]
|
||||
|
||||
# Scheduler drains commit the same logical host allocations and frees
|
||||
# across attention ranks. Layout compaction waits for local I/O before
|
||||
# allocating; transfer readiness does not change the capacity decision.
|
||||
allocated = domain.alloc_many(domain_requests)
|
||||
if allocated is not None:
|
||||
return allocated
|
||||
@@ -615,7 +634,8 @@ class HybridCacheController(BaseHiCacheController):
|
||||
// pool.page_size
|
||||
* pool.page_size
|
||||
)
|
||||
if entry.host_evict_fn(tokens) <= 0:
|
||||
evicted = entry.host_evict_fn(tokens)
|
||||
if evicted <= 0:
|
||||
continue
|
||||
made_progress = True
|
||||
allocated = domain.alloc_many(domain_requests)
|
||||
@@ -665,6 +685,134 @@ class HybridCacheController(BaseHiCacheController):
|
||||
return None
|
||||
return host_indices, resolved
|
||||
|
||||
def _shared_prefetch_requests(
|
||||
self, operation: PrefetchOperation, need_size: int
|
||||
) -> tuple[list[tuple[PoolName, int]], list[PoolTransfer]]:
|
||||
anchor = self.mem_pool_host.anchor_entry
|
||||
pool_transfers = operation.pool_transfers or []
|
||||
requests = [(anchor.name, need_size)]
|
||||
sidecar_hit_pages = (
|
||||
operation.sidecar_hit_pages
|
||||
if operation.sidecar_hash_values is not None
|
||||
else need_size // self.page_size
|
||||
)
|
||||
independent_transfers = []
|
||||
for transfer in pool_transfers:
|
||||
if transfer.indices_from_pool is not None:
|
||||
continue
|
||||
if transfer.host_indices is not None:
|
||||
raise AssertionError(
|
||||
"Shared prefetch pools must be allocated atomically."
|
||||
)
|
||||
entry = self.mem_pool_host.entry_map.get(transfer.name)
|
||||
if entry is None:
|
||||
continue
|
||||
num_pages = len(transfer.keys or [])
|
||||
if transfer.hit_policy == PoolHitPolicy.TRAILING_PAGES:
|
||||
num_pages = min(num_pages or 1, sidecar_hit_pages)
|
||||
requests.append((transfer.name, num_pages * entry.host_pool.page_size))
|
||||
independent_transfers.append(transfer)
|
||||
return requests, independent_transfers
|
||||
|
||||
def can_fit_prefetch_host_buffers(
|
||||
self, operation: StorageOperation, need_size: int, *, empty: bool = True
|
||||
) -> bool:
|
||||
anchor = self.mem_pool_host.anchor_entry
|
||||
if not uses_shared_host_layout(anchor.host_pool):
|
||||
return super().can_fit_prefetch_host_buffers(operation, need_size)
|
||||
domain = anchor.host_pool.shared_allocation_domain
|
||||
requests, _ = self._shared_prefetch_requests(operation, need_size)
|
||||
domain_requests = [
|
||||
(self.mem_pool_host.entry_map[name].host_pool.pool_label, size)
|
||||
for name, size in requests
|
||||
]
|
||||
return domain.can_fit_many_then(domain_requests, (), empty=empty)
|
||||
|
||||
def prefetch_rate_limited(self) -> bool:
|
||||
if self.host_memory_mode == "buffer_only" and uses_shared_host_layout(
|
||||
self.mem_pool_host.anchor_entry.host_pool
|
||||
):
|
||||
# Shared-arena occupancy includes the Full and SWA byte footprint.
|
||||
return self.prefetch_tokens_occupied >= self.prefetch_capacity_limit
|
||||
return super().prefetch_rate_limited()
|
||||
|
||||
def alloc_prefetch_host_buffers(
|
||||
self, operation: StorageOperation, need_size: int
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Atomically allocate every prefetch slice from a shared arena."""
|
||||
anchor = self.mem_pool_host.anchor_entry
|
||||
if not uses_shared_host_layout(anchor.host_pool):
|
||||
return super().alloc_prefetch_host_buffers(operation, need_size)
|
||||
|
||||
requests, independent_transfers = self._shared_prefetch_requests(
|
||||
operation, need_size
|
||||
)
|
||||
|
||||
allocated = self._alloc_shared_host_requests_with_reclaim(requests)
|
||||
if allocated is None:
|
||||
return None
|
||||
self._sync_trailing_keys(
|
||||
operation.pool_transfers or [],
|
||||
operation.sidecar_hash_values or operation.hash_value,
|
||||
(
|
||||
operation.sidecar_hit_pages
|
||||
if operation.sidecar_hash_values is not None
|
||||
else need_size // self.page_size
|
||||
),
|
||||
)
|
||||
for transfer, indices in zip(independent_transfers, allocated[1:], strict=True):
|
||||
transfer.host_indices = indices
|
||||
return allocated[0]
|
||||
|
||||
def allocate_storage_hit(
|
||||
self,
|
||||
operation: StorageOperation,
|
||||
hit_tokens: int,
|
||||
*,
|
||||
allow_partial: bool,
|
||||
min_tokens: int,
|
||||
evict_host: Callable[[int], int],
|
||||
) -> tuple[Optional[torch.Tensor], int]:
|
||||
host_indices = self.alloc_prefetch_host_buffers(operation, hit_tokens)
|
||||
shared = uses_shared_host_layout(self.mem_pool_host.anchor_entry.host_pool)
|
||||
if host_indices is None and not shared:
|
||||
evict_host(hit_tokens)
|
||||
host_indices = self.alloc_prefetch_host_buffers(operation, hit_tokens)
|
||||
if host_indices is not None or not allow_partial:
|
||||
return host_indices, hit_tokens
|
||||
|
||||
if shared:
|
||||
low, high = 0, hit_tokens // self.page_size
|
||||
while low < high:
|
||||
mid = (low + high + 1) // 2
|
||||
if self.can_fit_prefetch_host_buffers(
|
||||
operation, mid * self.page_size, empty=False
|
||||
):
|
||||
low = mid
|
||||
else:
|
||||
high = mid - 1
|
||||
alloc_len = low * self.page_size
|
||||
else:
|
||||
available = self.mem_pool_host.available_size()
|
||||
alloc_len = min(hit_tokens, available - available % self.page_size)
|
||||
if alloc_len >= min_tokens:
|
||||
host_indices = self.alloc_prefetch_host_buffers(operation, alloc_len)
|
||||
return host_indices, alloc_len
|
||||
|
||||
def free_prefetch_host_buffers(
|
||||
self, operation: StorageOperation, host_indices: torch.Tensor
|
||||
) -> None:
|
||||
"""Roll back an unsubmitted prefetch's anchor and independent pools."""
|
||||
pool_transfers = operation.pool_transfers or []
|
||||
self.mem_pool_host.free(host_indices)
|
||||
for transfer in pool_transfers:
|
||||
if transfer.indices_from_pool is not None or transfer.host_indices is None:
|
||||
continue
|
||||
entry = self.mem_pool_host.entry_map.get(transfer.name)
|
||||
if entry is not None:
|
||||
entry.host_pool.free(transfer.host_indices)
|
||||
transfer.host_indices = None
|
||||
|
||||
def _move_op_indices(
|
||||
self, op: CacheOperation
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]:
|
||||
@@ -1308,16 +1456,26 @@ class HybridCacheController(BaseHiCacheController):
|
||||
)
|
||||
self._sync_trailing_keys(transfers_nonkv, sidecar_hashes, sidecar_hit_pages)
|
||||
self._resolve_sidecar_nonkv_derived_pool_transfers(operation)
|
||||
results = {}
|
||||
for extra_info, transfers in _trailing_chain_groups(
|
||||
operation.prefix_keys,
|
||||
sidecar_hashes,
|
||||
transfers_nonkv,
|
||||
):
|
||||
results.update(
|
||||
self.storage_backend.batch_get_v2(transfers, extra_info=extra_info)
|
||||
try:
|
||||
results = {}
|
||||
for extra_info, transfers in _trailing_chain_groups(
|
||||
operation.prefix_keys,
|
||||
sidecar_hashes,
|
||||
transfers_nonkv,
|
||||
):
|
||||
results.update(
|
||||
self.storage_backend.batch_get_v2(
|
||||
transfers, extra_info=extra_info
|
||||
)
|
||||
)
|
||||
pool_hits = count_pool_hits(results)
|
||||
except Exception:
|
||||
if not get_memory().enable_unified_memory:
|
||||
raise
|
||||
logger.exception(
|
||||
"HiCache sidecar prefetch failed for request %s",
|
||||
operation.request_id,
|
||||
)
|
||||
pool_hits = count_pool_hits(results)
|
||||
# Emit PrefetchAck to prefetch_sync_queue, even the operation has been canceled by the
|
||||
# scheduler thread. The prefetch sync thread expects the same number of PrefetchAck objects
|
||||
# to perform all_reduce.
|
||||
@@ -1521,7 +1679,7 @@ class HybridCacheController(BaseHiCacheController):
|
||||
anchor_transfers = []
|
||||
|
||||
def rollback_allocated() -> None:
|
||||
for prev_pool, prev_free_fn, prev_indices in newly_allocated:
|
||||
for prev_pool, prev_free_fn, prev_indices in reversed(newly_allocated):
|
||||
prev_free_fn(prev_indices)
|
||||
prev_pool.device_indices = None
|
||||
|
||||
|
||||
@@ -273,6 +273,13 @@ def build_kv_only_group(
|
||||
|
||||
def _swa_allocation_callbacks(allocator, bind=None, free_bound=None) -> dict:
|
||||
"""Keep allocation and rollback in the same ID space for every SWA stack."""
|
||||
from sglang.srt.mem_cache.allocator.unified_sub_pool import MultiEndedAllocator
|
||||
|
||||
if isinstance(allocator, MultiEndedAllocator):
|
||||
return dict(
|
||||
device_alloc_fn=allocator.alloc_physical,
|
||||
device_free_fn=allocator.cancel_physical_reservation,
|
||||
)
|
||||
if bind is not None:
|
||||
assert free_bound is not None
|
||||
return dict(
|
||||
@@ -300,7 +307,9 @@ def _uses_unified_page_envelope_host(
|
||||
for pool in (full_kv_pool, swa_kv_pool)
|
||||
)
|
||||
and {full_kv_pool.grow_direction, swa_kv_pool.grow_direction} == {"up", "down"}
|
||||
and get_memory().hicache_host_memory_mode != "buffer_only"
|
||||
and HybridCacheController.supports_page_envelope_host(
|
||||
get_memory().hicache_storage_backend
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -176,19 +176,6 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
|
||||
|
||||
backend = disagg.disaggregation_decode_retraction_backup
|
||||
unified_hybrid_swa = memory.enable_unified_memory and tp_worker.is_hybrid_swa
|
||||
unified_decode_radix = (
|
||||
memory.enable_unified_memory and disagg.disaggregation_decode_enable_radix_cache
|
||||
)
|
||||
if backend == "host_pool" and unified_decode_radix:
|
||||
raise ValueError(
|
||||
"Unified-memory H2D/D2H does not support host-pool decode "
|
||||
"retraction with decode radix cache yet."
|
||||
)
|
||||
if backend == "host_pool" and unified_hybrid_swa:
|
||||
raise ValueError(
|
||||
"Unified-memory hybrid-SWA H2D/D2H does not support host-pool "
|
||||
"decode retraction yet."
|
||||
)
|
||||
if backend is None:
|
||||
kv_cache = tp_worker.get_memory_pool()[1].get_kvcache()
|
||||
full_tokens_per_layer = (
|
||||
|
||||
@@ -2174,6 +2174,7 @@ class KVCacheConfigurator:
|
||||
swa_allocator = token_to_kv_pool_allocator.logical_attn_allocator
|
||||
else:
|
||||
swa_allocator = token_to_kv_pool_allocator
|
||||
assert isinstance(swa_allocator, SWATokenToKVPoolAllocator)
|
||||
uses_unified_virtual_ids = isinstance(
|
||||
swa_allocator, UnifiedSWAAllocatorBase
|
||||
)
|
||||
@@ -2194,7 +2195,6 @@ class KVCacheConfigurator:
|
||||
identity_mapping[-1] = -1
|
||||
token_to_kv_pool.register_mapping(identity_mapping)
|
||||
elif not uses_unified_virtual_ids:
|
||||
assert isinstance(swa_allocator, SWATokenToKVPoolAllocator)
|
||||
token_to_kv_pool.register_mapping(
|
||||
swa_allocator.full_to_swa_index_mapping
|
||||
)
|
||||
|
||||
@@ -64,6 +64,7 @@ class LogicalHostPool:
|
||||
"""
|
||||
|
||||
shared_allocation_domain = None
|
||||
stores_page_envelope = False
|
||||
storage_format_tag = None
|
||||
|
||||
def __init__(self, size: int, page_size: int, layout: str = "layer_first"):
|
||||
|
||||
@@ -164,6 +164,7 @@ class HostKVCache(abc.ABC):
|
||||
dcp_size = 1
|
||||
dcp_rank = 0
|
||||
shared_allocation_domain = None
|
||||
stores_page_envelope = False
|
||||
# Names this pool's page byte format in storage keys when it has one of its
|
||||
# own, so pages persisted in another format miss instead of loading.
|
||||
storage_format_tag: Optional[str] = None
|
||||
|
||||
@@ -291,6 +291,10 @@ class HostPoolGroup:
|
||||
def size_per_token(self):
|
||||
return self.anchor_entry.host_pool.size_per_token
|
||||
|
||||
@property
|
||||
def stores_page_envelope(self) -> bool:
|
||||
return self.anchor_entry.host_pool.stores_page_envelope
|
||||
|
||||
def clear(self) -> None:
|
||||
for entry in self.entries:
|
||||
entry.host_pool.clear()
|
||||
|
||||
@@ -303,15 +303,30 @@ class _SharedPageEnvelopeHostBacking:
|
||||
extension_bytes += extension_pages * side.page_bytes
|
||||
return extension_bytes <= self._current_gap_bytes()
|
||||
|
||||
def _can_fit_packed(self, page_counts: dict[str, int]) -> bool:
|
||||
def _can_fit_state(
|
||||
self,
|
||||
live_page_counts: dict[str, int],
|
||||
free_logical_page_counts: dict[str, int],
|
||||
request_page_counts: dict[str, int],
|
||||
) -> bool:
|
||||
used_bytes = 0
|
||||
for name, page_count in page_counts.items():
|
||||
side = self.sides[name]
|
||||
if page_count > self._logical_free_page_count(side):
|
||||
for name, page_count in request_page_counts.items():
|
||||
if page_count > free_logical_page_counts[name]:
|
||||
return False
|
||||
used_bytes += (side.live_page_count + page_count) * side.page_bytes
|
||||
side = self.sides[name]
|
||||
used_bytes += (live_page_counts[name] + page_count) * side.page_bytes
|
||||
return used_bytes <= self._allocatable_bytes
|
||||
|
||||
def _can_fit_packed(self, page_counts: dict[str, int]) -> bool:
|
||||
return self._can_fit_state(
|
||||
{name: side.live_page_count for name, side in self.sides.items()},
|
||||
{
|
||||
name: self._logical_free_page_count(side)
|
||||
for name, side in self.sides.items()
|
||||
},
|
||||
page_counts,
|
||||
)
|
||||
|
||||
def _begin_compaction(self) -> list:
|
||||
if self._layout_lease_state.depth:
|
||||
raise RuntimeError(
|
||||
@@ -543,6 +558,48 @@ class _SharedPageEnvelopeHostBacking:
|
||||
self.sides[name].free_logical_extents = extents
|
||||
return results
|
||||
|
||||
def can_fit_many_then(
|
||||
self,
|
||||
requests: Sequence[tuple[str, int]],
|
||||
following_requests: Sequence[tuple[str, int]],
|
||||
*,
|
||||
empty: bool = False,
|
||||
) -> bool:
|
||||
"""Whether both allocation groups fit in order without mutating state."""
|
||||
with self.lock:
|
||||
_, first_page_counts = self._request_page_counts(requests)
|
||||
_, following_page_counts = self._request_page_counts(following_requests)
|
||||
if empty:
|
||||
live_page_counts = {name: 0 for name in self.sides}
|
||||
free_logical_page_counts = {
|
||||
name: side.page_num for name, side in self.sides.items()
|
||||
}
|
||||
else:
|
||||
live_page_counts = {
|
||||
name: side.live_page_count for name, side in self.sides.items()
|
||||
}
|
||||
free_logical_page_counts = {
|
||||
name: self._logical_free_page_count(side)
|
||||
for name, side in self.sides.items()
|
||||
}
|
||||
if not self._can_fit_state(
|
||||
live_page_counts, free_logical_page_counts, first_page_counts
|
||||
):
|
||||
return False
|
||||
live_after_first = {
|
||||
name: live_page_counts[name] + first_page_counts[name]
|
||||
for name in self.sides
|
||||
}
|
||||
free_after_first = {
|
||||
name: free_logical_page_counts[name] - first_page_counts[name]
|
||||
for name in self.sides
|
||||
}
|
||||
return self._can_fit_state(
|
||||
live_after_first,
|
||||
free_after_first,
|
||||
following_page_counts,
|
||||
)
|
||||
|
||||
def free(self, name: str, indices: torch.Tensor) -> int:
|
||||
with self.lock:
|
||||
side = self.sides[name]
|
||||
@@ -610,6 +667,7 @@ class _SharedPageEnvelopeHostBacking:
|
||||
class UnifiedPageEnvelopeHostPool(HostKVCache):
|
||||
"""Host mirror that transfers complete unified-memory page envelopes."""
|
||||
|
||||
stores_page_envelope = True
|
||||
storage_format_tag = "unified-token-major"
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -15,7 +15,8 @@
|
||||
The scheduler supplies token demand and its chunk/decode limits. These objects
|
||||
account for admitted but not yet allocated work and query live cache capacity:
|
||||
locking a prefix or preempting a request must affect the next admission check.
|
||||
They neither select requests nor mutate the prefix cache or allocator.
|
||||
Selection checks do not mutate cache state. Shared-pool load preparation
|
||||
realizes the selected reservation before a host transfer pins device rows.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
@@ -72,6 +73,18 @@ class PrefillBudget:
|
||||
def ceil_paged_tokens(self, tokens: int) -> int:
|
||||
return -(-tokens // self.page_size) * self.page_size
|
||||
|
||||
def prepare_load_back(
|
||||
self,
|
||||
*,
|
||||
full_tokens: int,
|
||||
extend_input_len: int,
|
||||
max_new_tokens: int,
|
||||
swa_host_hit_length: int,
|
||||
chunk_limit: int | None,
|
||||
) -> bool:
|
||||
"""Prepare pools whose admission depends on movable shared space."""
|
||||
return True
|
||||
|
||||
def _available_and_evictable(self):
|
||||
evictable = (
|
||||
self.tree_cache.full_evictable_size()
|
||||
@@ -322,6 +335,36 @@ class SharedSWAPrefillBudget(SWAPrefillBudget):
|
||||
require_token_slack=empty_pool,
|
||||
)
|
||||
|
||||
def prepare_load_back(
|
||||
self,
|
||||
*,
|
||||
full_tokens: int,
|
||||
extend_input_len: int,
|
||||
max_new_tokens: int,
|
||||
swa_host_hit_length: int,
|
||||
chunk_limit: int | None,
|
||||
) -> bool:
|
||||
# H2D pins both bands. Realize the selected joint budget while their
|
||||
# holes can still be compacted, including pending prefill/decode demand.
|
||||
full_tokens = int(self.ceil_paged_tokens(full_tokens + self.total_offset))
|
||||
swa_tokens = int(
|
||||
self.ceil_paged_tokens(
|
||||
self.swa_tokens(
|
||||
extend_input_len,
|
||||
max_new_tokens,
|
||||
chunk_limit=chunk_limit,
|
||||
swa_host_hit_length=swa_host_hit_length,
|
||||
)
|
||||
+ self.swa_offset
|
||||
)
|
||||
)
|
||||
return (
|
||||
self.allocator.reclaim_for_prealloc(
|
||||
self.tree_cache, full_tokens, swa_tokens
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
def _joint_chunk_cap(self, *, max_chunk_tokens, chunk_limit, swa_host_hit_length=0):
|
||||
lo, hi = 0, max(0, max_chunk_tokens) // self.page_size
|
||||
while lo < hi:
|
||||
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from array import array
|
||||
from typing import TYPE_CHECKING, Optional, Sequence
|
||||
from typing import TYPE_CHECKING, Callable, Optional, Sequence
|
||||
|
||||
import torch
|
||||
|
||||
@@ -371,6 +371,19 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
|
||||
self._page_size = params.page_size
|
||||
self._swa_backup_index_mapper: Optional[
|
||||
Callable[[torch.Tensor], torch.Tensor]
|
||||
] = None
|
||||
allocator = params.token_to_kv_pool_allocator
|
||||
if allocator is not None and ComponentType.SWA in self.tree_components:
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWAAllocatorBase,
|
||||
)
|
||||
|
||||
if isinstance(allocator, UnifiedSWAAllocatorBase):
|
||||
self._swa_backup_index_mapper = (
|
||||
allocator.translate_swa_indices_for_transfer
|
||||
)
|
||||
self.is_eagle = (
|
||||
params.is_eagle and ComponentType.MAMBA not in self.tree_components
|
||||
)
|
||||
@@ -555,6 +568,11 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
result = EvictDeviceNextNodeResult(
|
||||
node_id=binding_result.node_id,
|
||||
made_progress=binding_result.made_progress,
|
||||
backup_kv=(
|
||||
_cache_action_from_tagged(binding_result.backup_kv)
|
||||
if binding_result.backup_kv is not None
|
||||
else None
|
||||
),
|
||||
unbacked_tokens=binding_result.unbacked_tokens,
|
||||
mamba_backup_node_id=binding_result.mamba_backup_node_id,
|
||||
swa_backup_node_id=binding_result.swa_backup_node_id,
|
||||
@@ -770,6 +788,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def set_hicache_enabled(self) -> None:
|
||||
self._binding.set_hicache_enabled()
|
||||
|
||||
def enable_swa_write_back_eviction_barrier(self) -> None:
|
||||
self._binding.enable_swa_write_back_eviction_barrier()
|
||||
|
||||
def set_host_memory_buffer_only(self) -> None:
|
||||
self._binding.set_host_memory_buffer_only()
|
||||
|
||||
@@ -868,11 +889,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
def _refresh_swa_backup_indices(self, transfers: Sequence[PoolTransfer]) -> None:
|
||||
if not transfers:
|
||||
return
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWAAllocatorBase,
|
||||
)
|
||||
|
||||
if not isinstance(self._allocator, UnifiedSWAAllocatorBase):
|
||||
if self._swa_backup_index_mapper is None:
|
||||
return
|
||||
for transfer in transfers:
|
||||
if transfer.name != PoolName.SWA or not transfer.nodes_to_load:
|
||||
@@ -883,7 +900,7 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
self.get_component_device_value(node_id, ComponentType.FULL)
|
||||
for node_id in transfer.nodes_to_load
|
||||
]
|
||||
transfer.device_indices = self._allocator.translate_loc_from_full_to_swa(
|
||||
transfer.device_indices = self._swa_backup_index_mapper(
|
||||
torch.cat(full_values)
|
||||
).to(torch.int64)
|
||||
|
||||
@@ -972,6 +989,9 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
) -> tuple[Optional[str], Optional[str]]:
|
||||
return self._binding.prefetch_anchor_info(node_id)
|
||||
|
||||
def is_write_through_compatible(self) -> bool:
|
||||
return self._binding.is_write_through_compatible()
|
||||
|
||||
def is_backuped(self, node_id: NodeId) -> bool:
|
||||
return self._binding.node_backuped(node_id)
|
||||
|
||||
|
||||
@@ -917,6 +917,7 @@ class UMBPStore(HiCacheStorage):
|
||||
# Initialize before the optional constructor-time pool registration.
|
||||
self.registered_pools: dict = {}
|
||||
self._kv_anchor_is_logical = False
|
||||
self._kv_anchor_is_page_envelope = False
|
||||
self._registered_regions: set = set()
|
||||
|
||||
self.client = UMBPClient(cfg)
|
||||
@@ -1001,6 +1002,7 @@ class UMBPStore(HiCacheStorage):
|
||||
|
||||
# A logical anchor owns indices; side pools carry the data.
|
||||
self._kv_anchor_is_logical = self.mem_pool_host.kv_buffer is None
|
||||
self._kv_anchor_is_page_envelope = self.mem_pool_host.stores_page_envelope
|
||||
self._zero_copy_registered = False
|
||||
|
||||
# Side-pool registration needs the mode even for a logical anchor.
|
||||
@@ -1163,68 +1165,61 @@ class UMBPStore(HiCacheStorage):
|
||||
# ------------------------------------------------------------------
|
||||
# Key suffix generation — mirrors MooncakeStore
|
||||
# ------------------------------------------------------------------
|
||||
def _get_mha_buffer_meta(self, keys, indices):
|
||||
ptr_list, element_size_list = self.mem_pool_host.get_page_buffer_meta(indices)
|
||||
key_list = []
|
||||
for key_ in keys:
|
||||
key_list.append(f"{key_}_{self.mha_suffix}_k")
|
||||
key_list.append(f"{key_}_{self.mha_suffix}_v")
|
||||
assert len(key_list) == len(ptr_list)
|
||||
return key_list, ptr_list, element_size_list
|
||||
|
||||
def _get_mha_split_heads_buffer_meta(self, keys, indices):
|
||||
ptr_list, element_size_list = (
|
||||
self.mem_pool_host.get_split_heads_page_buffer_meta(
|
||||
indices, self.split_factor
|
||||
)
|
||||
def _anchor_key_suffixes(self) -> tuple[str, ...]:
|
||||
if self._kv_anchor_is_page_envelope:
|
||||
return (f"_{self.mha_suffix}_kv",)
|
||||
if self.is_mla_backend:
|
||||
return (f"_{self.mla_suffix}_k",)
|
||||
ranks = (
|
||||
self.mha_suffix
|
||||
if self.storage_config and self.storage_config.should_split_heads
|
||||
else (self.mha_suffix,)
|
||||
)
|
||||
return tuple(
|
||||
f"_{rank}_{component}" for rank in ranks for component in ("k", "v")
|
||||
)
|
||||
key_list = []
|
||||
for key_ in keys:
|
||||
for suffix in self.mha_suffix:
|
||||
key_list.append(f"{key_}_{suffix}_k")
|
||||
key_list.append(f"{key_}_{suffix}_v")
|
||||
assert len(key_list) == len(ptr_list)
|
||||
return key_list, ptr_list, element_size_list
|
||||
|
||||
def _get_mla_buffer_meta(self, keys, indices):
|
||||
ptr_list, element_size_list = self.mem_pool_host.get_page_buffer_meta(indices)
|
||||
key_list = []
|
||||
for key_ in keys:
|
||||
key_list.append(f"{key_}_{self.mla_suffix}_k")
|
||||
assert len(key_list) == len(ptr_list)
|
||||
return key_list, ptr_list, element_size_list
|
||||
def _anchor_keys(self, keys) -> tuple[list[str], int]:
|
||||
suffixes = self._anchor_key_suffixes()
|
||||
return [f"{key}{suffix}" for key in keys for suffix in suffixes], len(suffixes)
|
||||
|
||||
def _batch_preprocess(self, keys, host_indices):
|
||||
assert len(keys) > 0
|
||||
assert len(keys) == len(host_indices) // self.mem_pool_host.page_size
|
||||
if self.is_mla_backend:
|
||||
return self._get_mla_buffer_meta(keys, host_indices)
|
||||
key_list, _ = self._anchor_keys(keys)
|
||||
if (
|
||||
not self._kv_anchor_is_page_envelope
|
||||
and not self.is_mla_backend
|
||||
and self.storage_config
|
||||
and self.storage_config.should_split_heads
|
||||
):
|
||||
ptr_list, element_size_list = (
|
||||
self.mem_pool_host.get_split_heads_page_buffer_meta(
|
||||
host_indices, self.split_factor
|
||||
)
|
||||
)
|
||||
else:
|
||||
if self.storage_config and self.storage_config.should_split_heads:
|
||||
return self._get_mha_split_heads_buffer_meta(keys, host_indices)
|
||||
else:
|
||||
return self._get_mha_buffer_meta(keys, host_indices)
|
||||
ptr_list, element_size_list = self.mem_pool_host.get_page_buffer_meta(
|
||||
host_indices
|
||||
)
|
||||
assert len(key_list) == len(ptr_list)
|
||||
return key_list, ptr_list, element_size_list
|
||||
|
||||
def _batch_postprocess(self, results: List[bool], is_set_operate=False):
|
||||
"""Convert per-key-component results to per-page results.
|
||||
|
||||
For MHA: each page has K+V → group pairs.
|
||||
For MLA: each page has K only.
|
||||
Unified page envelopes and MLA have one object per page. Ordinary MHA
|
||||
has K+V pairs, or one pair per split rank.
|
||||
"""
|
||||
if self.is_mla_backend:
|
||||
group_size = len(self._anchor_key_suffixes())
|
||||
if group_size == 1:
|
||||
return list(results)
|
||||
else:
|
||||
if self.storage_config and self.storage_config.should_split_heads:
|
||||
group_size = self.split_factor * 2
|
||||
groups = [
|
||||
results[i : i + group_size]
|
||||
for i in range(0, len(results), group_size)
|
||||
]
|
||||
return [all(g) for g in groups]
|
||||
else:
|
||||
# Group K/V pairs
|
||||
kv_pairs = zip(results[::2], results[1::2])
|
||||
return [k and v for k, v in kv_pairs]
|
||||
result_count = len(results)
|
||||
if not (self.storage_config and self.storage_config.should_split_heads):
|
||||
result_count -= result_count % group_size
|
||||
return [
|
||||
all(results[i : i + group_size]) for i in range(0, result_count, group_size)
|
||||
]
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Zero-copy v1 interface
|
||||
@@ -1282,19 +1277,8 @@ class UMBPStore(HiCacheStorage):
|
||||
depths_per_page = [prefix_len + i for i in range(len(keys))]
|
||||
|
||||
# Expand to match the key_strs layout produced by _batch_preprocess.
|
||||
expanded = []
|
||||
for d in depths_per_page:
|
||||
if self.is_mla_backend:
|
||||
expanded.append(d) # MLA: 1 key per page
|
||||
elif self.storage_config and self.storage_config.should_split_heads:
|
||||
# split heads: 2 keys per split rank, split_factor ranks per page
|
||||
for _ in range(self.split_factor):
|
||||
expanded.append(d)
|
||||
expanded.append(d)
|
||||
else:
|
||||
expanded.append(d) # K
|
||||
expanded.append(d) # V
|
||||
return expanded
|
||||
objects_per_page = len(self._anchor_key_suffixes())
|
||||
return [depth for depth in depths_per_page for _ in range(objects_per_page)]
|
||||
|
||||
def batch_set_v1(
|
||||
self,
|
||||
@@ -1352,23 +1336,7 @@ class UMBPStore(HiCacheStorage):
|
||||
self, keys: List[str], extra_info: Optional[HiCacheStorageExtraInfo] = None
|
||||
) -> int:
|
||||
"""Return count of consecutive existing keys from start."""
|
||||
if self.is_mla_backend:
|
||||
query_keys = [f"{key}_{self.mla_suffix}_k" for key in keys]
|
||||
key_multiplier = 1
|
||||
else:
|
||||
query_keys = []
|
||||
if self.storage_config and self.storage_config.should_split_heads:
|
||||
for key in keys:
|
||||
for suffix in self.mha_suffix:
|
||||
query_keys.append(f"{key}_{suffix}_k")
|
||||
query_keys.append(f"{key}_{suffix}_v")
|
||||
key_multiplier = 2 * self.split_factor
|
||||
else:
|
||||
for key in keys:
|
||||
query_keys.append(f"{key}_{self.mha_suffix}_k")
|
||||
query_keys.append(f"{key}_{self.mha_suffix}_v")
|
||||
key_multiplier = 2
|
||||
|
||||
query_keys, key_multiplier = self._anchor_keys(keys)
|
||||
hit_count = self.client.batch_exists_consecutive(query_keys)
|
||||
return hit_count // key_multiplier
|
||||
|
||||
|
||||
@@ -221,9 +221,14 @@ class SWAComponent(TreeComponent):
|
||||
)
|
||||
|
||||
def _translate_full_to_swa(self, full_indices: torch.Tensor) -> torch.Tensor:
|
||||
return self.cache.token_to_kv_pool_allocator.translate_loc_from_full_to_swa(
|
||||
full_indices
|
||||
swa_indices = (
|
||||
self.cache.token_to_kv_pool_allocator.translate_swa_indices_for_transfer(
|
||||
full_indices
|
||||
)
|
||||
)
|
||||
# Tree component values use int64 indices; normalize transfer ids at the
|
||||
# tree boundary.
|
||||
return swa_indices.to(torch.int64)
|
||||
|
||||
def _unified_allocator(self):
|
||||
"""The unified SWA composite, or None when running on the static pool."""
|
||||
|
||||
@@ -14,9 +14,11 @@ from __future__ import annotations
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from sglang.srt.mem_cache.buffer_mode.pipeline import validate_buffer_only_stack
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.observability.metrics_collector import (
|
||||
STAT_LOGGER_ROLE_STORAGE,
|
||||
StorageMetricsCollector,
|
||||
@@ -73,6 +75,23 @@ class StorageAttachment:
|
||||
"launch with --enable-hierarchical-cache to attach a backend.",
|
||||
)
|
||||
|
||||
# Write-back can retain auxiliary host rows without FULL, or backed
|
||||
# descendants below an unbacked FULL ancestor. Those states are invalid
|
||||
# for write-through insertion and eviction. Check before changing any
|
||||
# policy, including on the same-backend update and re-attach paths.
|
||||
if (
|
||||
cache.is_write_back
|
||||
and cache.host_memory_mode != "buffer_only"
|
||||
and hicache_write_policy in ("write_through", "write_through_selective")
|
||||
and not cache.tree_core.is_write_through_compatible()
|
||||
):
|
||||
return (
|
||||
False,
|
||||
"Cannot switch HiCache from write_back to write_through while "
|
||||
"the cache has host data without its FULL prefix. "
|
||||
"Flush the cache before changing the write policy.",
|
||||
)
|
||||
|
||||
if cache.enable_storage:
|
||||
current_backend = controller.storage_backend_type
|
||||
if current_backend != storage_backend:
|
||||
@@ -90,10 +109,6 @@ class StorageAttachment:
|
||||
"policies updated.",
|
||||
)
|
||||
|
||||
# Apply policies before the controller attach, so the storage threads
|
||||
# observe the new values as soon as they start.
|
||||
self._apply_policies(hicache_storage_prefetch_policy, hicache_write_policy)
|
||||
|
||||
logger.info(f"Attaching HiCache storage backend: {storage_backend}")
|
||||
try:
|
||||
(
|
||||
@@ -113,8 +128,23 @@ class StorageAttachment:
|
||||
f"'{storage_backend_extra_config_json}': {e}",
|
||||
)
|
||||
|
||||
original_policies = (
|
||||
cache.prefetch_stop_policy,
|
||||
controller.write_policy,
|
||||
cache.write_through_threshold,
|
||||
cache.is_write_back,
|
||||
)
|
||||
try:
|
||||
prefetch_threshold = self.resolve_prefetch_threshold(prefetch_threshold)
|
||||
if cache.host_memory_mode == "buffer_only":
|
||||
validate_buffer_only_stack(
|
||||
sidecar_pool_specs=cache.sidecar_pool_specs,
|
||||
host_pool_group=cache.host_pool_group,
|
||||
swa_component=cache.components.get(ComponentType.SWA),
|
||||
storage_prefetch_threshold=prefetch_threshold,
|
||||
)
|
||||
# New workers must see the requested policy from their first operation.
|
||||
self._apply_policies(hicache_storage_prefetch_policy, hicache_write_policy)
|
||||
controller.attach_storage_backend(
|
||||
storage_backend=storage_backend,
|
||||
prefetch_threshold=prefetch_threshold,
|
||||
@@ -123,6 +153,12 @@ class StorageAttachment:
|
||||
host_pools=controller.mem_pool_host.entries,
|
||||
)
|
||||
except Exception as e:
|
||||
(
|
||||
cache.prefetch_stop_policy,
|
||||
controller.write_policy,
|
||||
cache.write_through_threshold,
|
||||
cache.is_write_back,
|
||||
) = original_policies
|
||||
logger.exception(
|
||||
f"Failed to attach storage backend '{storage_backend}': {e}"
|
||||
)
|
||||
@@ -395,15 +431,19 @@ class StorageAttachment:
|
||||
cache.buffer_pipeline.release_anchor_lock(handle)
|
||||
controller.append_host_mem_release(
|
||||
host_indices=info.host_indices[:completed_tokens],
|
||||
extra_pools=[
|
||||
x for xfers in info.comp_xfers.values() for x in xfers
|
||||
],
|
||||
extra_pools=(
|
||||
[x for xfers in info.comp_xfers.values() for x in xfers]
|
||||
if info.operation.pool_transfers_done
|
||||
else None
|
||||
),
|
||||
)
|
||||
controller.prefetch_tokens_occupied = max(
|
||||
0,
|
||||
controller.prefetch_tokens_occupied
|
||||
- cache._prefetch_occupied_span(
|
||||
info.prefetch_key, info.host_indices
|
||||
info.prefetch_key,
|
||||
info.host_indices,
|
||||
operation=info.operation,
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
|
||||
@@ -544,6 +544,22 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
"""
|
||||
return self._node_arena[node_id]
|
||||
|
||||
def is_write_through_compatible(self) -> bool:
|
||||
for node in self._node_arena.values():
|
||||
if node is self.root_node:
|
||||
continue
|
||||
full_host = node.component_data[BASE_COMPONENT_TYPE].host_value
|
||||
if full_host is None:
|
||||
if any(
|
||||
node.component_data[ct].host_value is not None
|
||||
for ct in self.component_types
|
||||
if ct != BASE_COMPONENT_TYPE
|
||||
):
|
||||
return False
|
||||
elif node.parent is not self.root_node and not node.parent.backuped:
|
||||
return False
|
||||
return True
|
||||
|
||||
def is_backuped(self, node_id: NodeId) -> bool:
|
||||
"""Whether the node's KV is already backed up to host."""
|
||||
return self._node_arena[node_id].backuped
|
||||
@@ -1756,14 +1772,20 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
# A failed backup never issues the D->H copy, so the subtree root has
|
||||
# no host state and no in-flight DMA reading its device slots.
|
||||
assert not node.backuped and node.write_through_pending_id is None
|
||||
if any(cd.host_lock_ref > 0 for cd in node.component_data):
|
||||
if node.load_back_pending_id is not None or any(
|
||||
cd.lock_ref > 0 or cd.host_lock_ref > 0 for cd in node.component_data
|
||||
):
|
||||
return result
|
||||
descendants: list[UnifiedTreeNode] = []
|
||||
stack = list(node.children.values())
|
||||
while stack:
|
||||
cur = stack.pop()
|
||||
if any(
|
||||
cd.lock_ref > 0 or cd.host_lock_ref > 0 for cd in cur.component_data
|
||||
if (
|
||||
cur.write_through_pending_id is not None
|
||||
or cur.load_back_pending_id is not None
|
||||
or any(
|
||||
cd.lock_ref > 0 or cd.host_lock_ref > 0 for cd in cur.component_data
|
||||
)
|
||||
):
|
||||
return result
|
||||
descendants.append(cur)
|
||||
@@ -2275,12 +2297,17 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
|
||||
child_key = key.child_key(self.page_size)
|
||||
matched_length = 0
|
||||
host_prefix_len = 0
|
||||
refilled_host_node = None
|
||||
cache_actions: list[CacheAction | ComponentAction] = []
|
||||
while len(key) > 0 and child_key in node.children:
|
||||
node = node.children[child_key]
|
||||
self._touch_node(node)
|
||||
prefix_len = node.key.match(key, page_size=self.page_size)
|
||||
|
||||
matched_host_value = host_value[:prefix_len]
|
||||
matched_hash_value = hash_value[: prefix_len // self.page_size]
|
||||
|
||||
key = key[prefix_len:]
|
||||
host_value = host_value[prefix_len:]
|
||||
hash_value = hash_value[prefix_len // self.page_size :]
|
||||
@@ -2291,18 +2318,37 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
if action is not None:
|
||||
cache_actions.append(action)
|
||||
|
||||
if not self.is_write_back:
|
||||
full_data = node.component_data[BASE_COMPONENT_TYPE]
|
||||
if full_data.host_value is None:
|
||||
full_data.host_value = matched_host_value.clone()
|
||||
if node.hash_value is None:
|
||||
node.hash_value = list(matched_hash_value)
|
||||
self.kv_events.record_store(node, medium=StorageMedium.CPU)
|
||||
self._update_evictable_leaf_sets(node)
|
||||
if node.parent is not None:
|
||||
self._update_evictable_leaf_sets(node.parent)
|
||||
self._update_duplicate_tracking(node)
|
||||
refilled_host_node = node
|
||||
else:
|
||||
assert refilled_host_node is None, (
|
||||
"write-through host-prefix invariant broken: encountered "
|
||||
f"backed node {node.id} below a refilled host tombstone"
|
||||
)
|
||||
host_prefix_len += prefix_len
|
||||
|
||||
if len(key):
|
||||
child_key = key.child_key(self.page_size)
|
||||
|
||||
result = InsertResult(
|
||||
prefix_len=matched_length,
|
||||
prefix_len=(matched_length if self.is_write_back else host_prefix_len),
|
||||
total_len=total_len,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
if len(key) == 0:
|
||||
if (
|
||||
node is not self.root_node
|
||||
and node.component_data[BASE_COMPONENT_TYPE].host_value is not None
|
||||
if node is not self.root_node and (
|
||||
refilled_host_node is not None
|
||||
or node.component_data[BASE_COMPONENT_TYPE].host_value is not None
|
||||
):
|
||||
result.inserted_host_node = node.id
|
||||
return result
|
||||
|
||||
@@ -52,6 +52,7 @@ class EvictDeviceNextNodeResult(BaseEvictionResult):
|
||||
"""
|
||||
|
||||
node_id: Optional[NodeId] = None
|
||||
backup_kv: Optional[BackupKV] = None
|
||||
made_progress: bool = False
|
||||
unbacked_tokens: int = 0
|
||||
mamba_backup_node_id: Optional[NodeId] = None
|
||||
@@ -183,6 +184,14 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def is_write_through_compatible(self) -> bool:
|
||||
"""Whether current host ownership satisfies write-through invariants.
|
||||
|
||||
Read-only; the caller must have drained in-flight cache operations.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def is_backuped(self, node_id: NodeId) -> bool:
|
||||
"""Whether the node's KV is already backed up to host."""
|
||||
@@ -510,6 +519,13 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
"""Mark the host tier (HiCache) as wired."""
|
||||
...
|
||||
|
||||
def enable_swa_write_back_eviction_barrier(self) -> None:
|
||||
"""Enable a backend-managed barrier when needed.
|
||||
|
||||
The Python core demotes through SWAComponent directly. Native cores
|
||||
may return a backup action to the cache executor before eviction.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def set_host_memory_buffer_only(self) -> None:
|
||||
"""Mark the host tier as buffer-only: one node staged per backup intent."""
|
||||
|
||||
@@ -287,6 +287,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
"declined_rate_limited": 0,
|
||||
"declined_anchor_lost": 0,
|
||||
"declined_device_covered": 0,
|
||||
"declined_host_oversize": 0,
|
||||
"revoked_insufficient": 0,
|
||||
"revoked_full_miss": 0,
|
||||
"l3_demand_requests": 0,
|
||||
@@ -470,6 +471,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
sidecar_pool_specs=self.sidecar_pool_specs,
|
||||
host_pool_group=self.host_pool_group,
|
||||
swa_component=swa,
|
||||
storage_prefetch_threshold=storage_prefetch_threshold,
|
||||
)
|
||||
self.buffer_pipeline = BufferModePipeline(
|
||||
cache=self,
|
||||
@@ -485,11 +487,11 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# content would drop-newest and punch storage holes.
|
||||
write_backlog_cap=2 * self.token_to_kv_pool_allocator.size_full,
|
||||
)
|
||||
# State initialization
|
||||
if self.buffer_pipeline is not None:
|
||||
self.cache_controller.host_write_staged_tokens_fn = lambda: (
|
||||
self.buffer_pipeline.write_staged_tokens_
|
||||
)
|
||||
|
||||
# State initialization
|
||||
self.write_through_threshold = (
|
||||
1 if get_memory().hicache_write_policy == "write_through" else 2
|
||||
)
|
||||
@@ -497,6 +499,13 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.cache_controller is not None
|
||||
and self.cache_controller.write_policy == "write_back"
|
||||
)
|
||||
# Preserve the SWA host window before device eviction makes it unrecoverable.
|
||||
if (
|
||||
get_memory().enable_unified_memory
|
||||
and self.host_memory_mode == "cache"
|
||||
and self.tree_core.has_swa_host_pool
|
||||
):
|
||||
self.tree_core.enable_swa_write_back_eviction_barrier()
|
||||
# Pre-seed the logical dropped-tokens series.
|
||||
if self.metrics_collector is not None and self.cache_controller is not None:
|
||||
reasons = ["host_pressure"]
|
||||
@@ -749,45 +758,74 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def _evict_device_next_node(
|
||||
self, component_type: ComponentType, tracker: dict[ComponentType, int]
|
||||
) -> tuple[Optional[NodeId], bool]:
|
||||
"""Advance the eviction walk one node, consuming its step result."""
|
||||
result = self.tree_core.evict_device_next_node(component_type, tracker)
|
||||
if result.mamba_backup_node_id is not None:
|
||||
assert component_type == ComponentType.MAMBA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees and not result.host_frees and not result.tracker
|
||||
"""Advance the walk, completing pending host-backup barriers."""
|
||||
while True:
|
||||
result = self.tree_core.evict_device_next_node(component_type, tracker)
|
||||
if result.mamba_backup_node_id is not None:
|
||||
assert component_type == ComponentType.MAMBA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees
|
||||
and not result.host_frees
|
||||
and not result.tracker
|
||||
)
|
||||
# Reserve a host state slot and wait for the backup acknowledgment
|
||||
# before freeing device state. If allocation fails, eviction still
|
||||
# proceeds to make room on the device.
|
||||
node_id = result.mamba_backup_node_id
|
||||
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
|
||||
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
|
||||
self.evict_host(1, ComponentType.MAMBA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_mamba_state_eviction(node_id)
|
||||
elif result.swa_backup_node_id is not None:
|
||||
assert component_type == ComponentType.SWA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees
|
||||
and not result.host_frees
|
||||
and not result.tracker
|
||||
)
|
||||
# The backup can cover several unbacked SWA segments. Reserve host
|
||||
# space for the whole window before copying it, then resume eviction
|
||||
# even if host allocation fails.
|
||||
node_id = result.swa_backup_node_id
|
||||
needed = result.swa_backup_num_tokens
|
||||
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
|
||||
if (
|
||||
swa_host_pool is not None
|
||||
and swa_host_pool.available_size() < needed
|
||||
):
|
||||
self.evict_host(needed, ComponentType.SWA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_swa_state_eviction(node_id)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
if self._tracks_write_through_unbacked_evictions():
|
||||
self._record_dropped_tokens(
|
||||
result.unbacked_tokens,
|
||||
reason="write_through_unbacked_eviction",
|
||||
)
|
||||
self._accumulate_tracker(tracker, result.tracker)
|
||||
if result.backup_kv is None:
|
||||
return result.node_id, result.made_progress
|
||||
|
||||
assert result.node_id is None
|
||||
assert self.buffer_pipeline is None, (
|
||||
"SWA write-back eviction barriers are cache-mode only"
|
||||
)
|
||||
# Reserve a host state slot and wait for the backup acknowledgment
|
||||
# before freeing device state. If allocation fails, eviction still
|
||||
# proceeds to make room on the device.
|
||||
node_id = result.mamba_backup_node_id
|
||||
mamba_host_pool = self.host_pool_group.get_pool(PoolName.MAMBA)
|
||||
if mamba_host_pool is not None and mamba_host_pool.available_size() < 1:
|
||||
self.evict_host(1, ComponentType.MAMBA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_mamba_state_eviction(node_id)
|
||||
elif result.swa_backup_node_id is not None:
|
||||
assert component_type == ComponentType.SWA and result.node_id is None
|
||||
assert (
|
||||
not result.device_frees and not result.host_frees and not result.tracker
|
||||
written = self._execute_and_commit_kv_backup(
|
||||
result.backup_kv, write_back=True
|
||||
)
|
||||
# The backup can cover several unbacked SWA segments. Reserve host
|
||||
# space for the whole window before copying it, then resume eviction
|
||||
# even if host allocation fails.
|
||||
node_id = result.swa_backup_node_id
|
||||
needed = result.swa_backup_num_tokens
|
||||
swa_host_pool = self.host_pool_group.get_pool(PoolName.SWA)
|
||||
if swa_host_pool is not None and swa_host_pool.available_size() < needed:
|
||||
self.evict_host(needed, ComponentType.SWA)
|
||||
self.backup_node_for_write_back(node_id)
|
||||
result = self.tree_core.finish_swa_state_eviction(node_id)
|
||||
self._free_values(result.device_frees, result.host_frees)
|
||||
if self._tracks_write_through_unbacked_evictions():
|
||||
self._record_dropped_tokens(
|
||||
result.unbacked_tokens,
|
||||
reason="write_through_unbacked_eviction",
|
||||
)
|
||||
self._accumulate_tracker(tracker, result.tracker)
|
||||
return result.node_id, result.made_progress
|
||||
if written <= 0:
|
||||
node_id = result.backup_kv.node_ids[0]
|
||||
logger.warning(
|
||||
"write_back: auxiliary backup failed under host pressure "
|
||||
"(component=%s, node=%d); dropping only the component",
|
||||
component_type.name,
|
||||
node_id,
|
||||
)
|
||||
# Match the Python SWA demotion: preserve FULL and descendants;
|
||||
# the resumed native walk tombstones this component after one try.
|
||||
continue
|
||||
self.writing_check(write_back=True)
|
||||
|
||||
def _evict_device_leaf(
|
||||
self, node_id: NodeId, tracker: dict[ComponentType, int]
|
||||
@@ -1366,14 +1404,17 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
component_transfers: dict[ComponentType, list[PoolTransfer]] = {}
|
||||
if self.supports_swa():
|
||||
kv_cache = self.token_to_kv_pool_allocator.get_kvcache()
|
||||
assert self.sliding_window_size is not None
|
||||
window_start = max(0, num_tokens - self.sliding_window_size)
|
||||
window_start = window_start // self.page_size * self.page_size
|
||||
window_indices = self.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, window_start:num_tokens
|
||||
].to(torch.int64)
|
||||
swa_indices = kv_cache.translate_loc_from_full_to_swa(window_indices)
|
||||
swa_indices = (
|
||||
self.token_to_kv_pool_allocator.translate_swa_indices_for_transfer(
|
||||
window_indices
|
||||
)
|
||||
)
|
||||
assert bool((swa_indices > 0).all()), (
|
||||
f"unmapped SWA window positions for request {req.rid}"
|
||||
)
|
||||
@@ -1392,13 +1433,12 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
for transfers in component_transfers.values()
|
||||
for transfer in transfers
|
||||
]
|
||||
extra_transfers.extend(
|
||||
self._build_sidecar_transfers(
|
||||
CacheTransferPhase.BACKUP_HOST,
|
||||
kv_transfer,
|
||||
component_transfers,
|
||||
)
|
||||
sidecar_transfers = self._build_sidecar_transfers(
|
||||
CacheTransferPhase.BACKUP_HOST,
|
||||
kv_transfer,
|
||||
component_transfers,
|
||||
)
|
||||
extra_transfers.extend(sidecar_transfers)
|
||||
return full_indices, extra_transfers
|
||||
|
||||
def _reclaim_retraction_host(self, num_tokens: int) -> int:
|
||||
@@ -2397,7 +2437,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self.buffer_pipeline.release_anchor_lock(request)
|
||||
del self.ongoing_prefetch[request]
|
||||
self.cache_controller.prefetch_tokens_occupied -= (
|
||||
self._prefetch_occupied_span(prefetch_key, host_indices)
|
||||
self._prefetch_occupied_span(
|
||||
prefetch_key, host_indices, operation=operation
|
||||
)
|
||||
)
|
||||
self.prefetch_loaded_tokens_by_reqid[request] = 0
|
||||
self.prefetch_loaded_storage_start_by_reqid.pop(request, None)
|
||||
@@ -2584,7 +2626,7 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# Buffer mode granted occupancy at hit-alloc, sized to the bounce;
|
||||
# cache mode reserved the requested span at enqueue.
|
||||
self.cache_controller.prefetch_tokens_occupied -= self._prefetch_occupied_span(
|
||||
prefetch_key, host_indices
|
||||
prefetch_key, host_indices, operation=operation
|
||||
)
|
||||
|
||||
def _invalidate_absent_from_hit_query(self, operation) -> None:
|
||||
@@ -2621,11 +2663,18 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def prefetch_outcome_stats_snapshot(self) -> dict:
|
||||
return self._prefetch_outcome_stats.copy()
|
||||
|
||||
def _prefetch_occupied_span(self, prefetch_key, host_indices) -> int:
|
||||
def _prefetch_occupied_span(
|
||||
self, prefetch_key, host_indices, *, operation=None
|
||||
) -> int:
|
||||
"""Occupancy units held by a prefetch: cache mode reserves the
|
||||
requested span at enqueue; buffer mode grants at hit-alloc, sized
|
||||
to the allocation (0 while still querying / parked)."""
|
||||
if self.host_memory_mode == "buffer_only":
|
||||
if (
|
||||
operation is not None
|
||||
and operation.buffer_host_occupied_units is not None
|
||||
):
|
||||
return operation.buffer_host_occupied_units
|
||||
return len(host_indices) if host_indices is not None else 0
|
||||
return len(prefetch_key)
|
||||
|
||||
@@ -2742,7 +2791,9 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
cc.prefetch_tokens_occupied = max(
|
||||
0,
|
||||
cc.prefetch_tokens_occupied
|
||||
- self._prefetch_occupied_span(prefetch_key, _host_indices),
|
||||
- self._prefetch_occupied_span(
|
||||
prefetch_key, _host_indices, operation=operation
|
||||
),
|
||||
)
|
||||
|
||||
def _drain_storage_control_queues_impl(
|
||||
@@ -2827,20 +2878,24 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
else:
|
||||
aux_hit_tokens = hit_tokens
|
||||
alloc_len = hit_tokens
|
||||
host_indices = cc.mem_pool_host.alloc(alloc_len)
|
||||
if host_indices is None:
|
||||
self.evict_host(alloc_len)
|
||||
host_indices = cc.mem_pool_host.alloc(alloc_len)
|
||||
if host_indices is None and not buffer_mode:
|
||||
# Memory-pressure fallback: a shorter page-aligned prefix.
|
||||
# (Cache mode only — buffer mode parks for the full hit.)
|
||||
available_size = cc.mem_pool_host.available_size()
|
||||
alloc_len = min(
|
||||
hit_tokens,
|
||||
available_size - (available_size % self.page_size),
|
||||
if buffer_mode and not cc.can_fit_prefetch_host_buffers(
|
||||
operation, alloc_len
|
||||
):
|
||||
self._prefetch_outcome_stats["declined_host_oversize"] += 1
|
||||
logger.warning(
|
||||
"HiCache buffer prefetch declined req=%s: the Full/sidecar "
|
||||
"bounce cannot fit in an empty host arena",
|
||||
request,
|
||||
)
|
||||
if alloc_len >= self.prefetch_threshold:
|
||||
host_indices = cc.mem_pool_host.alloc(alloc_len)
|
||||
self.revoke_pending_prefetch(request)
|
||||
return True
|
||||
host_indices, alloc_len = cc.allocate_storage_hit(
|
||||
operation,
|
||||
hit_tokens,
|
||||
allow_partial=not buffer_mode,
|
||||
min_tokens=self.prefetch_threshold,
|
||||
evict_host=self.evict_host,
|
||||
)
|
||||
if host_indices is None:
|
||||
if buffer_mode:
|
||||
# Parked ops hold no pin: release and re-take at the next
|
||||
@@ -2890,7 +2945,12 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
operation.host_indices = host_indices
|
||||
self.ongoing_prefetch[request] = info._replace(host_indices=host_indices)
|
||||
if buffer_mode:
|
||||
cc.prefetch_tokens_occupied += alloc_len
|
||||
operation.buffer_host_occupied_units = (
|
||||
self.buffer_pipeline.host_allocation_units(
|
||||
host_indices, operation.pool_transfers
|
||||
)
|
||||
)
|
||||
cc.prefetch_tokens_occupied += operation.buffer_host_occupied_units
|
||||
cc.prefetch_buffer.put(operation)
|
||||
return True
|
||||
|
||||
|
||||
Reference in New Issue
Block a user