[HiCache] Buffer-only L3 for an auxiliary SWA window; load back only missing window rows (#43932)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: houseroad <houseroad@users.noreply.github.com>
Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-14 16:43:24 -07:00
committed by GitHub
co-authored by Hanming Lu houseroad xiezhq-hermann
parent aa8793f33b
commit e7d6a88bfc
22 changed files with 356 additions and 243 deletions
@@ -1480,12 +1480,20 @@ class Req(ReqDllmMixin):
return self.output_ids[: self.finished_len]
return self.output_ids
@property
def auxiliary_swa_host_hit_length(self) -> int:
plan = self.staged_prefetch_plan
return (
plan.window_tokens(ComponentType.AUXILIARY_SWA) if plan is not None else 0
)
def needs_host_load_back(self) -> bool:
"""Whether any cache layer has a host hit that needs L2 H2D load_back."""
return (
self.host_hit_length > 0
or self.swa_host_hit_length > 0
or self.mamba_host_hit_length > 0
or self.auxiliary_swa_host_hit_length > 0
)
def materialized_host_hit_len(self) -> int:
@@ -795,6 +795,7 @@ class PrefillAdder:
input_tokens=len(req.full_untruncated_fill_ids),
swa_host_hit_length=swa_host_hit_length,
chunk_limit=self.rem_chunk_tokens,
auxiliary_swa_host_hit_length=req.auxiliary_swa_host_hit_length,
)
def _mamba_gap_budget_for_req(
@@ -1333,6 +1334,16 @@ class PrefillAdder:
return self.budget_state()
def _defer_staged_for_capacity_if_idle(self, req: Req, *, pool: str) -> None:
# Running work and admitted prefills (including continuing chunks) can
# release capacity. Only idle budget failures should force recomputation.
if self.can_run_list or (
self.running_batch is not None and self.running_batch.reqs
):
return
if self.tree_cache.buffer_pipeline is not None:
self.tree_cache.buffer_pipeline.defer_staged_admission(req, pool)
def add_one_req(
self, req: Req, has_chunked_req: bool, truncation_align_size: Optional[int]
):
@@ -1379,6 +1390,8 @@ class PrefillAdder:
has_chunked_req=has_chunked_req,
)
if isinstance(admission, AddReqResult):
if admission is AddReqResult.NO_TOKEN:
self._defer_staged_for_capacity_if_idle(req, pool="prefill_budget")
return admission
# A rejected candidate must not report prefillable or queue H2D.
@@ -1435,6 +1448,9 @@ class PrefillAdder:
swa_host_hit_length=req.swa_host_hit_length,
chunk_limit=self.rem_chunk_tokens,
):
self._defer_staged_for_capacity_if_idle(
req, pool="load_back_budget"
)
return AddReqResult.NO_TOKEN
promised_host_hit = req.host_hit_length
loaded = self.tree_cache.init_load_back(
@@ -28,6 +28,7 @@ from __future__ import annotations
import logging
from array import array
from collections import deque
from dataclasses import replace
from typing import TYPE_CHECKING, Optional
import msgspec
@@ -52,8 +53,10 @@ from sglang.srt.mem_cache.hicache_storage import (
)
from sglang.srt.mem_cache.pool_host.base import HostKVCache
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.storage_prefetch import StagedPrefetchPlan
from sglang.srt.mem_cache.unified_cache.cache_action import RebuildFullToSWAMapping
from sglang.srt.mem_cache.storage_prefetch import (
StagedPrefetchPlan,
WindowLoadBackPlan,
)
from sglang.srt.mem_cache.unified_cache.components import (
CacheTransferPhase,
ComponentType,
@@ -146,10 +149,11 @@ class _StagedPrefetch(msgspec.Struct):
class _OngoingBufferLoadBack(msgspec.Struct):
"""A queued H2D load, registered after its prefix is inserted.
The full host bounce and redundant auxiliary device slots remain owned
here until the copy completes. ``load_transfers`` is the transfer list
submitted to H2D; ``aux_xfers`` retains the original staging owners and
``load_contexts`` each component's part in the load.
The full host bounce, and any redundant device slots a component's load
context reports, remain owned here until the copy completes.
``load_transfers`` is the transfer list submitted to H2D; ``aux_xfers``
retains the original staging owners and ``load_contexts`` each
component's part in the load.
"""
request: CacheRequestHandle
@@ -176,6 +180,8 @@ def validate_buffer_only_stack(
host_pool_group: HostPoolGroup,
swa_component: Optional[SWAComponent],
storage_prefetch_threshold: int = 256,
*,
auxiliary_swa_component: Optional[SWAComponent] = None,
) -> None:
"""Post-assembly buffer-mode fences.
@@ -202,50 +208,48 @@ def validate_buffer_only_stack(
f"pool={spec.pool_name}, host_slots={sidecar_size}, "
f"source={spec.indices_from_pool}, source_slots={source_size}."
)
swa = swa_component
if swa is not None and swa._swa_kv_pool_host is None:
# Only reachable on SWA models with the unified_kv layout (SWA as
# a device-only ring): without a host pool the window can neither
# stage for writes nor fetch for load-backs.
raise ValueError(
"--hicache-host-memory-mode buffer_only on SWA models "
"requires an SWA host staging pool; the unified_kv layout "
"keeps SWA as a device-only ring."
)
if swa is not None and swa._swa_kv_pool_host is not None:
# Below two windows the pool cannot hold a staging write AND the
# loads-priority reserve (_aux_loads_margin floors at one
# window), so every window-carrying intent would be dropped as
# oversize and SWA storage coverage would silently be zero.
window_tokens = swa.full_window_pages * swa._swa_kv_pool_host.page_size
window_pools = []
for component in (swa_component, auxiliary_swa_component):
if component is None:
continue
pool = component._swa_kv_pool_host
if pool is None:
raise ValueError(
"--hicache-host-memory-mode buffer_only on SWA models "
"requires an SWA host staging pool; the unified_kv layout "
"keeps SWA as a device-only ring."
)
window_pools.append((pool, component.full_window_pages * pool.page_size))
for pool, window_tokens in window_pools:
# Every window needs a write bounce plus a loads-priority reserve.
# For a shared arena the reserve must hold all windows together.
shared_domain = (
swa._swa_kv_pool_host.shared_allocation_domain
if isinstance(swa._swa_kv_pool_host, HostKVCache)
else None
pool.shared_allocation_domain if isinstance(pool, HostKVCache) else None
)
if shared_domain is not None:
full_host_pool = (
swa.cache.cache_controller.mem_pool_host.anchor_entry.host_pool
)
full_host_pool = host_pool_group.anchor_entry.host_pool
min_full_tokens = _minimum_full_transfer_tokens(
full_host_pool, storage_prefetch_threshold
)
one_transfer_bytes = (
window_tokens * swa._swa_kv_pool_host.size_per_token
+ min_full_tokens * full_host_pool.size_per_token
)
one_transfer = [
(full_host_pool.pool_label, min_full_tokens),
(swa._swa_kv_pool_host.pool_label, window_tokens),
shared_pools = [
(host, tokens)
for host, tokens in [(full_host_pool, min_full_tokens), *window_pools]
if isinstance(host, HostKVCache)
and host.shared_allocation_domain is shared_domain
]
one_transfer_bytes = sum(
tokens * host.size_per_token for host, tokens in shared_pools
)
one_transfer = [(host.pool_label, tokens) for host, tokens in shared_pools]
enough_capacity = shared_domain.can_fit_many_then(
one_transfer, one_transfer, empty=True
)
capacity = f"{shared_domain.capacity_bytes} shared bytes"
requirement = f"{2 * one_transfer_bytes} shared bytes"
else:
enough_capacity = swa._swa_kv_pool_host.size >= 2 * window_tokens
capacity = f"{swa._swa_kv_pool_host.size} SWA tokens"
enough_capacity = pool.size >= 2 * window_tokens
capacity = f"{pool.size} SWA tokens"
requirement = f"{2 * window_tokens} SWA tokens"
if not enough_capacity:
raise ValueError(
@@ -275,15 +279,15 @@ class BufferModePipeline:
def __init__(
self,
cache: UnifiedRadixCache,
swa_window_pages: int,
swa_component: Optional[SWAComponent],
write_backlog_cap: int,
max_context_len: int = 0,
*,
auxiliary_swa_component: Optional[SWAComponent] = None,
):
self._cache = cache
# SWA window size in KV pages when the SWA component stages through
# a host pool (0 = KV-only: no trailing window staged). Static after
# pool assembly.
self._swa_window_pages = swa_window_pages
self._swa_component = swa_component
self._auxiliary_swa_component = auxiliary_swa_component
# Metadata-only pending-write backlog cap, measured in unique FULL
# snapshot spans; beyond it new spans are dropped at admission.
self.write_backlog_cap = write_backlog_cap
@@ -352,6 +356,13 @@ class BufferModePipeline:
self.anchor_locked_tokens_ = 0
self._anchor_lock_cap_skips = 0
def _swa_components(self):
return (
component
for component in (self._swa_component, self._auxiliary_swa_component)
if component is not None
)
def _shared_host_domain(self):
cc = self._cache.cache_controller
anchor = cc.mem_pool_host.anchor_entry.host_pool
@@ -421,16 +432,16 @@ class BufferModePipeline:
def _shared_load_reserve_requests(self) -> list[tuple[str, int]]:
cc = self._cache.cache_controller
anchor = cc.mem_pool_host.anchor_entry.host_pool
swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA)
min_full_tokens = _minimum_full_transfer_tokens(
anchor, self._cache.prefetch_threshold
)
requests = [(anchor.pool_label, min_full_tokens)]
if swa_entry is not None and self._swa_window_pages:
for component in self._swa_components():
entry = cc.mem_pool_host.entry_map[component.pool_name]
requests.append(
(
swa_entry.host_pool.pool_label,
self._swa_window_pages * swa_entry.host_pool.page_size,
entry.host_pool.pool_label,
component.full_window_pages * entry.host_pool.page_size,
)
)
return requests
@@ -460,15 +471,6 @@ class BufferModePipeline:
or self.anchor_locks
)
def swa_transient_size(self) -> int:
"""SWA destinations kept alive only until an in-flight H2D completes."""
return sum(
len(device_indices)
for load_back in self.ongoing_buffer_load_back.values()
for pool_name, device_indices in load_back.aux_device_releases
if pool_name == PoolName.SWA
)
# ---- backup pipeline (device -> staging -> storage) ----
def _anchor_pool_aux_tokens(self, aux_xfers: list[PoolTransfer]) -> int:
@@ -729,19 +731,26 @@ class BufferModePipeline:
need = len(t.keys) * entry.host_pool.page_size
if entry.host_pool is anchor_pool:
anchor_need += need
elif need > entry.host_pool.size - self._aux_loads_margin(entry.host_pool):
elif need > entry.host_pool.size - self._aux_loads_margin(
t.name, entry.host_pool
):
return True
return anchor_need > cc.mem_pool_host.size
def _aux_loads_margin(self, host_pool) -> int:
def _aux_loads_margin(self, pool_name, host_pool) -> int:
"""Aux-pool tokens reserved for loads: at least one trailing window
(prepare_prefetch allocates its window here and a failed alloc
forfeits the whole prefetch), plus a 10% burst absorber mirroring
live_cap."""
return max(
self._swa_window_pages * host_pool.page_size,
host_pool.size // 10,
)
window_pages = 0
if pool_name == PoolName.SWA and self._swa_component is not None:
window_pages = self._swa_component.full_window_pages
elif (
pool_name == PoolName.AUXILIARY_SWA
and self._auxiliary_swa_component is not None
):
window_pages = self._auxiliary_swa_component.full_window_pages
return max(window_pages * host_pool.page_size, host_pool.size // 10)
def _validate_backup_intent(
self, intent: _UnifiedBackupIntent
@@ -1081,7 +1090,7 @@ class BufferModePipeline:
continue
need = len(t.keys) * entry.host_pool.page_size
headroom = entry.host_pool.available_size() - self._aux_loads_margin(
entry.host_pool
t.name, entry.host_pool
)
if need > headroom:
return True
@@ -1444,7 +1453,7 @@ class BufferModePipeline:
self._resolve_device_covered(req)
return True
req.host_hit_length = plan.full_tokens
req.swa_host_hit_length = plan.swa_tokens
req.swa_host_hit_length = plan.window_tokens(ComponentType.SWA)
req.mamba_host_hit_length = plan.mamba_slots
# The device alone serves only this pass's joint match; the rest of the
# span, fetched FULL or resident FULL the aux tail unlocks, is storage's.
@@ -1465,21 +1474,46 @@ class BufferModePipeline:
self, f: _StagedPrefetch, key: RadixKey, matched_len: int
) -> Optional[StagedPrefetchPlan]:
full_tokens = max(0, f.matched_len + f.num_tokens - matched_len)
swa_tokens = sum(
len(t.host_indices)
for t in f.aux_xfers
if t.name == PoolName.SWA and t.host_indices is not None
)
span_end = f.matched_len + f.num_tokens
windows = {}
for component in self._swa_components():
staged_tokens = sum(
len(t.host_indices)
for t in f.aux_xfers
if t.name == component.pool_name and t.host_indices is not None
)
window_start = span_end - staged_tokens
repair_end = min(matched_len, span_end)
missing_ranges = (
self._cache.tree_core.swa_tombstone_ranges(
key,
window_start,
repair_end,
component_type=component.component_type,
)
if window_start < repair_end
else []
)
tail_start = max(matched_len, window_start)
if tail_start < span_end:
missing_ranges.append((tail_start, span_end))
windows[component.component_type] = WindowLoadBackPlan(
window_start, span_end, tuple(missing_ranges)
)
mamba_slots = sum(
len(t.host_indices)
for t in f.aux_xfers
if t.name == PoolName.MAMBA and t.host_indices is not None
)
if full_tokens == 0 and swa_tokens == 0 and mamba_slots == 0:
if (
full_tokens == 0
and mamba_slots == 0
and not any(w.num_tokens for w in windows.values())
):
# The device already covers every pool's remaining demand.
return None
return StagedPrefetchPlan(
f.operation_id, key, matched_len, full_tokens, swa_tokens, mamba_slots
f.operation_id, key, matched_len, full_tokens, windows, mamba_slots
)
def _resolve_device_covered(self, req: Req) -> None:
@@ -1666,7 +1700,7 @@ class BufferModePipeline:
req.mamba_host_hit_length,
) == (
plan.full_tokens,
plan.swa_tokens,
plan.window_tokens(ComponentType.SWA),
plan.mamba_slots,
), f"staged load-back budget changed for {req.rid}"
@@ -1682,17 +1716,32 @@ class BufferModePipeline:
key = plan.key
span_end = f.matched_len + f.num_tokens
load_back_id = -(f.operation_id) - 1
# The full trailing-window aux transfer is independent of the shorter
# FULL suffix and may remain nonempty for an aux-only load.
# Preserve the complete bounce in f until ack; H2D uses only the rows
# admitted by the plan, including repairs inside the resident FULL prefix.
load_xfers = list(f.aux_xfers)
staged_swa = next(
(
len(t.host_indices)
for t in load_xfers
if t.name == PoolName.SWA and t.host_indices is not None
),
0,
)
for component in self._swa_components():
pool_name = component.pool_name
window = plan.windows[component.component_type]
if not window.missing_ranges:
load_xfers = [
t
for t in load_xfers
if t.name != pool_name and t.indices_from_pool != pool_name
]
continue
host_parts = [
slice(start - window.start, end - window.start)
for start, end in window.missing_ranges
]
for i, transfer in enumerate(load_xfers):
if transfer.name != pool_name:
continue
load_xfers[i] = replace(
transfer,
host_indices=torch.cat(
[transfer.host_indices[part] for part in host_parts]
),
)
state, failure_pool = self._allocate_and_queue_load_back(
req, f, plan, load_back_id, load_xfers
@@ -1709,71 +1758,43 @@ class BufferModePipeline:
request, credited_tokens=req.storage_hit_length
)
swa_dev = next(
(
t.device_indices
for t in state.load_transfers
if t.name == PoolName.SWA
and t.device_indices is not None
and t.device_indices.numel() > 0
),
None,
)
if swa_dev is not None:
# SWA loads into fresh reservations. Register the window's FULL->SWA
# translation now (attention reads through it); slots another request
# still holds are kept, and their redundant destinations freed at ack.
full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :]
allocator = cache.token_to_kv_pool_allocator
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
missing = old_swa <= 0
window_start = span_end - len(swa_dev)
repair_end = min(splice_base, span_end)
tree_missing = torch.zeros_like(missing)
tail_start = max(splice_base, window_start)
tree_missing[tail_start - window_start :] = True
repair_ranges = []
if window_start < repair_end:
repair_ranges = cache.tree_core.swa_tombstone_ranges(
key, window_start, repair_end
)
for repair_start, repair_end_ in repair_ranges:
repair_slice = slice(
repair_start - window_start, repair_end_ - window_start
)
tree_missing[repair_slice] = True
assert torch.equal(missing, tree_missing), (
"SWA tree and allocator residency disagree for restored window "
f"[{window_start}, {span_end})"
full_indices = torch.cat([prefix_indices, device_indices])
for component in self._swa_components():
pool_name = component.pool_name
window_dev = next(
(
t.device_indices
for t in state.load_transfers
if t.name == pool_name
and t.device_indices is not None
and t.device_indices.numel() > 0
),
None,
)
for repair_start, repair_end_ in repair_ranges:
repair_slice = slice(
repair_start - window_start, repair_end_ - window_start
)
for action in cache.tree_core.attach_swa_window(
key,
repair_start,
repair_end_,
swa_dev[repair_slice],
):
cache._apply_cache_action(action)
if bool(missing.any()):
cache._apply_cache_action(
RebuildFullToSWAMapping(
[full_window[missing]],
[swa_dev[missing]],
)
)
if bool((~missing).any()):
state.aux_device_releases.append((PoolName.SWA, swa_dev[~missing]))
if window_dev is None:
continue
ct = component.component_type
window = plan.windows[ct]
assert len(window_dev) == window.num_tokens
full_parts = []
offset = 0
for start, end in window.missing_ranges:
full_part = full_indices[start:end]
full_parts.append(full_part)
if end <= splice_base:
for action in cache.tree_core.attach_swa_window(
key,
start,
end,
window_dev[offset : offset + end - start],
component_type=ct,
):
cache._apply_cache_action(action)
offset += end - start
component.restore_buffer_mapping(torch.cat(full_parts), window_dev)
insert_result = self._insert_loaded_prefix(
state,
req,
plan,
prefix_indices=prefix_indices,
span_end=span_end,
staged_swa=staged_swa,
state, req, plan, prefix_indices=prefix_indices
)
self.ongoing_buffer_load_back[load_back_id] = state
match = cache.match_prefix(MatchPrefixParams(key=key))
@@ -1871,8 +1892,6 @@ class BufferModePipeline:
plan: StagedPrefetchPlan,
*,
prefix_indices: torch.Tensor,
span_end: int,
staged_swa: int,
) -> InsertResult:
# Publish via a plain insert under the admission lock choreography;
# the caller's request lock then pins the span (load_back pattern).
@@ -1887,7 +1906,7 @@ class BufferModePipeline:
value=torch.cat([prefix_indices, state.device_indices]),
prev_prefix_len=plan.device_prefix_len,
component_evicted_seqlens={
ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0
ct: window.start for ct, window in plan.windows.items()
},
**insert_fields,
)
@@ -1925,25 +1944,13 @@ class BufferModePipeline:
return True
def _free_completed_load_back_buffers(self, f: _OngoingBufferLoadBack) -> None:
cache = self._cache
cc = cache.cache_controller
cc = self._cache.cache_controller
# The H2D consumed the bounce buffers; free them outright.
self._free_staging_now(f.host_indices, f.aux_xfers)
for pool_name, device_indices in f.aux_device_releases:
entry = cc.mem_pool_host.entry_map[pool_name]
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWAAllocatorBase,
)
allocator = cache.token_to_kv_pool_allocator
if pool_name == PoolName.SWA and isinstance(
allocator, UnifiedSWAAllocatorBase
):
# H2D has completed; these redundant slots are no longer pending.
allocator.swa_attn_allocator.free_physical(device_indices)
else:
free_fn = entry.device_free_fn or entry.device_pool.free
free_fn(device_indices)
free_fn = entry.device_free_fn or entry.device_pool.free
free_fn(device_indices)
def release_staged_hold(
self, request: CacheRequestHandle, reason: Optional[str] = None
@@ -56,6 +56,7 @@ class PoolName(str, Enum):
KV = "kv"
MAMBA = "mamba"
SWA = "swa"
AUXILIARY_SWA = "auxiliary_swa"
INDEXER = "indexer"
# TODO(hzh0425): Current DeepSeek V4 pool naming is verbose; will be normalized to
# 'COMPRESSED_KV / COMPRESSED_INDEXER / COMPRESSED_STATE' in the next PR.
@@ -120,6 +120,7 @@ class PrefillBudget:
input_tokens: int,
swa_host_hit_length: int,
chunk_limit: int | None,
auxiliary_swa_host_hit_length: int = 0,
) -> tuple[bool, int | None]:
"""Return admission feasibility and a memory bound on the requested chunk."""
return (
@@ -235,6 +236,7 @@ class SWAPrefillBudget(PrefillBudget):
input_tokens: int,
swa_host_hit_length: int,
chunk_limit: int | None,
auxiliary_swa_host_hit_length: int = 0,
) -> tuple[bool, int | None]:
if total_tokens >= self.remaining_total:
return False, None
@@ -398,6 +400,7 @@ class SharedSWAPrefillBudget(SWAPrefillBudget):
input_tokens: int,
swa_host_hit_length: int,
chunk_limit: int | None,
auxiliary_swa_host_hit_length: int = 0,
) -> tuple[bool, int | None]:
needed = self.swa_tokens(
extend_input_len,
@@ -520,8 +520,14 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
raise NotImplementedError("root_node: not yet ported to the Rust tree core")
def swa_tombstone_ranges(
self, key: RadixKey, start: int, end: int
self,
key: RadixKey,
start: int,
end: int,
component_type: ComponentType = ComponentType.SWA,
) -> list[tuple[int, int]]:
if component_type != ComponentType.SWA:
raise NotImplementedError("The Rust tree core supports one SWA window")
return self._binding.swa_tombstone_ranges(
self._bindings.MatchParamsBinding(
key=_radix_key_buffer(key),
@@ -538,7 +544,10 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
window_start: int,
window_end: int,
swa_values: torch.Tensor,
component_type: ComponentType = ComponentType.SWA,
) -> list[CacheAction | ComponentAction]:
if component_type != ComponentType.SWA:
raise NotImplementedError("The Rust tree core supports one SWA window")
return _cache_actions_from_tagged(
self._binding.attach_swa_window(
self._bindings.MatchParamsBinding(
@@ -3,6 +3,10 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import TYPE_CHECKING, Optional
import msgspec
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.mem_cache.radix_cache import RadixKey
@@ -74,13 +78,29 @@ class StoragePrefetchRetries:
return ready
class WindowLoadBackPlan(msgspec.Struct, frozen=True):
"""Logical coverage and the missing subranges that need new device rows."""
start: int
end: int
missing_ranges: tuple[tuple[int, int], ...]
@property
def num_tokens(self) -> int:
return sum(end - start for start, end in self.missing_ranges)
@dataclass(frozen=True)
class StagedPrefetchPlan:
"""One admission pass, computed before promoting the joint FULL/SWA match."""
"""One admission pass, computed before promoting the joint component match."""
operation_id: int
key: RadixKey
device_prefix_len: int
full_tokens: int
swa_tokens: int
windows: dict[ComponentType, WindowLoadBackPlan]
mamba_slots: int = 0
def window_tokens(self, component_type: ComponentType) -> int:
window = self.windows.get(component_type)
return window.num_tokens if window is not None else 0
@@ -39,6 +39,7 @@ if TYPE_CHECKING:
CacheAction,
ComponentAction,
)
from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore
from sglang.srt.mem_cache.unified_radix_cache import (
NodeId,
@@ -779,16 +780,18 @@ class TreeComponent(ABC):
def create_buffer_pipeline(
self,
*,
swa_window_pages: int,
swa_component: Optional[SWAComponent],
write_backlog_cap: int,
max_context_len: int = 0,
auxiliary_swa_component: Optional[SWAComponent] = None,
) -> BufferModePipeline:
"""Called on the FULL component to construct the buffer-mode pipeline."""
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
return BufferModePipeline(
cache=self.cache,
swa_window_pages=swa_window_pages,
swa_component=swa_component,
auxiliary_swa_component=auxiliary_swa_component,
write_backlog_cap=write_backlog_cap,
max_context_len=max_context_len,
)
@@ -97,8 +97,14 @@ class SWAComponent(TreeComponent):
self._swa_kv_pool_host = None
component_type = ComponentType.SWA
pool_name = PoolName.SWA
_independent = False
def restore_buffer_mapping(self, full_indices, window_indices):
self.cache._apply_cache_action(
RebuildFullToSWAMapping([full_indices], [window_indices])
)
def _collect_unbacked_swa_nodes(
self, node: UnifiedTreeNode
) -> list[UnifiedTreeNode]:
@@ -1074,7 +1080,7 @@ class SWAComponent(TreeComponent):
assert self._swa_kv_pool_host is not None
host_indices = self._swa_kv_pool_host.alloc(num_tokens)
if host_indices is None:
self.cache.evict_host(num_tokens, ComponentType.SWA)
self.cache.evict_host(num_tokens, self.component_type)
host_indices = self._swa_kv_pool_host.alloc(num_tokens)
return host_indices
@@ -1087,7 +1093,7 @@ class SWAComponent(TreeComponent):
return {}
cd = node.component_data[self.component_type]
if not self.tree_core.has_swa_host_pool or cd.value is None:
return {PoolName.SWA: []}
return {self.pool_name: []}
rows = (
node.component_data[BASE_COMPONENT_TYPE].value
if self._unified_allocator() is not None
@@ -1095,8 +1101,8 @@ class SWAComponent(TreeComponent):
)
num_pages = len(rows) // self._swa_kv_pool_host.page_size
if num_pages == 0 or num_pages > len(hash_values):
return {PoolName.SWA: []}
return {PoolName.SWA: list(hash_values[-num_pages:])}
return {self.pool_name: []}
return {self.pool_name: list(hash_values[-num_pages:])}
def build_hicache_transfers(
self,
@@ -1123,7 +1129,7 @@ class SWAComponent(TreeComponent):
unbacked_swa_nodes.reverse()
return [
PoolTransfer(
name=PoolName.SWA,
name=self.pool_name,
device_indices=(
self._translate_full_to_swa(
torch.cat(
@@ -1172,7 +1178,7 @@ class SWAComponent(TreeComponent):
return [
PoolTransfer(
name=PoolName.SWA,
name=self.pool_name,
host_indices=torch.cat(backed_up),
device_indices=None,
nodes_to_load=[n.id for n in nodes],
@@ -1188,7 +1194,7 @@ class SWAComponent(TreeComponent):
return None
return [
PoolTransfer(
name=PoolName.SWA,
name=self.pool_name,
host_indices=cd.host_value[-num_pages * self.tree_core.page_size :],
keys=node.hash_value[-num_pages:],
hit_policy=PoolHitPolicy.TRAILING_PAGES,
@@ -1203,7 +1209,7 @@ class SWAComponent(TreeComponent):
return None
return [
PoolTransfer(
name=PoolName.SWA,
name=self.pool_name,
keys=["__placeholder__"] * num_pages,
hit_policy=PoolHitPolicy.TRAILING_PAGES,
)
@@ -1229,7 +1235,7 @@ class SWAComponent(TreeComponent):
num_pages = len(value) // page
return PoolTransfer(
name=PoolName.SWA,
name=self.pool_name,
device_indices=value[-num_pages * page :].to(torch.int64),
keys=node.hash_value[-num_pages:],
hit_policy=PoolHitPolicy.TRAILING_PAGES,
@@ -1245,7 +1251,7 @@ class SWAComponent(TreeComponent):
return None
transfer = PoolTransfer(
name=PoolName.SWA,
name=self.pool_name,
keys=tail_keys,
hit_policy=PoolHitPolicy.TRAILING_PAGES,
)
@@ -1408,7 +1414,7 @@ class SWAComponent(TreeComponent):
host_indices.numel() // page_size if host_indices is not None else 0
)
loaded_pages = (
pool_storage_result.extra_pool_hit_pages.get(PoolName.SWA, 0)
pool_storage_result.extra_pool_hit_pages.get(self.pool_name, 0)
if pool_storage_result
else 0
)
@@ -1522,7 +1528,7 @@ class SWAComponent(TreeComponent):
if self._swa_kv_pool_host is None:
return
for host_value in host_values:
self.cache.host_pool_group.free(host_value, pool=PoolName.SWA)
self.cache.host_pool_group.free(host_value, pool=self.pool_name)
def apply_component_action(self, action: ComponentAction) -> None:
alloc = self.cache.token_to_kv_pool_allocator
@@ -1536,7 +1542,7 @@ class SWAComponent(TreeComponent):
if host_indices is not None and host_indices.numel() > 0:
self.cache.cache_controller.append_host_mem_release(
extra_pools=[
PoolTransfer(name=PoolName.SWA, host_indices=host_indices)
PoolTransfer(name=self.pool_name, host_indices=host_indices)
]
)
return
@@ -141,6 +141,9 @@ class StorageAttachment:
sidecar_pool_specs=cache.sidecar_pool_specs,
host_pool_group=cache.host_pool_group,
swa_component=cache.components.get(ComponentType.SWA),
auxiliary_swa_component=cache.components.get(
ComponentType.AUXILIARY_SWA
),
storage_prefetch_threshold=prefetch_threshold,
)
# New workers must see the requested policy from their first operation.
@@ -293,12 +296,14 @@ class StorageAttachment:
def resolve_prefetch_threshold(self, configured: int) -> int:
"""Use the same complete-window minimum for every buffer-mode anchor."""
cache = self._cache
window = cache.sliding_window_size
if not window or cache.host_memory_mode != "buffer_only":
return configured
page_size = cache.page_size
window_tokens = ((window + page_size - 1) // page_size) * page_size
return max(configured, window_tokens)
if cache.host_memory_mode == "buffer_only":
for ct in (ComponentType.SWA, ComponentType.AUXILIARY_SWA):
component = cache.components.get(ct)
if component is not None:
configured = max(
configured, component.full_window_pages * cache.page_size
)
return configured
def _resolve_metrics_collector(
self,
@@ -2786,7 +2786,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
remaining = remaining[prefix_len:]
def swa_tombstone_ranges(
self, key: RadixKey, start: int, end: int
self,
key: RadixKey,
start: int,
end: int,
component_type: ComponentType = ComponentType.SWA,
) -> list[tuple[int, int]]:
"""Return the maximal SWA-tombstoned ranges within ``[start, end)``."""
ranges: list[tuple[int, int]] = []
@@ -2794,7 +2798,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
seg_end = pos + prefix_len
if seg_end <= start:
continue
if child.component_data[ComponentType.SWA].value is not None:
if child.component_data[component_type].value is not None:
continue
lo, hi = max(start, pos), min(end, seg_end)
if ranges and ranges[-1][1] == lo:
@@ -2811,6 +2815,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
window_start: int,
window_end: int,
swa_values: torch.Tensor,
component_type: ComponentType = ComponentType.SWA,
) -> list[CacheAction | ComponentAction]:
"""Attach a loaded SWA slice to an existing tombstoned tree span."""
assert len(swa_values) == window_end - window_start, (
@@ -2825,7 +2830,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
continue
seg_start = max(pos, window_start)
assign_end = min(seg_end, window_end)
assert child.component_data[ComponentType.SWA].value is None, (
assert child.component_data[component_type].value is None, (
f"attach_swa_window over live SWA at [{seg_start}, {assign_end}) "
f"of [{window_start}, {window_end})"
)
@@ -2836,6 +2841,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
assign_end,
swa_values[seg_start - window_start : assign_end - window_start],
actions,
component_type,
)
covered = assign_end
if covered >= window_end:
@@ -2853,6 +2859,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
seg_end: int,
values: torch.Tensor,
actions: list[CacheAction | ComponentAction],
component_type: ComponentType,
) -> None:
target = node
if seg_start > node_start:
@@ -2864,7 +2871,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
if action is not None:
actions.append(action)
target = fragment
self.set_component_device_value(target.id, ComponentType.SWA, values.clone())
self.set_component_device_value(target.id, component_type, values.clone())
def set_component_device_value(
self, node_id: NodeId, component_type: ComponentType, value: torch.Tensor
@@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Optional, Sequence
import msgspec
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
# Tree node id -- the node handle used outside the TreeCore. The concrete tree
# node is a TreeCore-internal type.
NodeId = int
@@ -151,7 +153,6 @@ if TYPE_CHECKING:
)
from sglang.srt.mem_cache.unified_cache.components import (
CacheTransferPhase,
ComponentType,
)
from sglang.srt.mem_cache.unified_cache.unified_tree_core import (
StorageBackupSpec,
@@ -728,7 +729,11 @@ class UnifiedTreeCoreInterface(ABC):
@abstractmethod
def swa_tombstone_ranges(
self, key: RadixKey, start: int, end: int
self,
key: RadixKey,
start: int,
end: int,
component_type: ComponentType = ComponentType.SWA,
) -> list[tuple[int, int]]:
"""Return maximal missing SWA ranges within the matched [start, end) span."""
...
@@ -740,6 +745,7 @@ class UnifiedTreeCoreInterface(ABC):
window_start: int,
window_end: int,
swa_values: torch.Tensor,
component_type: ComponentType = ComponentType.SWA,
) -> list[CacheAction | ComponentAction]:
"""Attach a loaded SWA window to tombstoned spans, returning split actions.
@@ -427,11 +427,16 @@ class UnifiedRadixCache(BasePrefixCache):
"""Initialize HiCache infrastructure."""
self.host_memory_mode = get_memory().hicache_host_memory_mode
if self.host_memory_mode == "buffer_only":
supported = {ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA}
supported = {
ComponentType.FULL,
ComponentType.SWA,
ComponentType.AUXILIARY_SWA,
ComponentType.MAMBA,
}
if not set(self.tree_components) <= supported:
raise ValueError(
"--hicache-host-memory-mode buffer_only supports only "
"FULL/SWA/MAMBA unified trees; got components "
"FULL/SWA/AUXILIARY_SWA/MAMBA unified trees; got components "
f"{sorted(ct.name for ct in self.tree_components)}."
)
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
@@ -490,21 +495,20 @@ class UnifiedRadixCache(BasePrefixCache):
if self.host_memory_mode == "buffer_only":
self.tree_core.set_host_memory_buffer_only()
swa = self.components.get(ComponentType.SWA)
auxiliary_swa = self.components.get(ComponentType.AUXILIARY_SWA)
validate_buffer_only_stack(
sidecar_pool_specs=self.sidecar_pool_specs,
host_pool_group=self.host_pool_group,
swa_component=swa,
auxiliary_swa_component=auxiliary_swa,
storage_prefetch_threshold=storage_prefetch_threshold,
)
self.buffer_pipeline = self.components[
ComponentType.FULL
].create_buffer_pipeline(
max_context_len=get_model().context_length or 0,
swa_window_pages=(
swa.full_window_pages
if swa is not None and self.tree_core.has_swa_host_pool
else 0
),
swa_component=swa,
auxiliary_swa_component=auxiliary_swa,
# Leak backstop only: the queue is charged once per unique node
# span across pools; a cap binding on live content would
# drop-newest and punch storage holes.
@@ -3676,11 +3680,6 @@ class UnifiedRadixCache(BasePrefixCache):
def swa_protected_size(self) -> int:
return self.tree_core.swa_protected_size()
def swa_transient_size(self) -> int:
if self.buffer_pipeline is None:
return 0
return self.buffer_pipeline.swa_transient_size()
def mamba_protected_size(self) -> int:
return self.tree_core.mamba_protected_size()