[HiCache] Buffer-only L3 for an auxiliary SWA window; load back only missing window rows (#43932)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: houseroad <houseroad@users.noreply.github.com> Co-authored-by: xiezhq-hermann <xiezhq-hermann@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
houseroad
xiezhq-hermann
parent
aa8793f33b
commit
e7d6a88bfc
@@ -1480,12 +1480,20 @@ class Req(ReqDllmMixin):
|
||||
return self.output_ids[: self.finished_len]
|
||||
return self.output_ids
|
||||
|
||||
@property
|
||||
def auxiliary_swa_host_hit_length(self) -> int:
|
||||
plan = self.staged_prefetch_plan
|
||||
return (
|
||||
plan.window_tokens(ComponentType.AUXILIARY_SWA) if plan is not None else 0
|
||||
)
|
||||
|
||||
def needs_host_load_back(self) -> bool:
|
||||
"""Whether any cache layer has a host hit that needs L2 H2D load_back."""
|
||||
return (
|
||||
self.host_hit_length > 0
|
||||
or self.swa_host_hit_length > 0
|
||||
or self.mamba_host_hit_length > 0
|
||||
or self.auxiliary_swa_host_hit_length > 0
|
||||
)
|
||||
|
||||
def materialized_host_hit_len(self) -> int:
|
||||
|
||||
@@ -795,6 +795,7 @@ class PrefillAdder:
|
||||
input_tokens=len(req.full_untruncated_fill_ids),
|
||||
swa_host_hit_length=swa_host_hit_length,
|
||||
chunk_limit=self.rem_chunk_tokens,
|
||||
auxiliary_swa_host_hit_length=req.auxiliary_swa_host_hit_length,
|
||||
)
|
||||
|
||||
def _mamba_gap_budget_for_req(
|
||||
@@ -1333,6 +1334,16 @@ class PrefillAdder:
|
||||
|
||||
return self.budget_state()
|
||||
|
||||
def _defer_staged_for_capacity_if_idle(self, req: Req, *, pool: str) -> None:
|
||||
# Running work and admitted prefills (including continuing chunks) can
|
||||
# release capacity. Only idle budget failures should force recomputation.
|
||||
if self.can_run_list or (
|
||||
self.running_batch is not None and self.running_batch.reqs
|
||||
):
|
||||
return
|
||||
if self.tree_cache.buffer_pipeline is not None:
|
||||
self.tree_cache.buffer_pipeline.defer_staged_admission(req, pool)
|
||||
|
||||
def add_one_req(
|
||||
self, req: Req, has_chunked_req: bool, truncation_align_size: Optional[int]
|
||||
):
|
||||
@@ -1379,6 +1390,8 @@ class PrefillAdder:
|
||||
has_chunked_req=has_chunked_req,
|
||||
)
|
||||
if isinstance(admission, AddReqResult):
|
||||
if admission is AddReqResult.NO_TOKEN:
|
||||
self._defer_staged_for_capacity_if_idle(req, pool="prefill_budget")
|
||||
return admission
|
||||
|
||||
# A rejected candidate must not report prefillable or queue H2D.
|
||||
@@ -1435,6 +1448,9 @@ class PrefillAdder:
|
||||
swa_host_hit_length=req.swa_host_hit_length,
|
||||
chunk_limit=self.rem_chunk_tokens,
|
||||
):
|
||||
self._defer_staged_for_capacity_if_idle(
|
||||
req, pool="load_back_budget"
|
||||
)
|
||||
return AddReqResult.NO_TOKEN
|
||||
promised_host_hit = req.host_hit_length
|
||||
loaded = self.tree_cache.init_load_back(
|
||||
|
||||
@@ -28,6 +28,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
from array import array
|
||||
from collections import deque
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import msgspec
|
||||
@@ -52,8 +53,10 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.base import HostKVCache
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.storage_prefetch import StagedPrefetchPlan
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import RebuildFullToSWAMapping
|
||||
from sglang.srt.mem_cache.storage_prefetch import (
|
||||
StagedPrefetchPlan,
|
||||
WindowLoadBackPlan,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components import (
|
||||
CacheTransferPhase,
|
||||
ComponentType,
|
||||
@@ -146,10 +149,11 @@ class _StagedPrefetch(msgspec.Struct):
|
||||
class _OngoingBufferLoadBack(msgspec.Struct):
|
||||
"""A queued H2D load, registered after its prefix is inserted.
|
||||
|
||||
The full host bounce and redundant auxiliary device slots remain owned
|
||||
here until the copy completes. ``load_transfers`` is the transfer list
|
||||
submitted to H2D; ``aux_xfers`` retains the original staging owners and
|
||||
``load_contexts`` each component's part in the load.
|
||||
The full host bounce, and any redundant device slots a component's load
|
||||
context reports, remain owned here until the copy completes.
|
||||
``load_transfers`` is the transfer list submitted to H2D; ``aux_xfers``
|
||||
retains the original staging owners and ``load_contexts`` each
|
||||
component's part in the load.
|
||||
"""
|
||||
|
||||
request: CacheRequestHandle
|
||||
@@ -176,6 +180,8 @@ def validate_buffer_only_stack(
|
||||
host_pool_group: HostPoolGroup,
|
||||
swa_component: Optional[SWAComponent],
|
||||
storage_prefetch_threshold: int = 256,
|
||||
*,
|
||||
auxiliary_swa_component: Optional[SWAComponent] = None,
|
||||
) -> None:
|
||||
"""Post-assembly buffer-mode fences.
|
||||
|
||||
@@ -202,50 +208,48 @@ def validate_buffer_only_stack(
|
||||
f"pool={spec.pool_name}, host_slots={sidecar_size}, "
|
||||
f"source={spec.indices_from_pool}, source_slots={source_size}."
|
||||
)
|
||||
swa = swa_component
|
||||
if swa is not None and swa._swa_kv_pool_host is None:
|
||||
# Only reachable on SWA models with the unified_kv layout (SWA as
|
||||
# a device-only ring): without a host pool the window can neither
|
||||
# stage for writes nor fetch for load-backs.
|
||||
raise ValueError(
|
||||
"--hicache-host-memory-mode buffer_only on SWA models "
|
||||
"requires an SWA host staging pool; the unified_kv layout "
|
||||
"keeps SWA as a device-only ring."
|
||||
)
|
||||
if swa is not None and swa._swa_kv_pool_host is not None:
|
||||
# Below two windows the pool cannot hold a staging write AND the
|
||||
# loads-priority reserve (_aux_loads_margin floors at one
|
||||
# window), so every window-carrying intent would be dropped as
|
||||
# oversize and SWA storage coverage would silently be zero.
|
||||
window_tokens = swa.full_window_pages * swa._swa_kv_pool_host.page_size
|
||||
window_pools = []
|
||||
for component in (swa_component, auxiliary_swa_component):
|
||||
if component is None:
|
||||
continue
|
||||
pool = component._swa_kv_pool_host
|
||||
if pool is None:
|
||||
raise ValueError(
|
||||
"--hicache-host-memory-mode buffer_only on SWA models "
|
||||
"requires an SWA host staging pool; the unified_kv layout "
|
||||
"keeps SWA as a device-only ring."
|
||||
)
|
||||
window_pools.append((pool, component.full_window_pages * pool.page_size))
|
||||
|
||||
for pool, window_tokens in window_pools:
|
||||
# Every window needs a write bounce plus a loads-priority reserve.
|
||||
# For a shared arena the reserve must hold all windows together.
|
||||
shared_domain = (
|
||||
swa._swa_kv_pool_host.shared_allocation_domain
|
||||
if isinstance(swa._swa_kv_pool_host, HostKVCache)
|
||||
else None
|
||||
pool.shared_allocation_domain if isinstance(pool, HostKVCache) else None
|
||||
)
|
||||
if shared_domain is not None:
|
||||
full_host_pool = (
|
||||
swa.cache.cache_controller.mem_pool_host.anchor_entry.host_pool
|
||||
)
|
||||
full_host_pool = host_pool_group.anchor_entry.host_pool
|
||||
min_full_tokens = _minimum_full_transfer_tokens(
|
||||
full_host_pool, storage_prefetch_threshold
|
||||
)
|
||||
one_transfer_bytes = (
|
||||
window_tokens * swa._swa_kv_pool_host.size_per_token
|
||||
+ min_full_tokens * full_host_pool.size_per_token
|
||||
)
|
||||
one_transfer = [
|
||||
(full_host_pool.pool_label, min_full_tokens),
|
||||
(swa._swa_kv_pool_host.pool_label, window_tokens),
|
||||
shared_pools = [
|
||||
(host, tokens)
|
||||
for host, tokens in [(full_host_pool, min_full_tokens), *window_pools]
|
||||
if isinstance(host, HostKVCache)
|
||||
and host.shared_allocation_domain is shared_domain
|
||||
]
|
||||
one_transfer_bytes = sum(
|
||||
tokens * host.size_per_token for host, tokens in shared_pools
|
||||
)
|
||||
one_transfer = [(host.pool_label, tokens) for host, tokens in shared_pools]
|
||||
enough_capacity = shared_domain.can_fit_many_then(
|
||||
one_transfer, one_transfer, empty=True
|
||||
)
|
||||
capacity = f"{shared_domain.capacity_bytes} shared bytes"
|
||||
requirement = f"{2 * one_transfer_bytes} shared bytes"
|
||||
else:
|
||||
enough_capacity = swa._swa_kv_pool_host.size >= 2 * window_tokens
|
||||
capacity = f"{swa._swa_kv_pool_host.size} SWA tokens"
|
||||
enough_capacity = pool.size >= 2 * window_tokens
|
||||
capacity = f"{pool.size} SWA tokens"
|
||||
requirement = f"{2 * window_tokens} SWA tokens"
|
||||
if not enough_capacity:
|
||||
raise ValueError(
|
||||
@@ -275,15 +279,15 @@ class BufferModePipeline:
|
||||
def __init__(
|
||||
self,
|
||||
cache: UnifiedRadixCache,
|
||||
swa_window_pages: int,
|
||||
swa_component: Optional[SWAComponent],
|
||||
write_backlog_cap: int,
|
||||
max_context_len: int = 0,
|
||||
*,
|
||||
auxiliary_swa_component: Optional[SWAComponent] = None,
|
||||
):
|
||||
self._cache = cache
|
||||
# SWA window size in KV pages when the SWA component stages through
|
||||
# a host pool (0 = KV-only: no trailing window staged). Static after
|
||||
# pool assembly.
|
||||
self._swa_window_pages = swa_window_pages
|
||||
self._swa_component = swa_component
|
||||
self._auxiliary_swa_component = auxiliary_swa_component
|
||||
# Metadata-only pending-write backlog cap, measured in unique FULL
|
||||
# snapshot spans; beyond it new spans are dropped at admission.
|
||||
self.write_backlog_cap = write_backlog_cap
|
||||
@@ -352,6 +356,13 @@ class BufferModePipeline:
|
||||
self.anchor_locked_tokens_ = 0
|
||||
self._anchor_lock_cap_skips = 0
|
||||
|
||||
def _swa_components(self):
|
||||
return (
|
||||
component
|
||||
for component in (self._swa_component, self._auxiliary_swa_component)
|
||||
if component is not None
|
||||
)
|
||||
|
||||
def _shared_host_domain(self):
|
||||
cc = self._cache.cache_controller
|
||||
anchor = cc.mem_pool_host.anchor_entry.host_pool
|
||||
@@ -421,16 +432,16 @@ class BufferModePipeline:
|
||||
def _shared_load_reserve_requests(self) -> list[tuple[str, int]]:
|
||||
cc = self._cache.cache_controller
|
||||
anchor = cc.mem_pool_host.anchor_entry.host_pool
|
||||
swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA)
|
||||
min_full_tokens = _minimum_full_transfer_tokens(
|
||||
anchor, self._cache.prefetch_threshold
|
||||
)
|
||||
requests = [(anchor.pool_label, min_full_tokens)]
|
||||
if swa_entry is not None and self._swa_window_pages:
|
||||
for component in self._swa_components():
|
||||
entry = cc.mem_pool_host.entry_map[component.pool_name]
|
||||
requests.append(
|
||||
(
|
||||
swa_entry.host_pool.pool_label,
|
||||
self._swa_window_pages * swa_entry.host_pool.page_size,
|
||||
entry.host_pool.pool_label,
|
||||
component.full_window_pages * entry.host_pool.page_size,
|
||||
)
|
||||
)
|
||||
return requests
|
||||
@@ -460,15 +471,6 @@ class BufferModePipeline:
|
||||
or self.anchor_locks
|
||||
)
|
||||
|
||||
def swa_transient_size(self) -> int:
|
||||
"""SWA destinations kept alive only until an in-flight H2D completes."""
|
||||
return sum(
|
||||
len(device_indices)
|
||||
for load_back in self.ongoing_buffer_load_back.values()
|
||||
for pool_name, device_indices in load_back.aux_device_releases
|
||||
if pool_name == PoolName.SWA
|
||||
)
|
||||
|
||||
# ---- backup pipeline (device -> staging -> storage) ----
|
||||
|
||||
def _anchor_pool_aux_tokens(self, aux_xfers: list[PoolTransfer]) -> int:
|
||||
@@ -729,19 +731,26 @@ class BufferModePipeline:
|
||||
need = len(t.keys) * entry.host_pool.page_size
|
||||
if entry.host_pool is anchor_pool:
|
||||
anchor_need += need
|
||||
elif need > entry.host_pool.size - self._aux_loads_margin(entry.host_pool):
|
||||
elif need > entry.host_pool.size - self._aux_loads_margin(
|
||||
t.name, entry.host_pool
|
||||
):
|
||||
return True
|
||||
return anchor_need > cc.mem_pool_host.size
|
||||
|
||||
def _aux_loads_margin(self, host_pool) -> int:
|
||||
def _aux_loads_margin(self, pool_name, host_pool) -> int:
|
||||
"""Aux-pool tokens reserved for loads: at least one trailing window
|
||||
(prepare_prefetch allocates its window here and a failed alloc
|
||||
forfeits the whole prefetch), plus a 10% burst absorber mirroring
|
||||
live_cap."""
|
||||
return max(
|
||||
self._swa_window_pages * host_pool.page_size,
|
||||
host_pool.size // 10,
|
||||
)
|
||||
window_pages = 0
|
||||
if pool_name == PoolName.SWA and self._swa_component is not None:
|
||||
window_pages = self._swa_component.full_window_pages
|
||||
elif (
|
||||
pool_name == PoolName.AUXILIARY_SWA
|
||||
and self._auxiliary_swa_component is not None
|
||||
):
|
||||
window_pages = self._auxiliary_swa_component.full_window_pages
|
||||
return max(window_pages * host_pool.page_size, host_pool.size // 10)
|
||||
|
||||
def _validate_backup_intent(
|
||||
self, intent: _UnifiedBackupIntent
|
||||
@@ -1081,7 +1090,7 @@ class BufferModePipeline:
|
||||
continue
|
||||
need = len(t.keys) * entry.host_pool.page_size
|
||||
headroom = entry.host_pool.available_size() - self._aux_loads_margin(
|
||||
entry.host_pool
|
||||
t.name, entry.host_pool
|
||||
)
|
||||
if need > headroom:
|
||||
return True
|
||||
@@ -1444,7 +1453,7 @@ class BufferModePipeline:
|
||||
self._resolve_device_covered(req)
|
||||
return True
|
||||
req.host_hit_length = plan.full_tokens
|
||||
req.swa_host_hit_length = plan.swa_tokens
|
||||
req.swa_host_hit_length = plan.window_tokens(ComponentType.SWA)
|
||||
req.mamba_host_hit_length = plan.mamba_slots
|
||||
# The device alone serves only this pass's joint match; the rest of the
|
||||
# span, fetched FULL or resident FULL the aux tail unlocks, is storage's.
|
||||
@@ -1465,21 +1474,46 @@ class BufferModePipeline:
|
||||
self, f: _StagedPrefetch, key: RadixKey, matched_len: int
|
||||
) -> Optional[StagedPrefetchPlan]:
|
||||
full_tokens = max(0, f.matched_len + f.num_tokens - matched_len)
|
||||
swa_tokens = sum(
|
||||
len(t.host_indices)
|
||||
for t in f.aux_xfers
|
||||
if t.name == PoolName.SWA and t.host_indices is not None
|
||||
)
|
||||
span_end = f.matched_len + f.num_tokens
|
||||
windows = {}
|
||||
for component in self._swa_components():
|
||||
staged_tokens = sum(
|
||||
len(t.host_indices)
|
||||
for t in f.aux_xfers
|
||||
if t.name == component.pool_name and t.host_indices is not None
|
||||
)
|
||||
window_start = span_end - staged_tokens
|
||||
repair_end = min(matched_len, span_end)
|
||||
missing_ranges = (
|
||||
self._cache.tree_core.swa_tombstone_ranges(
|
||||
key,
|
||||
window_start,
|
||||
repair_end,
|
||||
component_type=component.component_type,
|
||||
)
|
||||
if window_start < repair_end
|
||||
else []
|
||||
)
|
||||
tail_start = max(matched_len, window_start)
|
||||
if tail_start < span_end:
|
||||
missing_ranges.append((tail_start, span_end))
|
||||
windows[component.component_type] = WindowLoadBackPlan(
|
||||
window_start, span_end, tuple(missing_ranges)
|
||||
)
|
||||
mamba_slots = sum(
|
||||
len(t.host_indices)
|
||||
for t in f.aux_xfers
|
||||
if t.name == PoolName.MAMBA and t.host_indices is not None
|
||||
)
|
||||
if full_tokens == 0 and swa_tokens == 0 and mamba_slots == 0:
|
||||
if (
|
||||
full_tokens == 0
|
||||
and mamba_slots == 0
|
||||
and not any(w.num_tokens for w in windows.values())
|
||||
):
|
||||
# The device already covers every pool's remaining demand.
|
||||
return None
|
||||
return StagedPrefetchPlan(
|
||||
f.operation_id, key, matched_len, full_tokens, swa_tokens, mamba_slots
|
||||
f.operation_id, key, matched_len, full_tokens, windows, mamba_slots
|
||||
)
|
||||
|
||||
def _resolve_device_covered(self, req: Req) -> None:
|
||||
@@ -1666,7 +1700,7 @@ class BufferModePipeline:
|
||||
req.mamba_host_hit_length,
|
||||
) == (
|
||||
plan.full_tokens,
|
||||
plan.swa_tokens,
|
||||
plan.window_tokens(ComponentType.SWA),
|
||||
plan.mamba_slots,
|
||||
), f"staged load-back budget changed for {req.rid}"
|
||||
|
||||
@@ -1682,17 +1716,32 @@ class BufferModePipeline:
|
||||
key = plan.key
|
||||
span_end = f.matched_len + f.num_tokens
|
||||
load_back_id = -(f.operation_id) - 1
|
||||
# The full trailing-window aux transfer is independent of the shorter
|
||||
# FULL suffix and may remain nonempty for an aux-only load.
|
||||
# Preserve the complete bounce in f until ack; H2D uses only the rows
|
||||
# admitted by the plan, including repairs inside the resident FULL prefix.
|
||||
load_xfers = list(f.aux_xfers)
|
||||
staged_swa = next(
|
||||
(
|
||||
len(t.host_indices)
|
||||
for t in load_xfers
|
||||
if t.name == PoolName.SWA and t.host_indices is not None
|
||||
),
|
||||
0,
|
||||
)
|
||||
for component in self._swa_components():
|
||||
pool_name = component.pool_name
|
||||
window = plan.windows[component.component_type]
|
||||
if not window.missing_ranges:
|
||||
load_xfers = [
|
||||
t
|
||||
for t in load_xfers
|
||||
if t.name != pool_name and t.indices_from_pool != pool_name
|
||||
]
|
||||
continue
|
||||
host_parts = [
|
||||
slice(start - window.start, end - window.start)
|
||||
for start, end in window.missing_ranges
|
||||
]
|
||||
for i, transfer in enumerate(load_xfers):
|
||||
if transfer.name != pool_name:
|
||||
continue
|
||||
load_xfers[i] = replace(
|
||||
transfer,
|
||||
host_indices=torch.cat(
|
||||
[transfer.host_indices[part] for part in host_parts]
|
||||
),
|
||||
)
|
||||
|
||||
state, failure_pool = self._allocate_and_queue_load_back(
|
||||
req, f, plan, load_back_id, load_xfers
|
||||
@@ -1709,71 +1758,43 @@ class BufferModePipeline:
|
||||
request, credited_tokens=req.storage_hit_length
|
||||
)
|
||||
|
||||
swa_dev = next(
|
||||
(
|
||||
t.device_indices
|
||||
for t in state.load_transfers
|
||||
if t.name == PoolName.SWA
|
||||
and t.device_indices is not None
|
||||
and t.device_indices.numel() > 0
|
||||
),
|
||||
None,
|
||||
)
|
||||
if swa_dev is not None:
|
||||
# SWA loads into fresh reservations. Register the window's FULL->SWA
|
||||
# translation now (attention reads through it); slots another request
|
||||
# still holds are kept, and their redundant destinations freed at ack.
|
||||
full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :]
|
||||
allocator = cache.token_to_kv_pool_allocator
|
||||
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
|
||||
missing = old_swa <= 0
|
||||
window_start = span_end - len(swa_dev)
|
||||
repair_end = min(splice_base, span_end)
|
||||
tree_missing = torch.zeros_like(missing)
|
||||
tail_start = max(splice_base, window_start)
|
||||
tree_missing[tail_start - window_start :] = True
|
||||
repair_ranges = []
|
||||
if window_start < repair_end:
|
||||
repair_ranges = cache.tree_core.swa_tombstone_ranges(
|
||||
key, window_start, repair_end
|
||||
)
|
||||
for repair_start, repair_end_ in repair_ranges:
|
||||
repair_slice = slice(
|
||||
repair_start - window_start, repair_end_ - window_start
|
||||
)
|
||||
tree_missing[repair_slice] = True
|
||||
assert torch.equal(missing, tree_missing), (
|
||||
"SWA tree and allocator residency disagree for restored window "
|
||||
f"[{window_start}, {span_end})"
|
||||
full_indices = torch.cat([prefix_indices, device_indices])
|
||||
for component in self._swa_components():
|
||||
pool_name = component.pool_name
|
||||
window_dev = next(
|
||||
(
|
||||
t.device_indices
|
||||
for t in state.load_transfers
|
||||
if t.name == pool_name
|
||||
and t.device_indices is not None
|
||||
and t.device_indices.numel() > 0
|
||||
),
|
||||
None,
|
||||
)
|
||||
for repair_start, repair_end_ in repair_ranges:
|
||||
repair_slice = slice(
|
||||
repair_start - window_start, repair_end_ - window_start
|
||||
)
|
||||
for action in cache.tree_core.attach_swa_window(
|
||||
key,
|
||||
repair_start,
|
||||
repair_end_,
|
||||
swa_dev[repair_slice],
|
||||
):
|
||||
cache._apply_cache_action(action)
|
||||
if bool(missing.any()):
|
||||
cache._apply_cache_action(
|
||||
RebuildFullToSWAMapping(
|
||||
[full_window[missing]],
|
||||
[swa_dev[missing]],
|
||||
)
|
||||
)
|
||||
if bool((~missing).any()):
|
||||
state.aux_device_releases.append((PoolName.SWA, swa_dev[~missing]))
|
||||
if window_dev is None:
|
||||
continue
|
||||
ct = component.component_type
|
||||
window = plan.windows[ct]
|
||||
assert len(window_dev) == window.num_tokens
|
||||
full_parts = []
|
||||
offset = 0
|
||||
for start, end in window.missing_ranges:
|
||||
full_part = full_indices[start:end]
|
||||
full_parts.append(full_part)
|
||||
if end <= splice_base:
|
||||
for action in cache.tree_core.attach_swa_window(
|
||||
key,
|
||||
start,
|
||||
end,
|
||||
window_dev[offset : offset + end - start],
|
||||
component_type=ct,
|
||||
):
|
||||
cache._apply_cache_action(action)
|
||||
offset += end - start
|
||||
component.restore_buffer_mapping(torch.cat(full_parts), window_dev)
|
||||
|
||||
insert_result = self._insert_loaded_prefix(
|
||||
state,
|
||||
req,
|
||||
plan,
|
||||
prefix_indices=prefix_indices,
|
||||
span_end=span_end,
|
||||
staged_swa=staged_swa,
|
||||
state, req, plan, prefix_indices=prefix_indices
|
||||
)
|
||||
self.ongoing_buffer_load_back[load_back_id] = state
|
||||
match = cache.match_prefix(MatchPrefixParams(key=key))
|
||||
@@ -1871,8 +1892,6 @@ class BufferModePipeline:
|
||||
plan: StagedPrefetchPlan,
|
||||
*,
|
||||
prefix_indices: torch.Tensor,
|
||||
span_end: int,
|
||||
staged_swa: int,
|
||||
) -> InsertResult:
|
||||
# Publish via a plain insert under the admission lock choreography;
|
||||
# the caller's request lock then pins the span (load_back pattern).
|
||||
@@ -1887,7 +1906,7 @@ class BufferModePipeline:
|
||||
value=torch.cat([prefix_indices, state.device_indices]),
|
||||
prev_prefix_len=plan.device_prefix_len,
|
||||
component_evicted_seqlens={
|
||||
ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0
|
||||
ct: window.start for ct, window in plan.windows.items()
|
||||
},
|
||||
**insert_fields,
|
||||
)
|
||||
@@ -1925,25 +1944,13 @@ class BufferModePipeline:
|
||||
return True
|
||||
|
||||
def _free_completed_load_back_buffers(self, f: _OngoingBufferLoadBack) -> None:
|
||||
cache = self._cache
|
||||
cc = cache.cache_controller
|
||||
cc = self._cache.cache_controller
|
||||
# The H2D consumed the bounce buffers; free them outright.
|
||||
self._free_staging_now(f.host_indices, f.aux_xfers)
|
||||
for pool_name, device_indices in f.aux_device_releases:
|
||||
entry = cc.mem_pool_host.entry_map[pool_name]
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWAAllocatorBase,
|
||||
)
|
||||
|
||||
allocator = cache.token_to_kv_pool_allocator
|
||||
if pool_name == PoolName.SWA and isinstance(
|
||||
allocator, UnifiedSWAAllocatorBase
|
||||
):
|
||||
# H2D has completed; these redundant slots are no longer pending.
|
||||
allocator.swa_attn_allocator.free_physical(device_indices)
|
||||
else:
|
||||
free_fn = entry.device_free_fn or entry.device_pool.free
|
||||
free_fn(device_indices)
|
||||
free_fn = entry.device_free_fn or entry.device_pool.free
|
||||
free_fn(device_indices)
|
||||
|
||||
def release_staged_hold(
|
||||
self, request: CacheRequestHandle, reason: Optional[str] = None
|
||||
|
||||
@@ -56,6 +56,7 @@ class PoolName(str, Enum):
|
||||
KV = "kv"
|
||||
MAMBA = "mamba"
|
||||
SWA = "swa"
|
||||
AUXILIARY_SWA = "auxiliary_swa"
|
||||
INDEXER = "indexer"
|
||||
# TODO(hzh0425): Current DeepSeek V4 pool naming is verbose; will be normalized to
|
||||
# 'COMPRESSED_KV / COMPRESSED_INDEXER / COMPRESSED_STATE' in the next PR.
|
||||
|
||||
@@ -120,6 +120,7 @@ class PrefillBudget:
|
||||
input_tokens: int,
|
||||
swa_host_hit_length: int,
|
||||
chunk_limit: int | None,
|
||||
auxiliary_swa_host_hit_length: int = 0,
|
||||
) -> tuple[bool, int | None]:
|
||||
"""Return admission feasibility and a memory bound on the requested chunk."""
|
||||
return (
|
||||
@@ -235,6 +236,7 @@ class SWAPrefillBudget(PrefillBudget):
|
||||
input_tokens: int,
|
||||
swa_host_hit_length: int,
|
||||
chunk_limit: int | None,
|
||||
auxiliary_swa_host_hit_length: int = 0,
|
||||
) -> tuple[bool, int | None]:
|
||||
if total_tokens >= self.remaining_total:
|
||||
return False, None
|
||||
@@ -398,6 +400,7 @@ class SharedSWAPrefillBudget(SWAPrefillBudget):
|
||||
input_tokens: int,
|
||||
swa_host_hit_length: int,
|
||||
chunk_limit: int | None,
|
||||
auxiliary_swa_host_hit_length: int = 0,
|
||||
) -> tuple[bool, int | None]:
|
||||
needed = self.swa_tokens(
|
||||
extend_input_len,
|
||||
|
||||
@@ -520,8 +520,14 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
raise NotImplementedError("root_node: not yet ported to the Rust tree core")
|
||||
|
||||
def swa_tombstone_ranges(
|
||||
self, key: RadixKey, start: int, end: int
|
||||
self,
|
||||
key: RadixKey,
|
||||
start: int,
|
||||
end: int,
|
||||
component_type: ComponentType = ComponentType.SWA,
|
||||
) -> list[tuple[int, int]]:
|
||||
if component_type != ComponentType.SWA:
|
||||
raise NotImplementedError("The Rust tree core supports one SWA window")
|
||||
return self._binding.swa_tombstone_ranges(
|
||||
self._bindings.MatchParamsBinding(
|
||||
key=_radix_key_buffer(key),
|
||||
@@ -538,7 +544,10 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
window_start: int,
|
||||
window_end: int,
|
||||
swa_values: torch.Tensor,
|
||||
component_type: ComponentType = ComponentType.SWA,
|
||||
) -> list[CacheAction | ComponentAction]:
|
||||
if component_type != ComponentType.SWA:
|
||||
raise NotImplementedError("The Rust tree core supports one SWA window")
|
||||
return _cache_actions_from_tagged(
|
||||
self._binding.attach_swa_window(
|
||||
self._bindings.MatchParamsBinding(
|
||||
|
||||
@@ -3,6 +3,10 @@ from __future__ import annotations
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
@@ -74,13 +78,29 @@ class StoragePrefetchRetries:
|
||||
return ready
|
||||
|
||||
|
||||
class WindowLoadBackPlan(msgspec.Struct, frozen=True):
|
||||
"""Logical coverage and the missing subranges that need new device rows."""
|
||||
|
||||
start: int
|
||||
end: int
|
||||
missing_ranges: tuple[tuple[int, int], ...]
|
||||
|
||||
@property
|
||||
def num_tokens(self) -> int:
|
||||
return sum(end - start for start, end in self.missing_ranges)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StagedPrefetchPlan:
|
||||
"""One admission pass, computed before promoting the joint FULL/SWA match."""
|
||||
"""One admission pass, computed before promoting the joint component match."""
|
||||
|
||||
operation_id: int
|
||||
key: RadixKey
|
||||
device_prefix_len: int
|
||||
full_tokens: int
|
||||
swa_tokens: int
|
||||
windows: dict[ComponentType, WindowLoadBackPlan]
|
||||
mamba_slots: int = 0
|
||||
|
||||
def window_tokens(self, component_type: ComponentType) -> int:
|
||||
window = self.windows.get(component_type)
|
||||
return window.num_tokens if window is not None else 0
|
||||
|
||||
@@ -39,6 +39,7 @@ if TYPE_CHECKING:
|
||||
CacheAction,
|
||||
ComponentAction,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore
|
||||
from sglang.srt.mem_cache.unified_radix_cache import (
|
||||
NodeId,
|
||||
@@ -779,16 +780,18 @@ class TreeComponent(ABC):
|
||||
def create_buffer_pipeline(
|
||||
self,
|
||||
*,
|
||||
swa_window_pages: int,
|
||||
swa_component: Optional[SWAComponent],
|
||||
write_backlog_cap: int,
|
||||
max_context_len: int = 0,
|
||||
auxiliary_swa_component: Optional[SWAComponent] = None,
|
||||
) -> BufferModePipeline:
|
||||
"""Called on the FULL component to construct the buffer-mode pipeline."""
|
||||
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
|
||||
|
||||
return BufferModePipeline(
|
||||
cache=self.cache,
|
||||
swa_window_pages=swa_window_pages,
|
||||
swa_component=swa_component,
|
||||
auxiliary_swa_component=auxiliary_swa_component,
|
||||
write_backlog_cap=write_backlog_cap,
|
||||
max_context_len=max_context_len,
|
||||
)
|
||||
|
||||
@@ -97,8 +97,14 @@ class SWAComponent(TreeComponent):
|
||||
self._swa_kv_pool_host = None
|
||||
|
||||
component_type = ComponentType.SWA
|
||||
pool_name = PoolName.SWA
|
||||
_independent = False
|
||||
|
||||
def restore_buffer_mapping(self, full_indices, window_indices):
|
||||
self.cache._apply_cache_action(
|
||||
RebuildFullToSWAMapping([full_indices], [window_indices])
|
||||
)
|
||||
|
||||
def _collect_unbacked_swa_nodes(
|
||||
self, node: UnifiedTreeNode
|
||||
) -> list[UnifiedTreeNode]:
|
||||
@@ -1074,7 +1080,7 @@ class SWAComponent(TreeComponent):
|
||||
assert self._swa_kv_pool_host is not None
|
||||
host_indices = self._swa_kv_pool_host.alloc(num_tokens)
|
||||
if host_indices is None:
|
||||
self.cache.evict_host(num_tokens, ComponentType.SWA)
|
||||
self.cache.evict_host(num_tokens, self.component_type)
|
||||
host_indices = self._swa_kv_pool_host.alloc(num_tokens)
|
||||
return host_indices
|
||||
|
||||
@@ -1087,7 +1093,7 @@ class SWAComponent(TreeComponent):
|
||||
return {}
|
||||
cd = node.component_data[self.component_type]
|
||||
if not self.tree_core.has_swa_host_pool or cd.value is None:
|
||||
return {PoolName.SWA: []}
|
||||
return {self.pool_name: []}
|
||||
rows = (
|
||||
node.component_data[BASE_COMPONENT_TYPE].value
|
||||
if self._unified_allocator() is not None
|
||||
@@ -1095,8 +1101,8 @@ class SWAComponent(TreeComponent):
|
||||
)
|
||||
num_pages = len(rows) // self._swa_kv_pool_host.page_size
|
||||
if num_pages == 0 or num_pages > len(hash_values):
|
||||
return {PoolName.SWA: []}
|
||||
return {PoolName.SWA: list(hash_values[-num_pages:])}
|
||||
return {self.pool_name: []}
|
||||
return {self.pool_name: list(hash_values[-num_pages:])}
|
||||
|
||||
def build_hicache_transfers(
|
||||
self,
|
||||
@@ -1123,7 +1129,7 @@ class SWAComponent(TreeComponent):
|
||||
unbacked_swa_nodes.reverse()
|
||||
return [
|
||||
PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
name=self.pool_name,
|
||||
device_indices=(
|
||||
self._translate_full_to_swa(
|
||||
torch.cat(
|
||||
@@ -1172,7 +1178,7 @@ class SWAComponent(TreeComponent):
|
||||
|
||||
return [
|
||||
PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
name=self.pool_name,
|
||||
host_indices=torch.cat(backed_up),
|
||||
device_indices=None,
|
||||
nodes_to_load=[n.id for n in nodes],
|
||||
@@ -1188,7 +1194,7 @@ class SWAComponent(TreeComponent):
|
||||
return None
|
||||
return [
|
||||
PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
name=self.pool_name,
|
||||
host_indices=cd.host_value[-num_pages * self.tree_core.page_size :],
|
||||
keys=node.hash_value[-num_pages:],
|
||||
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
||||
@@ -1203,7 +1209,7 @@ class SWAComponent(TreeComponent):
|
||||
return None
|
||||
return [
|
||||
PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
name=self.pool_name,
|
||||
keys=["__placeholder__"] * num_pages,
|
||||
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
||||
)
|
||||
@@ -1229,7 +1235,7 @@ class SWAComponent(TreeComponent):
|
||||
|
||||
num_pages = len(value) // page
|
||||
return PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
name=self.pool_name,
|
||||
device_indices=value[-num_pages * page :].to(torch.int64),
|
||||
keys=node.hash_value[-num_pages:],
|
||||
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
||||
@@ -1245,7 +1251,7 @@ class SWAComponent(TreeComponent):
|
||||
return None
|
||||
|
||||
transfer = PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
name=self.pool_name,
|
||||
keys=tail_keys,
|
||||
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
||||
)
|
||||
@@ -1408,7 +1414,7 @@ class SWAComponent(TreeComponent):
|
||||
host_indices.numel() // page_size if host_indices is not None else 0
|
||||
)
|
||||
loaded_pages = (
|
||||
pool_storage_result.extra_pool_hit_pages.get(PoolName.SWA, 0)
|
||||
pool_storage_result.extra_pool_hit_pages.get(self.pool_name, 0)
|
||||
if pool_storage_result
|
||||
else 0
|
||||
)
|
||||
@@ -1522,7 +1528,7 @@ class SWAComponent(TreeComponent):
|
||||
if self._swa_kv_pool_host is None:
|
||||
return
|
||||
for host_value in host_values:
|
||||
self.cache.host_pool_group.free(host_value, pool=PoolName.SWA)
|
||||
self.cache.host_pool_group.free(host_value, pool=self.pool_name)
|
||||
|
||||
def apply_component_action(self, action: ComponentAction) -> None:
|
||||
alloc = self.cache.token_to_kv_pool_allocator
|
||||
@@ -1536,7 +1542,7 @@ class SWAComponent(TreeComponent):
|
||||
if host_indices is not None and host_indices.numel() > 0:
|
||||
self.cache.cache_controller.append_host_mem_release(
|
||||
extra_pools=[
|
||||
PoolTransfer(name=PoolName.SWA, host_indices=host_indices)
|
||||
PoolTransfer(name=self.pool_name, host_indices=host_indices)
|
||||
]
|
||||
)
|
||||
return
|
||||
|
||||
@@ -141,6 +141,9 @@ class StorageAttachment:
|
||||
sidecar_pool_specs=cache.sidecar_pool_specs,
|
||||
host_pool_group=cache.host_pool_group,
|
||||
swa_component=cache.components.get(ComponentType.SWA),
|
||||
auxiliary_swa_component=cache.components.get(
|
||||
ComponentType.AUXILIARY_SWA
|
||||
),
|
||||
storage_prefetch_threshold=prefetch_threshold,
|
||||
)
|
||||
# New workers must see the requested policy from their first operation.
|
||||
@@ -293,12 +296,14 @@ class StorageAttachment:
|
||||
def resolve_prefetch_threshold(self, configured: int) -> int:
|
||||
"""Use the same complete-window minimum for every buffer-mode anchor."""
|
||||
cache = self._cache
|
||||
window = cache.sliding_window_size
|
||||
if not window or cache.host_memory_mode != "buffer_only":
|
||||
return configured
|
||||
page_size = cache.page_size
|
||||
window_tokens = ((window + page_size - 1) // page_size) * page_size
|
||||
return max(configured, window_tokens)
|
||||
if cache.host_memory_mode == "buffer_only":
|
||||
for ct in (ComponentType.SWA, ComponentType.AUXILIARY_SWA):
|
||||
component = cache.components.get(ct)
|
||||
if component is not None:
|
||||
configured = max(
|
||||
configured, component.full_window_pages * cache.page_size
|
||||
)
|
||||
return configured
|
||||
|
||||
def _resolve_metrics_collector(
|
||||
self,
|
||||
|
||||
@@ -2786,7 +2786,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
remaining = remaining[prefix_len:]
|
||||
|
||||
def swa_tombstone_ranges(
|
||||
self, key: RadixKey, start: int, end: int
|
||||
self,
|
||||
key: RadixKey,
|
||||
start: int,
|
||||
end: int,
|
||||
component_type: ComponentType = ComponentType.SWA,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""Return the maximal SWA-tombstoned ranges within ``[start, end)``."""
|
||||
ranges: list[tuple[int, int]] = []
|
||||
@@ -2794,7 +2798,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
seg_end = pos + prefix_len
|
||||
if seg_end <= start:
|
||||
continue
|
||||
if child.component_data[ComponentType.SWA].value is not None:
|
||||
if child.component_data[component_type].value is not None:
|
||||
continue
|
||||
lo, hi = max(start, pos), min(end, seg_end)
|
||||
if ranges and ranges[-1][1] == lo:
|
||||
@@ -2811,6 +2815,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
window_start: int,
|
||||
window_end: int,
|
||||
swa_values: torch.Tensor,
|
||||
component_type: ComponentType = ComponentType.SWA,
|
||||
) -> list[CacheAction | ComponentAction]:
|
||||
"""Attach a loaded SWA slice to an existing tombstoned tree span."""
|
||||
assert len(swa_values) == window_end - window_start, (
|
||||
@@ -2825,7 +2830,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
continue
|
||||
seg_start = max(pos, window_start)
|
||||
assign_end = min(seg_end, window_end)
|
||||
assert child.component_data[ComponentType.SWA].value is None, (
|
||||
assert child.component_data[component_type].value is None, (
|
||||
f"attach_swa_window over live SWA at [{seg_start}, {assign_end}) "
|
||||
f"of [{window_start}, {window_end})"
|
||||
)
|
||||
@@ -2836,6 +2841,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
assign_end,
|
||||
swa_values[seg_start - window_start : assign_end - window_start],
|
||||
actions,
|
||||
component_type,
|
||||
)
|
||||
covered = assign_end
|
||||
if covered >= window_end:
|
||||
@@ -2853,6 +2859,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
seg_end: int,
|
||||
values: torch.Tensor,
|
||||
actions: list[CacheAction | ComponentAction],
|
||||
component_type: ComponentType,
|
||||
) -> None:
|
||||
target = node
|
||||
if seg_start > node_start:
|
||||
@@ -2864,7 +2871,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
if action is not None:
|
||||
actions.append(action)
|
||||
target = fragment
|
||||
self.set_component_device_value(target.id, ComponentType.SWA, values.clone())
|
||||
self.set_component_device_value(target.id, component_type, values.clone())
|
||||
|
||||
def set_component_device_value(
|
||||
self, node_id: NodeId, component_type: ComponentType, value: torch.Tensor
|
||||
|
||||
@@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Optional, Sequence
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
|
||||
# Tree node id -- the node handle used outside the TreeCore. The concrete tree
|
||||
# node is a TreeCore-internal type.
|
||||
NodeId = int
|
||||
@@ -151,7 +153,6 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.components import (
|
||||
CacheTransferPhase,
|
||||
ComponentType,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.unified_tree_core import (
|
||||
StorageBackupSpec,
|
||||
@@ -728,7 +729,11 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def swa_tombstone_ranges(
|
||||
self, key: RadixKey, start: int, end: int
|
||||
self,
|
||||
key: RadixKey,
|
||||
start: int,
|
||||
end: int,
|
||||
component_type: ComponentType = ComponentType.SWA,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""Return maximal missing SWA ranges within the matched [start, end) span."""
|
||||
...
|
||||
@@ -740,6 +745,7 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
window_start: int,
|
||||
window_end: int,
|
||||
swa_values: torch.Tensor,
|
||||
component_type: ComponentType = ComponentType.SWA,
|
||||
) -> list[CacheAction | ComponentAction]:
|
||||
"""Attach a loaded SWA window to tombstoned spans, returning split actions.
|
||||
|
||||
|
||||
@@ -427,11 +427,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
"""Initialize HiCache infrastructure."""
|
||||
self.host_memory_mode = get_memory().hicache_host_memory_mode
|
||||
if self.host_memory_mode == "buffer_only":
|
||||
supported = {ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA}
|
||||
supported = {
|
||||
ComponentType.FULL,
|
||||
ComponentType.SWA,
|
||||
ComponentType.AUXILIARY_SWA,
|
||||
ComponentType.MAMBA,
|
||||
}
|
||||
if not set(self.tree_components) <= supported:
|
||||
raise ValueError(
|
||||
"--hicache-host-memory-mode buffer_only supports only "
|
||||
"FULL/SWA/MAMBA unified trees; got components "
|
||||
"FULL/SWA/AUXILIARY_SWA/MAMBA unified trees; got components "
|
||||
f"{sorted(ct.name for ct in self.tree_components)}."
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
@@ -490,21 +495,20 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
if self.host_memory_mode == "buffer_only":
|
||||
self.tree_core.set_host_memory_buffer_only()
|
||||
swa = self.components.get(ComponentType.SWA)
|
||||
auxiliary_swa = self.components.get(ComponentType.AUXILIARY_SWA)
|
||||
validate_buffer_only_stack(
|
||||
sidecar_pool_specs=self.sidecar_pool_specs,
|
||||
host_pool_group=self.host_pool_group,
|
||||
swa_component=swa,
|
||||
auxiliary_swa_component=auxiliary_swa,
|
||||
storage_prefetch_threshold=storage_prefetch_threshold,
|
||||
)
|
||||
self.buffer_pipeline = self.components[
|
||||
ComponentType.FULL
|
||||
].create_buffer_pipeline(
|
||||
max_context_len=get_model().context_length or 0,
|
||||
swa_window_pages=(
|
||||
swa.full_window_pages
|
||||
if swa is not None and self.tree_core.has_swa_host_pool
|
||||
else 0
|
||||
),
|
||||
swa_component=swa,
|
||||
auxiliary_swa_component=auxiliary_swa,
|
||||
# Leak backstop only: the queue is charged once per unique node
|
||||
# span across pools; a cap binding on live content would
|
||||
# drop-newest and punch storage holes.
|
||||
@@ -3676,11 +3680,6 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def swa_protected_size(self) -> int:
|
||||
return self.tree_core.swa_protected_size()
|
||||
|
||||
def swa_transient_size(self) -> int:
|
||||
if self.buffer_pipeline is None:
|
||||
return 0
|
||||
return self.buffer_pipeline.swa_transient_size()
|
||||
|
||||
def mamba_protected_size(self) -> int:
|
||||
return self.tree_core.mamba_protected_size()
|
||||
|
||||
|
||||
@@ -260,6 +260,7 @@ fn pool_name_str(name: PoolName) -> &'static str {
|
||||
PoolName::Kv => "kv",
|
||||
PoolName::Mamba => "mamba",
|
||||
PoolName::Swa => "swa",
|
||||
PoolName::AuxiliarySwa => "auxiliary_swa",
|
||||
PoolName::Indexer => "indexer",
|
||||
PoolName::DeepseekV4C4 => "deepseek_v4_c4",
|
||||
PoolName::DeepseekV4C4Indexer => "deepseek_v4_c4_indexer",
|
||||
@@ -288,6 +289,7 @@ fn parse_pool_name(name: &str) -> PyResult<PoolName> {
|
||||
"kv" => Ok(PoolName::Kv),
|
||||
"mamba" => Ok(PoolName::Mamba),
|
||||
"swa" => Ok(PoolName::Swa),
|
||||
"auxiliary_swa" => Ok(PoolName::AuxiliarySwa),
|
||||
"indexer" => Ok(PoolName::Indexer),
|
||||
"deepseek_v4_c4" => Ok(PoolName::DeepseekV4C4),
|
||||
"deepseek_v4_c4_indexer" => Ok(PoolName::DeepseekV4C4Indexer),
|
||||
|
||||
@@ -395,6 +395,7 @@ pub enum PoolName {
|
||||
Kv,
|
||||
Mamba,
|
||||
Swa,
|
||||
AuxiliarySwa,
|
||||
Indexer,
|
||||
DeepseekV4C4,
|
||||
DeepseekV4C4Indexer,
|
||||
|
||||
@@ -138,6 +138,7 @@ class TestPrefillAdder(CustomTestCase):
|
||||
req.retracted_stain = False
|
||||
req.host_hit_length = 0
|
||||
req.swa_host_hit_length = 0
|
||||
req.auxiliary_swa_host_hit_length = 0
|
||||
req.mamba_host_hit_length = 0
|
||||
req.storage_hit_length = 0
|
||||
req.storage_hit_start = None
|
||||
|
||||
@@ -406,7 +406,12 @@ class TestBufferModeSidecar(unittest.TestCase):
|
||||
|
||||
pipeline = BufferModePipeline.__new__(BufferModePipeline)
|
||||
pipeline._cache = cache
|
||||
pipeline._swa_window_pages = 1
|
||||
pipeline._swa_component = SimpleNamespace(
|
||||
pool_name=PoolName.SWA,
|
||||
component_type=ComponentType.SWA,
|
||||
full_window_pages=1,
|
||||
)
|
||||
pipeline._auxiliary_swa_component = None
|
||||
pipeline.write_backlog_cap = 8
|
||||
pipeline.reset()
|
||||
pipeline._build_backup_transfers = lambda node_id, *, kv_only=False: (
|
||||
|
||||
@@ -32,6 +32,7 @@ from sglang.srt.mem_cache.pool_host.dsa import DSAIndexerPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
@@ -275,6 +276,12 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase):
|
||||
pipeline = BufferModePipeline.__new__(BufferModePipeline)
|
||||
pipeline._cache = mock.Mock()
|
||||
pipeline._cache.components = {}
|
||||
pipeline._swa_component = SimpleNamespace(
|
||||
pool_name=PoolName.SWA,
|
||||
component_type=ComponentType.SWA,
|
||||
full_window_pages=1,
|
||||
)
|
||||
pipeline._auxiliary_swa_component = None
|
||||
pipeline._cache.cache_controller.mem_pool_host.entry_map = {
|
||||
PoolName.SWA: SimpleNamespace(
|
||||
host_pool=SimpleNamespace(page_size=swa_page_size)
|
||||
@@ -304,39 +311,33 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase):
|
||||
return pipeline
|
||||
|
||||
def test_short_staged_swa_tail_keeps_complete_window(self):
|
||||
"""FULL-prefix growth trims only FULL; SWA keeps its complete window."""
|
||||
"""Logical SWA coverage survives pruning resident rows from H2D."""
|
||||
handle = CacheRequestHandle("r", 0)
|
||||
pipeline = self._short_swa_tail_pipeline(swa_page_size=2)
|
||||
pipeline._cache.tree_core.is_eagle = False
|
||||
pipeline._cache.tree_core.match_full_device_prefix.return_value = (6, 1, 6)
|
||||
pipeline._cache.tree_core.collect_full_device_indices.return_value = _indices(
|
||||
0, 6
|
||||
)
|
||||
req = SimpleNamespace(
|
||||
rid="r",
|
||||
cache_request_handle=handle,
|
||||
prefix_len=0,
|
||||
kv=SimpleNamespace(cache_protected_len=0),
|
||||
)
|
||||
self.assertTrue(pipeline.prepare_staged_prefetch(req))
|
||||
self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (2, 4))
|
||||
pipeline.release_staged_hold.assert_not_called()
|
||||
|
||||
pipeline = self._short_swa_tail_pipeline(swa_page_size=4)
|
||||
pipeline._cache.tree_core.is_eagle = False
|
||||
pipeline._cache.tree_core.match_full_device_prefix.return_value = (6, 1, 6)
|
||||
pipeline._cache.tree_core.collect_full_device_indices.return_value = _indices(
|
||||
0, 6
|
||||
)
|
||||
req = SimpleNamespace(
|
||||
rid="r",
|
||||
cache_request_handle=handle,
|
||||
prefix_len=0,
|
||||
kv=SimpleNamespace(cache_protected_len=0),
|
||||
)
|
||||
self.assertTrue(pipeline.prepare_staged_prefetch(req))
|
||||
self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (2, 4))
|
||||
pipeline.release_staged_hold.assert_not_called()
|
||||
for page_size in (2, 4):
|
||||
for resident_prefix in (False, True):
|
||||
with self.subTest(page_size=page_size, resident_prefix=resident_prefix):
|
||||
pipeline = self._short_swa_tail_pipeline(swa_page_size=page_size)
|
||||
tree = pipeline._cache.tree_core
|
||||
tree.is_eagle = False
|
||||
tree.match_full_device_prefix.return_value = (6, 1, 6)
|
||||
tree.collect_full_device_indices.return_value = _indices(0, 6)
|
||||
tree.swa_tombstone_ranges.return_value = (
|
||||
[] if resident_prefix else [(4, 6)]
|
||||
)
|
||||
req = SimpleNamespace(
|
||||
rid="r",
|
||||
cache_request_handle=handle,
|
||||
prefix_len=0,
|
||||
kv=SimpleNamespace(cache_protected_len=0),
|
||||
)
|
||||
self.assertTrue(pipeline.prepare_staged_prefetch(req))
|
||||
self.assertEqual(req.host_hit_length, 2)
|
||||
self.assertEqual(
|
||||
req.swa_host_hit_length, 2 if resident_prefix else 4
|
||||
)
|
||||
window = req.staged_prefetch_plan.windows[ComponentType.SWA]
|
||||
self.assertEqual((window.start, window.end), (4, 8))
|
||||
pipeline.release_staged_hold.assert_not_called()
|
||||
|
||||
def test_l2_transfer_maps_global_layers(self):
|
||||
host_pool = MHATokenToKVPoolHost.__new__(MHATokenToKVPoolHost)
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
)
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.mem_cache.unified_radix_cache import (
|
||||
UnifiedRadixCache,
|
||||
_OngoingPrefetch,
|
||||
@@ -47,6 +48,9 @@ def _staged_fixture(full_match=2):
|
||||
prefetch_anchor_info=lambda node: (None, None),
|
||||
match_full_device_prefix=Mock(return_value=(full_match, 1, full_match)),
|
||||
collect_full_device_indices=Mock(return_value=torch.arange(8)),
|
||||
swa_tombstone_ranges=Mock(
|
||||
side_effect=lambda key, start, end, **kwargs: [(start, end)]
|
||||
),
|
||||
root_node_handle=lambda extra_key=None: 0,
|
||||
inc_full_pin=Mock(),
|
||||
dec_full_pin=Mock(),
|
||||
@@ -93,6 +97,12 @@ def _staged_fixture(full_match=2):
|
||||
cache.cache_controller = cc
|
||||
pipeline = BufferModePipeline.__new__(BufferModePipeline)
|
||||
pipeline._cache = cache
|
||||
pipeline._swa_component = SimpleNamespace(
|
||||
pool_name=PoolName.SWA,
|
||||
component_type=ComponentType.SWA,
|
||||
full_window_pages=2,
|
||||
)
|
||||
pipeline._auxiliary_swa_component = None
|
||||
pipeline.reset()
|
||||
cache.buffer_pipeline = pipeline
|
||||
pipeline.staged_prefetches[_REQ] = _StagedPrefetch(
|
||||
|
||||
@@ -123,6 +123,7 @@ class TestSwaBackupAfterCompaction(unittest.TestCase):
|
||||
)
|
||||
component = SimpleNamespace(
|
||||
component_type=ComponentType.SWA,
|
||||
pool_name=PoolName.SWA,
|
||||
tree_core=SimpleNamespace(has_swa_host_pool=True),
|
||||
_collect_unbacked_swa_nodes=lambda n: [n],
|
||||
_unified_allocator=lambda: object(),
|
||||
|
||||
@@ -518,6 +518,7 @@ class TestLoadBackAdmission(_UnifiedHiCacheCase):
|
||||
req.storage_hit_start = None
|
||||
req.host_hit_is_storage = False
|
||||
req.host_loaded_length = 0
|
||||
req.auxiliary_swa_host_hit_length = 0
|
||||
req.retracted_stain = False
|
||||
req.needs_host_load_back.return_value = True
|
||||
req.finished.return_value = False
|
||||
|
||||
@@ -5428,7 +5428,7 @@ class UnifiedRadixCacheSuite:
|
||||
cons.sanity_check()
|
||||
|
||||
def test_buffer_only_load_back_uses_full_behind_swa_tombstone(self):
|
||||
"""FULL-only rematch keeps resident FULL and loads the complete SWA window."""
|
||||
"""FULL-only rematch keeps resident FULL and repairs missing SWA rows."""
|
||||
self._skip_unsupported_hicache_test()
|
||||
if not self.cfg.has_swa:
|
||||
self.skipTest("masked overlap requires an SWA component")
|
||||
@@ -5523,12 +5523,13 @@ class UnifiedRadixCacheSuite:
|
||||
dispatched = {}
|
||||
|
||||
def capture_swa_mapping():
|
||||
load = next(iter(cons.buffer_pipeline.ongoing_buffer_load_back.values()))
|
||||
swa_xfer = next(t for t in load.aux_xfers if t.name == PoolName.SWA)
|
||||
load = cons.cache_controller.load_queue[-1]
|
||||
swa_xfer = next(t for t in load.pool_transfers if t.name == PoolName.SWA)
|
||||
dispatched["device"] = swa_xfer.device_indices.clone()
|
||||
full_window = masked_full[-len(swa_xfer.device_indices) :]
|
||||
old_window = mapping_before[-len(swa_xfer.device_indices) :]
|
||||
expected = torch.where(old_window > 0, old_window, swa_xfer.device_indices)
|
||||
full_window = masked_full[-window_tokens:]
|
||||
old_window = mapping_before[-window_tokens:]
|
||||
expected = old_window.clone()
|
||||
expected[old_window <= 0] = swa_xfer.device_indices
|
||||
self.assertTrue(
|
||||
torch.equal(
|
||||
cons_alloc.full_to_swa_index_mapping[full_window],
|
||||
@@ -5546,7 +5547,7 @@ class UnifiedRadixCacheSuite:
|
||||
)
|
||||
self.assertEqual(int(spliced.numel()), 0)
|
||||
self.assertEqual(req.prefix_len, len(seq), "FULL rematch was lost")
|
||||
self.assertEqual(len(dispatched["device"]), window_tokens)
|
||||
self.assertEqual(len(dispatched["device"]), window_tokens - live_tail)
|
||||
self.assertEqual(cons_alloc.full_available_size(), full_avail0)
|
||||
k, v = self._snapshot_full_kv(cons_alloc, masked_full)
|
||||
self.assertTrue(torch.equal(k, masked_kv[0]))
|
||||
|
||||
Reference in New Issue
Block a user