From e7d6a88bfce232fb3441baff1293e9bedeeb37ef Mon Sep 17 00:00:00 2001 From: metamergebot Date: Wed, 14 Oct 2026 16:43:24 -0700 Subject: [PATCH] [HiCache] Buffer-only L3 for an auxiliary SWA window; load back only missing window rows (#43932) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: houseroad Co-authored-by: xiezhq-hermann --- python/sglang/srt/managers/schedule_batch.py | 8 + python/sglang/srt/managers/schedule_policy.py | 16 + .../srt/mem_cache/buffer_mode/pipeline.py | 329 +++++++++--------- .../sglang/srt/mem_cache/hicache_storage.py | 1 + python/sglang/srt/mem_cache/prefill_budget.py | 3 + .../srt/mem_cache/rust_tree_core/adapter.py | 11 +- .../sglang/srt/mem_cache/storage_prefetch.py | 24 +- .../unified_cache/components/base.py | 7 +- .../mem_cache/unified_cache/components/swa.py | 32 +- .../unified_cache/storage_attachment.py | 17 +- .../unified_cache/unified_tree_core.py | 15 +- .../unified_tree_core_interface.py | 10 +- .../srt/mem_cache/unified_radix_cache.py | 23 +- rust/sglang-radix-tree/src/python_bindings.rs | 2 + .../src/unified_tree_core.rs | 1 + .../unit/managers/test_prefill_adder.py | 1 + .../mem_cache/test_buffer_mode_sidecar.py | 7 +- ...test_hicache_staged_write_back_dispatch.py | 65 ++-- .../test_storage_prefetch_lifecycle.py | 10 + .../test_unified_hicache_regressions.py | 1 + ...test_unified_hicache_transfer_lifecycle.py | 1 + .../test_unified_radix_cache_unittest.py | 15 +- 22 files changed, 356 insertions(+), 243 deletions(-) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 8beec775a2..45ff08e36b 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1480,12 +1480,20 @@ class Req(ReqDllmMixin): return self.output_ids[: self.finished_len] return self.output_ids + @property + def auxiliary_swa_host_hit_length(self) -> int: + plan = self.staged_prefetch_plan + return ( + plan.window_tokens(ComponentType.AUXILIARY_SWA) if plan is not None else 0 + ) + def needs_host_load_back(self) -> bool: """Whether any cache layer has a host hit that needs L2 H2D load_back.""" return ( self.host_hit_length > 0 or self.swa_host_hit_length > 0 or self.mamba_host_hit_length > 0 + or self.auxiliary_swa_host_hit_length > 0 ) def materialized_host_hit_len(self) -> int: diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index 9f6d152173..ce0dffb826 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -795,6 +795,7 @@ class PrefillAdder: input_tokens=len(req.full_untruncated_fill_ids), swa_host_hit_length=swa_host_hit_length, chunk_limit=self.rem_chunk_tokens, + auxiliary_swa_host_hit_length=req.auxiliary_swa_host_hit_length, ) def _mamba_gap_budget_for_req( @@ -1333,6 +1334,16 @@ class PrefillAdder: return self.budget_state() + def _defer_staged_for_capacity_if_idle(self, req: Req, *, pool: str) -> None: + # Running work and admitted prefills (including continuing chunks) can + # release capacity. Only idle budget failures should force recomputation. + if self.can_run_list or ( + self.running_batch is not None and self.running_batch.reqs + ): + return + if self.tree_cache.buffer_pipeline is not None: + self.tree_cache.buffer_pipeline.defer_staged_admission(req, pool) + def add_one_req( self, req: Req, has_chunked_req: bool, truncation_align_size: Optional[int] ): @@ -1379,6 +1390,8 @@ class PrefillAdder: has_chunked_req=has_chunked_req, ) if isinstance(admission, AddReqResult): + if admission is AddReqResult.NO_TOKEN: + self._defer_staged_for_capacity_if_idle(req, pool="prefill_budget") return admission # A rejected candidate must not report prefillable or queue H2D. @@ -1435,6 +1448,9 @@ class PrefillAdder: swa_host_hit_length=req.swa_host_hit_length, chunk_limit=self.rem_chunk_tokens, ): + self._defer_staged_for_capacity_if_idle( + req, pool="load_back_budget" + ) return AddReqResult.NO_TOKEN promised_host_hit = req.host_hit_length loaded = self.tree_cache.init_load_back( diff --git a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py index 7119c8e829..e68a8f7517 100644 --- a/python/sglang/srt/mem_cache/buffer_mode/pipeline.py +++ b/python/sglang/srt/mem_cache/buffer_mode/pipeline.py @@ -28,6 +28,7 @@ from __future__ import annotations import logging from array import array from collections import deque +from dataclasses import replace from typing import TYPE_CHECKING, Optional import msgspec @@ -52,8 +53,10 @@ from sglang.srt.mem_cache.hicache_storage import ( ) from sglang.srt.mem_cache.pool_host.base import HostKVCache from sglang.srt.mem_cache.radix_cache import RadixKey -from sglang.srt.mem_cache.storage_prefetch import StagedPrefetchPlan -from sglang.srt.mem_cache.unified_cache.cache_action import RebuildFullToSWAMapping +from sglang.srt.mem_cache.storage_prefetch import ( + StagedPrefetchPlan, + WindowLoadBackPlan, +) from sglang.srt.mem_cache.unified_cache.components import ( CacheTransferPhase, ComponentType, @@ -146,10 +149,11 @@ class _StagedPrefetch(msgspec.Struct): class _OngoingBufferLoadBack(msgspec.Struct): """A queued H2D load, registered after its prefix is inserted. - The full host bounce and redundant auxiliary device slots remain owned - here until the copy completes. ``load_transfers`` is the transfer list - submitted to H2D; ``aux_xfers`` retains the original staging owners and - ``load_contexts`` each component's part in the load. + The full host bounce, and any redundant device slots a component's load + context reports, remain owned here until the copy completes. + ``load_transfers`` is the transfer list submitted to H2D; ``aux_xfers`` + retains the original staging owners and ``load_contexts`` each + component's part in the load. """ request: CacheRequestHandle @@ -176,6 +180,8 @@ def validate_buffer_only_stack( host_pool_group: HostPoolGroup, swa_component: Optional[SWAComponent], storage_prefetch_threshold: int = 256, + *, + auxiliary_swa_component: Optional[SWAComponent] = None, ) -> None: """Post-assembly buffer-mode fences. @@ -202,50 +208,48 @@ def validate_buffer_only_stack( f"pool={spec.pool_name}, host_slots={sidecar_size}, " f"source={spec.indices_from_pool}, source_slots={source_size}." ) - swa = swa_component - if swa is not None and swa._swa_kv_pool_host is None: - # Only reachable on SWA models with the unified_kv layout (SWA as - # a device-only ring): without a host pool the window can neither - # stage for writes nor fetch for load-backs. - raise ValueError( - "--hicache-host-memory-mode buffer_only on SWA models " - "requires an SWA host staging pool; the unified_kv layout " - "keeps SWA as a device-only ring." - ) - if swa is not None and swa._swa_kv_pool_host is not None: - # Below two windows the pool cannot hold a staging write AND the - # loads-priority reserve (_aux_loads_margin floors at one - # window), so every window-carrying intent would be dropped as - # oversize and SWA storage coverage would silently be zero. - window_tokens = swa.full_window_pages * swa._swa_kv_pool_host.page_size + window_pools = [] + for component in (swa_component, auxiliary_swa_component): + if component is None: + continue + pool = component._swa_kv_pool_host + if pool is None: + raise ValueError( + "--hicache-host-memory-mode buffer_only on SWA models " + "requires an SWA host staging pool; the unified_kv layout " + "keeps SWA as a device-only ring." + ) + window_pools.append((pool, component.full_window_pages * pool.page_size)) + + for pool, window_tokens in window_pools: + # Every window needs a write bounce plus a loads-priority reserve. + # For a shared arena the reserve must hold all windows together. shared_domain = ( - swa._swa_kv_pool_host.shared_allocation_domain - if isinstance(swa._swa_kv_pool_host, HostKVCache) - else None + pool.shared_allocation_domain if isinstance(pool, HostKVCache) else None ) if shared_domain is not None: - full_host_pool = ( - swa.cache.cache_controller.mem_pool_host.anchor_entry.host_pool - ) + full_host_pool = host_pool_group.anchor_entry.host_pool min_full_tokens = _minimum_full_transfer_tokens( full_host_pool, storage_prefetch_threshold ) - one_transfer_bytes = ( - window_tokens * swa._swa_kv_pool_host.size_per_token - + min_full_tokens * full_host_pool.size_per_token - ) - one_transfer = [ - (full_host_pool.pool_label, min_full_tokens), - (swa._swa_kv_pool_host.pool_label, window_tokens), + shared_pools = [ + (host, tokens) + for host, tokens in [(full_host_pool, min_full_tokens), *window_pools] + if isinstance(host, HostKVCache) + and host.shared_allocation_domain is shared_domain ] + one_transfer_bytes = sum( + tokens * host.size_per_token for host, tokens in shared_pools + ) + one_transfer = [(host.pool_label, tokens) for host, tokens in shared_pools] enough_capacity = shared_domain.can_fit_many_then( one_transfer, one_transfer, empty=True ) capacity = f"{shared_domain.capacity_bytes} shared bytes" requirement = f"{2 * one_transfer_bytes} shared bytes" else: - enough_capacity = swa._swa_kv_pool_host.size >= 2 * window_tokens - capacity = f"{swa._swa_kv_pool_host.size} SWA tokens" + enough_capacity = pool.size >= 2 * window_tokens + capacity = f"{pool.size} SWA tokens" requirement = f"{2 * window_tokens} SWA tokens" if not enough_capacity: raise ValueError( @@ -275,15 +279,15 @@ class BufferModePipeline: def __init__( self, cache: UnifiedRadixCache, - swa_window_pages: int, + swa_component: Optional[SWAComponent], write_backlog_cap: int, max_context_len: int = 0, + *, + auxiliary_swa_component: Optional[SWAComponent] = None, ): self._cache = cache - # SWA window size in KV pages when the SWA component stages through - # a host pool (0 = KV-only: no trailing window staged). Static after - # pool assembly. - self._swa_window_pages = swa_window_pages + self._swa_component = swa_component + self._auxiliary_swa_component = auxiliary_swa_component # Metadata-only pending-write backlog cap, measured in unique FULL # snapshot spans; beyond it new spans are dropped at admission. self.write_backlog_cap = write_backlog_cap @@ -352,6 +356,13 @@ class BufferModePipeline: self.anchor_locked_tokens_ = 0 self._anchor_lock_cap_skips = 0 + def _swa_components(self): + return ( + component + for component in (self._swa_component, self._auxiliary_swa_component) + if component is not None + ) + def _shared_host_domain(self): cc = self._cache.cache_controller anchor = cc.mem_pool_host.anchor_entry.host_pool @@ -421,16 +432,16 @@ class BufferModePipeline: def _shared_load_reserve_requests(self) -> list[tuple[str, int]]: cc = self._cache.cache_controller anchor = cc.mem_pool_host.anchor_entry.host_pool - swa_entry = cc.mem_pool_host.entry_map.get(PoolName.SWA) min_full_tokens = _minimum_full_transfer_tokens( anchor, self._cache.prefetch_threshold ) requests = [(anchor.pool_label, min_full_tokens)] - if swa_entry is not None and self._swa_window_pages: + for component in self._swa_components(): + entry = cc.mem_pool_host.entry_map[component.pool_name] requests.append( ( - swa_entry.host_pool.pool_label, - self._swa_window_pages * swa_entry.host_pool.page_size, + entry.host_pool.pool_label, + component.full_window_pages * entry.host_pool.page_size, ) ) return requests @@ -460,15 +471,6 @@ class BufferModePipeline: or self.anchor_locks ) - def swa_transient_size(self) -> int: - """SWA destinations kept alive only until an in-flight H2D completes.""" - return sum( - len(device_indices) - for load_back in self.ongoing_buffer_load_back.values() - for pool_name, device_indices in load_back.aux_device_releases - if pool_name == PoolName.SWA - ) - # ---- backup pipeline (device -> staging -> storage) ---- def _anchor_pool_aux_tokens(self, aux_xfers: list[PoolTransfer]) -> int: @@ -729,19 +731,26 @@ class BufferModePipeline: need = len(t.keys) * entry.host_pool.page_size if entry.host_pool is anchor_pool: anchor_need += need - elif need > entry.host_pool.size - self._aux_loads_margin(entry.host_pool): + elif need > entry.host_pool.size - self._aux_loads_margin( + t.name, entry.host_pool + ): return True return anchor_need > cc.mem_pool_host.size - def _aux_loads_margin(self, host_pool) -> int: + def _aux_loads_margin(self, pool_name, host_pool) -> int: """Aux-pool tokens reserved for loads: at least one trailing window (prepare_prefetch allocates its window here and a failed alloc forfeits the whole prefetch), plus a 10% burst absorber mirroring live_cap.""" - return max( - self._swa_window_pages * host_pool.page_size, - host_pool.size // 10, - ) + window_pages = 0 + if pool_name == PoolName.SWA and self._swa_component is not None: + window_pages = self._swa_component.full_window_pages + elif ( + pool_name == PoolName.AUXILIARY_SWA + and self._auxiliary_swa_component is not None + ): + window_pages = self._auxiliary_swa_component.full_window_pages + return max(window_pages * host_pool.page_size, host_pool.size // 10) def _validate_backup_intent( self, intent: _UnifiedBackupIntent @@ -1081,7 +1090,7 @@ class BufferModePipeline: continue need = len(t.keys) * entry.host_pool.page_size headroom = entry.host_pool.available_size() - self._aux_loads_margin( - entry.host_pool + t.name, entry.host_pool ) if need > headroom: return True @@ -1444,7 +1453,7 @@ class BufferModePipeline: self._resolve_device_covered(req) return True req.host_hit_length = plan.full_tokens - req.swa_host_hit_length = plan.swa_tokens + req.swa_host_hit_length = plan.window_tokens(ComponentType.SWA) req.mamba_host_hit_length = plan.mamba_slots # The device alone serves only this pass's joint match; the rest of the # span, fetched FULL or resident FULL the aux tail unlocks, is storage's. @@ -1465,21 +1474,46 @@ class BufferModePipeline: self, f: _StagedPrefetch, key: RadixKey, matched_len: int ) -> Optional[StagedPrefetchPlan]: full_tokens = max(0, f.matched_len + f.num_tokens - matched_len) - swa_tokens = sum( - len(t.host_indices) - for t in f.aux_xfers - if t.name == PoolName.SWA and t.host_indices is not None - ) + span_end = f.matched_len + f.num_tokens + windows = {} + for component in self._swa_components(): + staged_tokens = sum( + len(t.host_indices) + for t in f.aux_xfers + if t.name == component.pool_name and t.host_indices is not None + ) + window_start = span_end - staged_tokens + repair_end = min(matched_len, span_end) + missing_ranges = ( + self._cache.tree_core.swa_tombstone_ranges( + key, + window_start, + repair_end, + component_type=component.component_type, + ) + if window_start < repair_end + else [] + ) + tail_start = max(matched_len, window_start) + if tail_start < span_end: + missing_ranges.append((tail_start, span_end)) + windows[component.component_type] = WindowLoadBackPlan( + window_start, span_end, tuple(missing_ranges) + ) mamba_slots = sum( len(t.host_indices) for t in f.aux_xfers if t.name == PoolName.MAMBA and t.host_indices is not None ) - if full_tokens == 0 and swa_tokens == 0 and mamba_slots == 0: + if ( + full_tokens == 0 + and mamba_slots == 0 + and not any(w.num_tokens for w in windows.values()) + ): # The device already covers every pool's remaining demand. return None return StagedPrefetchPlan( - f.operation_id, key, matched_len, full_tokens, swa_tokens, mamba_slots + f.operation_id, key, matched_len, full_tokens, windows, mamba_slots ) def _resolve_device_covered(self, req: Req) -> None: @@ -1666,7 +1700,7 @@ class BufferModePipeline: req.mamba_host_hit_length, ) == ( plan.full_tokens, - plan.swa_tokens, + plan.window_tokens(ComponentType.SWA), plan.mamba_slots, ), f"staged load-back budget changed for {req.rid}" @@ -1682,17 +1716,32 @@ class BufferModePipeline: key = plan.key span_end = f.matched_len + f.num_tokens load_back_id = -(f.operation_id) - 1 - # The full trailing-window aux transfer is independent of the shorter - # FULL suffix and may remain nonempty for an aux-only load. + # Preserve the complete bounce in f until ack; H2D uses only the rows + # admitted by the plan, including repairs inside the resident FULL prefix. load_xfers = list(f.aux_xfers) - staged_swa = next( - ( - len(t.host_indices) - for t in load_xfers - if t.name == PoolName.SWA and t.host_indices is not None - ), - 0, - ) + for component in self._swa_components(): + pool_name = component.pool_name + window = plan.windows[component.component_type] + if not window.missing_ranges: + load_xfers = [ + t + for t in load_xfers + if t.name != pool_name and t.indices_from_pool != pool_name + ] + continue + host_parts = [ + slice(start - window.start, end - window.start) + for start, end in window.missing_ranges + ] + for i, transfer in enumerate(load_xfers): + if transfer.name != pool_name: + continue + load_xfers[i] = replace( + transfer, + host_indices=torch.cat( + [transfer.host_indices[part] for part in host_parts] + ), + ) state, failure_pool = self._allocate_and_queue_load_back( req, f, plan, load_back_id, load_xfers @@ -1709,71 +1758,43 @@ class BufferModePipeline: request, credited_tokens=req.storage_hit_length ) - swa_dev = next( - ( - t.device_indices - for t in state.load_transfers - if t.name == PoolName.SWA - and t.device_indices is not None - and t.device_indices.numel() > 0 - ), - None, - ) - if swa_dev is not None: - # SWA loads into fresh reservations. Register the window's FULL->SWA - # translation now (attention reads through it); slots another request - # still holds are kept, and their redundant destinations freed at ack. - full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :] - allocator = cache.token_to_kv_pool_allocator - old_swa = allocator.translate_swa_indices_for_transfer(full_window) - missing = old_swa <= 0 - window_start = span_end - len(swa_dev) - repair_end = min(splice_base, span_end) - tree_missing = torch.zeros_like(missing) - tail_start = max(splice_base, window_start) - tree_missing[tail_start - window_start :] = True - repair_ranges = [] - if window_start < repair_end: - repair_ranges = cache.tree_core.swa_tombstone_ranges( - key, window_start, repair_end - ) - for repair_start, repair_end_ in repair_ranges: - repair_slice = slice( - repair_start - window_start, repair_end_ - window_start - ) - tree_missing[repair_slice] = True - assert torch.equal(missing, tree_missing), ( - "SWA tree and allocator residency disagree for restored window " - f"[{window_start}, {span_end})" + full_indices = torch.cat([prefix_indices, device_indices]) + for component in self._swa_components(): + pool_name = component.pool_name + window_dev = next( + ( + t.device_indices + for t in state.load_transfers + if t.name == pool_name + and t.device_indices is not None + and t.device_indices.numel() > 0 + ), + None, ) - for repair_start, repair_end_ in repair_ranges: - repair_slice = slice( - repair_start - window_start, repair_end_ - window_start - ) - for action in cache.tree_core.attach_swa_window( - key, - repair_start, - repair_end_, - swa_dev[repair_slice], - ): - cache._apply_cache_action(action) - if bool(missing.any()): - cache._apply_cache_action( - RebuildFullToSWAMapping( - [full_window[missing]], - [swa_dev[missing]], - ) - ) - if bool((~missing).any()): - state.aux_device_releases.append((PoolName.SWA, swa_dev[~missing])) + if window_dev is None: + continue + ct = component.component_type + window = plan.windows[ct] + assert len(window_dev) == window.num_tokens + full_parts = [] + offset = 0 + for start, end in window.missing_ranges: + full_part = full_indices[start:end] + full_parts.append(full_part) + if end <= splice_base: + for action in cache.tree_core.attach_swa_window( + key, + start, + end, + window_dev[offset : offset + end - start], + component_type=ct, + ): + cache._apply_cache_action(action) + offset += end - start + component.restore_buffer_mapping(torch.cat(full_parts), window_dev) insert_result = self._insert_loaded_prefix( - state, - req, - plan, - prefix_indices=prefix_indices, - span_end=span_end, - staged_swa=staged_swa, + state, req, plan, prefix_indices=prefix_indices ) self.ongoing_buffer_load_back[load_back_id] = state match = cache.match_prefix(MatchPrefixParams(key=key)) @@ -1871,8 +1892,6 @@ class BufferModePipeline: plan: StagedPrefetchPlan, *, prefix_indices: torch.Tensor, - span_end: int, - staged_swa: int, ) -> InsertResult: # Publish via a plain insert under the admission lock choreography; # the caller's request lock then pins the span (load_back pattern). @@ -1887,7 +1906,7 @@ class BufferModePipeline: value=torch.cat([prefix_indices, state.device_indices]), prev_prefix_len=plan.device_prefix_len, component_evicted_seqlens={ - ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0 + ct: window.start for ct, window in plan.windows.items() }, **insert_fields, ) @@ -1925,25 +1944,13 @@ class BufferModePipeline: return True def _free_completed_load_back_buffers(self, f: _OngoingBufferLoadBack) -> None: - cache = self._cache - cc = cache.cache_controller + cc = self._cache.cache_controller # The H2D consumed the bounce buffers; free them outright. self._free_staging_now(f.host_indices, f.aux_xfers) for pool_name, device_indices in f.aux_device_releases: entry = cc.mem_pool_host.entry_map[pool_name] - from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( - UnifiedSWAAllocatorBase, - ) - - allocator = cache.token_to_kv_pool_allocator - if pool_name == PoolName.SWA and isinstance( - allocator, UnifiedSWAAllocatorBase - ): - # H2D has completed; these redundant slots are no longer pending. - allocator.swa_attn_allocator.free_physical(device_indices) - else: - free_fn = entry.device_free_fn or entry.device_pool.free - free_fn(device_indices) + free_fn = entry.device_free_fn or entry.device_pool.free + free_fn(device_indices) def release_staged_hold( self, request: CacheRequestHandle, reason: Optional[str] = None diff --git a/python/sglang/srt/mem_cache/hicache_storage.py b/python/sglang/srt/mem_cache/hicache_storage.py index a226069588..2aa43ed307 100644 --- a/python/sglang/srt/mem_cache/hicache_storage.py +++ b/python/sglang/srt/mem_cache/hicache_storage.py @@ -56,6 +56,7 @@ class PoolName(str, Enum): KV = "kv" MAMBA = "mamba" SWA = "swa" + AUXILIARY_SWA = "auxiliary_swa" INDEXER = "indexer" # TODO(hzh0425): Current DeepSeek V4 pool naming is verbose; will be normalized to # 'COMPRESSED_KV / COMPRESSED_INDEXER / COMPRESSED_STATE' in the next PR. diff --git a/python/sglang/srt/mem_cache/prefill_budget.py b/python/sglang/srt/mem_cache/prefill_budget.py index 5a22403596..7737277c83 100644 --- a/python/sglang/srt/mem_cache/prefill_budget.py +++ b/python/sglang/srt/mem_cache/prefill_budget.py @@ -120,6 +120,7 @@ class PrefillBudget: input_tokens: int, swa_host_hit_length: int, chunk_limit: int | None, + auxiliary_swa_host_hit_length: int = 0, ) -> tuple[bool, int | None]: """Return admission feasibility and a memory bound on the requested chunk.""" return ( @@ -235,6 +236,7 @@ class SWAPrefillBudget(PrefillBudget): input_tokens: int, swa_host_hit_length: int, chunk_limit: int | None, + auxiliary_swa_host_hit_length: int = 0, ) -> tuple[bool, int | None]: if total_tokens >= self.remaining_total: return False, None @@ -398,6 +400,7 @@ class SharedSWAPrefillBudget(SWAPrefillBudget): input_tokens: int, swa_host_hit_length: int, chunk_limit: int | None, + auxiliary_swa_host_hit_length: int = 0, ) -> tuple[bool, int | None]: needed = self.swa_tokens( extend_input_len, diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index 3b1040b208..e07d7ec339 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -520,8 +520,14 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): raise NotImplementedError("root_node: not yet ported to the Rust tree core") def swa_tombstone_ranges( - self, key: RadixKey, start: int, end: int + self, + key: RadixKey, + start: int, + end: int, + component_type: ComponentType = ComponentType.SWA, ) -> list[tuple[int, int]]: + if component_type != ComponentType.SWA: + raise NotImplementedError("The Rust tree core supports one SWA window") return self._binding.swa_tombstone_ranges( self._bindings.MatchParamsBinding( key=_radix_key_buffer(key), @@ -538,7 +544,10 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): window_start: int, window_end: int, swa_values: torch.Tensor, + component_type: ComponentType = ComponentType.SWA, ) -> list[CacheAction | ComponentAction]: + if component_type != ComponentType.SWA: + raise NotImplementedError("The Rust tree core supports one SWA window") return _cache_actions_from_tagged( self._binding.attach_swa_window( self._bindings.MatchParamsBinding( diff --git a/python/sglang/srt/mem_cache/storage_prefetch.py b/python/sglang/srt/mem_cache/storage_prefetch.py index a265c266ff..e319332faa 100644 --- a/python/sglang/srt/mem_cache/storage_prefetch.py +++ b/python/sglang/srt/mem_cache/storage_prefetch.py @@ -3,6 +3,10 @@ from __future__ import annotations from dataclasses import dataclass from typing import TYPE_CHECKING, Optional +import msgspec + +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType + if TYPE_CHECKING: from sglang.srt.managers.schedule_batch import Req from sglang.srt.mem_cache.radix_cache import RadixKey @@ -74,13 +78,29 @@ class StoragePrefetchRetries: return ready +class WindowLoadBackPlan(msgspec.Struct, frozen=True): + """Logical coverage and the missing subranges that need new device rows.""" + + start: int + end: int + missing_ranges: tuple[tuple[int, int], ...] + + @property + def num_tokens(self) -> int: + return sum(end - start for start, end in self.missing_ranges) + + @dataclass(frozen=True) class StagedPrefetchPlan: - """One admission pass, computed before promoting the joint FULL/SWA match.""" + """One admission pass, computed before promoting the joint component match.""" operation_id: int key: RadixKey device_prefix_len: int full_tokens: int - swa_tokens: int + windows: dict[ComponentType, WindowLoadBackPlan] mamba_slots: int = 0 + + def window_tokens(self, component_type: ComponentType) -> int: + window = self.windows.get(component_type) + return window.num_tokens if window is not None else 0 diff --git a/python/sglang/srt/mem_cache/unified_cache/components/base.py b/python/sglang/srt/mem_cache/unified_cache/components/base.py index 824d1a7baa..20859dfc36 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/base.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/base.py @@ -39,6 +39,7 @@ if TYPE_CHECKING: CacheAction, ComponentAction, ) + from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent from sglang.srt.mem_cache.unified_cache.unified_tree_core import UnifiedTreeCore from sglang.srt.mem_cache.unified_radix_cache import ( NodeId, @@ -779,16 +780,18 @@ class TreeComponent(ABC): def create_buffer_pipeline( self, *, - swa_window_pages: int, + swa_component: Optional[SWAComponent], write_backlog_cap: int, max_context_len: int = 0, + auxiliary_swa_component: Optional[SWAComponent] = None, ) -> BufferModePipeline: """Called on the FULL component to construct the buffer-mode pipeline.""" from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline return BufferModePipeline( cache=self.cache, - swa_window_pages=swa_window_pages, + swa_component=swa_component, + auxiliary_swa_component=auxiliary_swa_component, write_backlog_cap=write_backlog_cap, max_context_len=max_context_len, ) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index 2edd74446b..1612b74cc9 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -97,8 +97,14 @@ class SWAComponent(TreeComponent): self._swa_kv_pool_host = None component_type = ComponentType.SWA + pool_name = PoolName.SWA _independent = False + def restore_buffer_mapping(self, full_indices, window_indices): + self.cache._apply_cache_action( + RebuildFullToSWAMapping([full_indices], [window_indices]) + ) + def _collect_unbacked_swa_nodes( self, node: UnifiedTreeNode ) -> list[UnifiedTreeNode]: @@ -1074,7 +1080,7 @@ class SWAComponent(TreeComponent): assert self._swa_kv_pool_host is not None host_indices = self._swa_kv_pool_host.alloc(num_tokens) if host_indices is None: - self.cache.evict_host(num_tokens, ComponentType.SWA) + self.cache.evict_host(num_tokens, self.component_type) host_indices = self._swa_kv_pool_host.alloc(num_tokens) return host_indices @@ -1087,7 +1093,7 @@ class SWAComponent(TreeComponent): return {} cd = node.component_data[self.component_type] if not self.tree_core.has_swa_host_pool or cd.value is None: - return {PoolName.SWA: []} + return {self.pool_name: []} rows = ( node.component_data[BASE_COMPONENT_TYPE].value if self._unified_allocator() is not None @@ -1095,8 +1101,8 @@ class SWAComponent(TreeComponent): ) num_pages = len(rows) // self._swa_kv_pool_host.page_size if num_pages == 0 or num_pages > len(hash_values): - return {PoolName.SWA: []} - return {PoolName.SWA: list(hash_values[-num_pages:])} + return {self.pool_name: []} + return {self.pool_name: list(hash_values[-num_pages:])} def build_hicache_transfers( self, @@ -1123,7 +1129,7 @@ class SWAComponent(TreeComponent): unbacked_swa_nodes.reverse() return [ PoolTransfer( - name=PoolName.SWA, + name=self.pool_name, device_indices=( self._translate_full_to_swa( torch.cat( @@ -1172,7 +1178,7 @@ class SWAComponent(TreeComponent): return [ PoolTransfer( - name=PoolName.SWA, + name=self.pool_name, host_indices=torch.cat(backed_up), device_indices=None, nodes_to_load=[n.id for n in nodes], @@ -1188,7 +1194,7 @@ class SWAComponent(TreeComponent): return None return [ PoolTransfer( - name=PoolName.SWA, + name=self.pool_name, host_indices=cd.host_value[-num_pages * self.tree_core.page_size :], keys=node.hash_value[-num_pages:], hit_policy=PoolHitPolicy.TRAILING_PAGES, @@ -1203,7 +1209,7 @@ class SWAComponent(TreeComponent): return None return [ PoolTransfer( - name=PoolName.SWA, + name=self.pool_name, keys=["__placeholder__"] * num_pages, hit_policy=PoolHitPolicy.TRAILING_PAGES, ) @@ -1229,7 +1235,7 @@ class SWAComponent(TreeComponent): num_pages = len(value) // page return PoolTransfer( - name=PoolName.SWA, + name=self.pool_name, device_indices=value[-num_pages * page :].to(torch.int64), keys=node.hash_value[-num_pages:], hit_policy=PoolHitPolicy.TRAILING_PAGES, @@ -1245,7 +1251,7 @@ class SWAComponent(TreeComponent): return None transfer = PoolTransfer( - name=PoolName.SWA, + name=self.pool_name, keys=tail_keys, hit_policy=PoolHitPolicy.TRAILING_PAGES, ) @@ -1408,7 +1414,7 @@ class SWAComponent(TreeComponent): host_indices.numel() // page_size if host_indices is not None else 0 ) loaded_pages = ( - pool_storage_result.extra_pool_hit_pages.get(PoolName.SWA, 0) + pool_storage_result.extra_pool_hit_pages.get(self.pool_name, 0) if pool_storage_result else 0 ) @@ -1522,7 +1528,7 @@ class SWAComponent(TreeComponent): if self._swa_kv_pool_host is None: return for host_value in host_values: - self.cache.host_pool_group.free(host_value, pool=PoolName.SWA) + self.cache.host_pool_group.free(host_value, pool=self.pool_name) def apply_component_action(self, action: ComponentAction) -> None: alloc = self.cache.token_to_kv_pool_allocator @@ -1536,7 +1542,7 @@ class SWAComponent(TreeComponent): if host_indices is not None and host_indices.numel() > 0: self.cache.cache_controller.append_host_mem_release( extra_pools=[ - PoolTransfer(name=PoolName.SWA, host_indices=host_indices) + PoolTransfer(name=self.pool_name, host_indices=host_indices) ] ) return diff --git a/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py b/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py index 5aa0137501..6854a4ef91 100644 --- a/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py +++ b/python/sglang/srt/mem_cache/unified_cache/storage_attachment.py @@ -141,6 +141,9 @@ class StorageAttachment: sidecar_pool_specs=cache.sidecar_pool_specs, host_pool_group=cache.host_pool_group, swa_component=cache.components.get(ComponentType.SWA), + auxiliary_swa_component=cache.components.get( + ComponentType.AUXILIARY_SWA + ), storage_prefetch_threshold=prefetch_threshold, ) # New workers must see the requested policy from their first operation. @@ -293,12 +296,14 @@ class StorageAttachment: def resolve_prefetch_threshold(self, configured: int) -> int: """Use the same complete-window minimum for every buffer-mode anchor.""" cache = self._cache - window = cache.sliding_window_size - if not window or cache.host_memory_mode != "buffer_only": - return configured - page_size = cache.page_size - window_tokens = ((window + page_size - 1) // page_size) * page_size - return max(configured, window_tokens) + if cache.host_memory_mode == "buffer_only": + for ct in (ComponentType.SWA, ComponentType.AUXILIARY_SWA): + component = cache.components.get(ct) + if component is not None: + configured = max( + configured, component.full_window_pages * cache.page_size + ) + return configured def _resolve_metrics_collector( self, diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index c917ecf791..8e2eca094f 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -2786,7 +2786,11 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): remaining = remaining[prefix_len:] def swa_tombstone_ranges( - self, key: RadixKey, start: int, end: int + self, + key: RadixKey, + start: int, + end: int, + component_type: ComponentType = ComponentType.SWA, ) -> list[tuple[int, int]]: """Return the maximal SWA-tombstoned ranges within ``[start, end)``.""" ranges: list[tuple[int, int]] = [] @@ -2794,7 +2798,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): seg_end = pos + prefix_len if seg_end <= start: continue - if child.component_data[ComponentType.SWA].value is not None: + if child.component_data[component_type].value is not None: continue lo, hi = max(start, pos), min(end, seg_end) if ranges and ranges[-1][1] == lo: @@ -2811,6 +2815,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): window_start: int, window_end: int, swa_values: torch.Tensor, + component_type: ComponentType = ComponentType.SWA, ) -> list[CacheAction | ComponentAction]: """Attach a loaded SWA slice to an existing tombstoned tree span.""" assert len(swa_values) == window_end - window_start, ( @@ -2825,7 +2830,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): continue seg_start = max(pos, window_start) assign_end = min(seg_end, window_end) - assert child.component_data[ComponentType.SWA].value is None, ( + assert child.component_data[component_type].value is None, ( f"attach_swa_window over live SWA at [{seg_start}, {assign_end}) " f"of [{window_start}, {window_end})" ) @@ -2836,6 +2841,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): assign_end, swa_values[seg_start - window_start : assign_end - window_start], actions, + component_type, ) covered = assign_end if covered >= window_end: @@ -2853,6 +2859,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): seg_end: int, values: torch.Tensor, actions: list[CacheAction | ComponentAction], + component_type: ComponentType, ) -> None: target = node if seg_start > node_start: @@ -2864,7 +2871,7 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): if action is not None: actions.append(action) target = fragment - self.set_component_device_value(target.id, ComponentType.SWA, values.clone()) + self.set_component_device_value(target.id, component_type, values.clone()) def set_component_device_value( self, node_id: NodeId, component_type: ComponentType, value: torch.Tensor diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py index 304318b18f..a2163b1c92 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py @@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Optional, Sequence import msgspec +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType + # Tree node id -- the node handle used outside the TreeCore. The concrete tree # node is a TreeCore-internal type. NodeId = int @@ -151,7 +153,6 @@ if TYPE_CHECKING: ) from sglang.srt.mem_cache.unified_cache.components import ( CacheTransferPhase, - ComponentType, ) from sglang.srt.mem_cache.unified_cache.unified_tree_core import ( StorageBackupSpec, @@ -728,7 +729,11 @@ class UnifiedTreeCoreInterface(ABC): @abstractmethod def swa_tombstone_ranges( - self, key: RadixKey, start: int, end: int + self, + key: RadixKey, + start: int, + end: int, + component_type: ComponentType = ComponentType.SWA, ) -> list[tuple[int, int]]: """Return maximal missing SWA ranges within the matched [start, end) span.""" ... @@ -740,6 +745,7 @@ class UnifiedTreeCoreInterface(ABC): window_start: int, window_end: int, swa_values: torch.Tensor, + component_type: ComponentType = ComponentType.SWA, ) -> list[CacheAction | ComponentAction]: """Attach a loaded SWA window to tombstoned spans, returning split actions. diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index d9db734a1c..ee3087eecf 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -427,11 +427,16 @@ class UnifiedRadixCache(BasePrefixCache): """Initialize HiCache infrastructure.""" self.host_memory_mode = get_memory().hicache_host_memory_mode if self.host_memory_mode == "buffer_only": - supported = {ComponentType.FULL, ComponentType.SWA, ComponentType.MAMBA} + supported = { + ComponentType.FULL, + ComponentType.SWA, + ComponentType.AUXILIARY_SWA, + ComponentType.MAMBA, + } if not set(self.tree_components) <= supported: raise ValueError( "--hicache-host-memory-mode buffer_only supports only " - "FULL/SWA/MAMBA unified trees; got components " + "FULL/SWA/AUXILIARY_SWA/MAMBA unified trees; got components " f"{sorted(ct.name for ct in self.tree_components)}." ) from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( @@ -490,21 +495,20 @@ class UnifiedRadixCache(BasePrefixCache): if self.host_memory_mode == "buffer_only": self.tree_core.set_host_memory_buffer_only() swa = self.components.get(ComponentType.SWA) + auxiliary_swa = self.components.get(ComponentType.AUXILIARY_SWA) validate_buffer_only_stack( sidecar_pool_specs=self.sidecar_pool_specs, host_pool_group=self.host_pool_group, swa_component=swa, + auxiliary_swa_component=auxiliary_swa, storage_prefetch_threshold=storage_prefetch_threshold, ) self.buffer_pipeline = self.components[ ComponentType.FULL ].create_buffer_pipeline( max_context_len=get_model().context_length or 0, - swa_window_pages=( - swa.full_window_pages - if swa is not None and self.tree_core.has_swa_host_pool - else 0 - ), + swa_component=swa, + auxiliary_swa_component=auxiliary_swa, # Leak backstop only: the queue is charged once per unique node # span across pools; a cap binding on live content would # drop-newest and punch storage holes. @@ -3676,11 +3680,6 @@ class UnifiedRadixCache(BasePrefixCache): def swa_protected_size(self) -> int: return self.tree_core.swa_protected_size() - def swa_transient_size(self) -> int: - if self.buffer_pipeline is None: - return 0 - return self.buffer_pipeline.swa_transient_size() - def mamba_protected_size(self) -> int: return self.tree_core.mamba_protected_size() diff --git a/rust/sglang-radix-tree/src/python_bindings.rs b/rust/sglang-radix-tree/src/python_bindings.rs index 20a95366b1..d0ff21f63a 100644 --- a/rust/sglang-radix-tree/src/python_bindings.rs +++ b/rust/sglang-radix-tree/src/python_bindings.rs @@ -260,6 +260,7 @@ fn pool_name_str(name: PoolName) -> &'static str { PoolName::Kv => "kv", PoolName::Mamba => "mamba", PoolName::Swa => "swa", + PoolName::AuxiliarySwa => "auxiliary_swa", PoolName::Indexer => "indexer", PoolName::DeepseekV4C4 => "deepseek_v4_c4", PoolName::DeepseekV4C4Indexer => "deepseek_v4_c4_indexer", @@ -288,6 +289,7 @@ fn parse_pool_name(name: &str) -> PyResult { "kv" => Ok(PoolName::Kv), "mamba" => Ok(PoolName::Mamba), "swa" => Ok(PoolName::Swa), + "auxiliary_swa" => Ok(PoolName::AuxiliarySwa), "indexer" => Ok(PoolName::Indexer), "deepseek_v4_c4" => Ok(PoolName::DeepseekV4C4), "deepseek_v4_c4_indexer" => Ok(PoolName::DeepseekV4C4Indexer), diff --git a/rust/sglang-radix-tree/src/unified_tree_core.rs b/rust/sglang-radix-tree/src/unified_tree_core.rs index 6ff9e5d523..2445e15320 100644 --- a/rust/sglang-radix-tree/src/unified_tree_core.rs +++ b/rust/sglang-radix-tree/src/unified_tree_core.rs @@ -395,6 +395,7 @@ pub enum PoolName { Kv, Mamba, Swa, + AuxiliarySwa, Indexer, DeepseekV4C4, DeepseekV4C4Indexer, diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index 702140ff4b..df62e9a032 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -138,6 +138,7 @@ class TestPrefillAdder(CustomTestCase): req.retracted_stain = False req.host_hit_length = 0 req.swa_host_hit_length = 0 + req.auxiliary_swa_host_hit_length = 0 req.mamba_host_hit_length = 0 req.storage_hit_length = 0 req.storage_hit_start = None diff --git a/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py b/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py index e25988c02e..297d348065 100644 --- a/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py +++ b/test/registered/unit/mem_cache/test_buffer_mode_sidecar.py @@ -406,7 +406,12 @@ class TestBufferModeSidecar(unittest.TestCase): pipeline = BufferModePipeline.__new__(BufferModePipeline) pipeline._cache = cache - pipeline._swa_window_pages = 1 + pipeline._swa_component = SimpleNamespace( + pool_name=PoolName.SWA, + component_type=ComponentType.SWA, + full_window_pages=1, + ) + pipeline._auxiliary_swa_component = None pipeline.write_backlog_cap = 8 pipeline.reset() pipeline._build_backup_transfers = lambda node_id, *, kv_only=False: ( diff --git a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py index 89d3d8930b..ab9b810b71 100644 --- a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py +++ b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py @@ -32,6 +32,7 @@ from sglang.srt.mem_cache.pool_host.dsa import DSAIndexerPoolHost from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -275,6 +276,12 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): pipeline = BufferModePipeline.__new__(BufferModePipeline) pipeline._cache = mock.Mock() pipeline._cache.components = {} + pipeline._swa_component = SimpleNamespace( + pool_name=PoolName.SWA, + component_type=ComponentType.SWA, + full_window_pages=1, + ) + pipeline._auxiliary_swa_component = None pipeline._cache.cache_controller.mem_pool_host.entry_map = { PoolName.SWA: SimpleNamespace( host_pool=SimpleNamespace(page_size=swa_page_size) @@ -304,39 +311,33 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): return pipeline def test_short_staged_swa_tail_keeps_complete_window(self): - """FULL-prefix growth trims only FULL; SWA keeps its complete window.""" + """Logical SWA coverage survives pruning resident rows from H2D.""" handle = CacheRequestHandle("r", 0) - pipeline = self._short_swa_tail_pipeline(swa_page_size=2) - pipeline._cache.tree_core.is_eagle = False - pipeline._cache.tree_core.match_full_device_prefix.return_value = (6, 1, 6) - pipeline._cache.tree_core.collect_full_device_indices.return_value = _indices( - 0, 6 - ) - req = SimpleNamespace( - rid="r", - cache_request_handle=handle, - prefix_len=0, - kv=SimpleNamespace(cache_protected_len=0), - ) - self.assertTrue(pipeline.prepare_staged_prefetch(req)) - self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (2, 4)) - pipeline.release_staged_hold.assert_not_called() - - pipeline = self._short_swa_tail_pipeline(swa_page_size=4) - pipeline._cache.tree_core.is_eagle = False - pipeline._cache.tree_core.match_full_device_prefix.return_value = (6, 1, 6) - pipeline._cache.tree_core.collect_full_device_indices.return_value = _indices( - 0, 6 - ) - req = SimpleNamespace( - rid="r", - cache_request_handle=handle, - prefix_len=0, - kv=SimpleNamespace(cache_protected_len=0), - ) - self.assertTrue(pipeline.prepare_staged_prefetch(req)) - self.assertEqual((req.host_hit_length, req.swa_host_hit_length), (2, 4)) - pipeline.release_staged_hold.assert_not_called() + for page_size in (2, 4): + for resident_prefix in (False, True): + with self.subTest(page_size=page_size, resident_prefix=resident_prefix): + pipeline = self._short_swa_tail_pipeline(swa_page_size=page_size) + tree = pipeline._cache.tree_core + tree.is_eagle = False + tree.match_full_device_prefix.return_value = (6, 1, 6) + tree.collect_full_device_indices.return_value = _indices(0, 6) + tree.swa_tombstone_ranges.return_value = ( + [] if resident_prefix else [(4, 6)] + ) + req = SimpleNamespace( + rid="r", + cache_request_handle=handle, + prefix_len=0, + kv=SimpleNamespace(cache_protected_len=0), + ) + self.assertTrue(pipeline.prepare_staged_prefetch(req)) + self.assertEqual(req.host_hit_length, 2) + self.assertEqual( + req.swa_host_hit_length, 2 if resident_prefix else 4 + ) + window = req.staged_prefetch_plan.windows[ComponentType.SWA] + self.assertEqual((window.start, window.end), (4, 8)) + pipeline.release_staged_hold.assert_not_called() def test_l2_transfer_maps_global_layers(self): host_pool = MHATokenToKVPoolHost.__new__(MHATokenToKVPoolHost) diff --git a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py index 41c3370666..6489db3e11 100644 --- a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py +++ b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py @@ -26,6 +26,7 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( ) from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.mem_cache.unified_radix_cache import ( UnifiedRadixCache, _OngoingPrefetch, @@ -47,6 +48,9 @@ def _staged_fixture(full_match=2): prefetch_anchor_info=lambda node: (None, None), match_full_device_prefix=Mock(return_value=(full_match, 1, full_match)), collect_full_device_indices=Mock(return_value=torch.arange(8)), + swa_tombstone_ranges=Mock( + side_effect=lambda key, start, end, **kwargs: [(start, end)] + ), root_node_handle=lambda extra_key=None: 0, inc_full_pin=Mock(), dec_full_pin=Mock(), @@ -93,6 +97,12 @@ def _staged_fixture(full_match=2): cache.cache_controller = cc pipeline = BufferModePipeline.__new__(BufferModePipeline) pipeline._cache = cache + pipeline._swa_component = SimpleNamespace( + pool_name=PoolName.SWA, + component_type=ComponentType.SWA, + full_window_pages=2, + ) + pipeline._auxiliary_swa_component = None pipeline.reset() cache.buffer_pipeline = pipeline pipeline.staged_prefetches[_REQ] = _StagedPrefetch( diff --git a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py index 113c5d6346..64540b0b2c 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py @@ -123,6 +123,7 @@ class TestSwaBackupAfterCompaction(unittest.TestCase): ) component = SimpleNamespace( component_type=ComponentType.SWA, + pool_name=PoolName.SWA, tree_core=SimpleNamespace(has_swa_host_pool=True), _collect_unbacked_swa_nodes=lambda n: [n], _unified_allocator=lambda: object(), diff --git a/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py b/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py index 8a341b0f17..150facf216 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py @@ -518,6 +518,7 @@ class TestLoadBackAdmission(_UnifiedHiCacheCase): req.storage_hit_start = None req.host_hit_is_storage = False req.host_loaded_length = 0 + req.auxiliary_swa_host_hit_length = 0 req.retracted_stain = False req.needs_host_load_back.return_value = True req.finished.return_value = False diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index e3d3684627..187aea8bfa 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -5428,7 +5428,7 @@ class UnifiedRadixCacheSuite: cons.sanity_check() def test_buffer_only_load_back_uses_full_behind_swa_tombstone(self): - """FULL-only rematch keeps resident FULL and loads the complete SWA window.""" + """FULL-only rematch keeps resident FULL and repairs missing SWA rows.""" self._skip_unsupported_hicache_test() if not self.cfg.has_swa: self.skipTest("masked overlap requires an SWA component") @@ -5523,12 +5523,13 @@ class UnifiedRadixCacheSuite: dispatched = {} def capture_swa_mapping(): - load = next(iter(cons.buffer_pipeline.ongoing_buffer_load_back.values())) - swa_xfer = next(t for t in load.aux_xfers if t.name == PoolName.SWA) + load = cons.cache_controller.load_queue[-1] + swa_xfer = next(t for t in load.pool_transfers if t.name == PoolName.SWA) dispatched["device"] = swa_xfer.device_indices.clone() - full_window = masked_full[-len(swa_xfer.device_indices) :] - old_window = mapping_before[-len(swa_xfer.device_indices) :] - expected = torch.where(old_window > 0, old_window, swa_xfer.device_indices) + full_window = masked_full[-window_tokens:] + old_window = mapping_before[-window_tokens:] + expected = old_window.clone() + expected[old_window <= 0] = swa_xfer.device_indices self.assertTrue( torch.equal( cons_alloc.full_to_swa_index_mapping[full_window], @@ -5546,7 +5547,7 @@ class UnifiedRadixCacheSuite: ) self.assertEqual(int(spliced.numel()), 0) self.assertEqual(req.prefix_len, len(seq), "FULL rematch was lost") - self.assertEqual(len(dispatched["device"]), window_tokens) + self.assertEqual(len(dispatched["device"]), window_tokens - live_tail) self.assertEqual(cons_alloc.full_available_size(), full_avail0) k, v = self._snapshot_full_kv(cons_alloc, masked_full) self.assertTrue(torch.equal(k, masked_kv[0]))