From e4cfc0c42a41c48b239f50f885703f795dbacf5d Mon Sep 17 00:00:00 2001 From: metamergebot Date: Sun, 11 Oct 2026 20:11:52 -0700 Subject: [PATCH] [MemCache] Split the SWA mapping move out of locked-full recovery as `RecoverEvictedSWA` (#43721) Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: Jialin --- .../mem_cache/unified_cache/cache_action.py | 9 +++ .../mem_cache/unified_cache/components/swa.py | 40 +++++++----- .../test_swa_locked_full_recover_unified.py | 65 ++++++++++++++++++- 3 files changed, 96 insertions(+), 18 deletions(-) diff --git a/python/sglang/srt/mem_cache/unified_cache/cache_action.py b/python/sglang/srt/mem_cache/unified_cache/cache_action.py index e94ebdb92e..cb7066744d 100644 --- a/python/sglang/srt/mem_cache/unified_cache/cache_action.py +++ b/python/sglang/srt/mem_cache/unified_cache/cache_action.py @@ -98,6 +98,15 @@ class RecoverSWAWithLockedFull(ComponentAction, frozen=True): component_type: ComponentType = ComponentType.SWA +class RecoverEvictedSWA(ComponentAction, frozen=True): + """Recover evicted SWA on kept Full IDs by moving the incoming SWA mapping, + leaving Full cleanup and tree publication to the caller.""" + + component_type: ComponentType + kept_full: torch.Tensor + incoming_full: torch.Tensor + + class SWARebuild(ComponentAction, frozen=True): """Rebuild a node's SWA value by translating its source full value, then store it.""" diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index 5458debfbb..bc52833a22 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -27,6 +27,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import ( FreeComponentHostSlot, FreeDeviceKVFullOnly, RebuildFullToSWAMapping, + RecoverEvictedSWA, RecoverSWAWithLockedFull, SWARebuild, ) @@ -305,6 +306,21 @@ class SWAComponent(TreeComponent): swa.virtual_to_physical.index_fill_(0, incoming_pages, -1) swa.clear_inverse_history() + def _set_swa( + self, kept_full: torch.Tensor, incoming_full: torch.Tensor + ) -> torch.Tensor: + """Move incoming SWA mappings to kept Full IDs and return SWA indices.""" + unified = self._unified_allocator() + if unified is not None: + self._transfer_swa_pages(unified, kept_full, incoming_full) + return self._translate_full_to_swa(kept_full) + + allocator = self.cache.token_to_kv_pool_allocator + swa_value = self._translate_full_to_swa(incoming_full) + allocator.clear_full_to_swa_mapping(incoming_full) + allocator.set_full_to_swa_mapping(kept_full, swa_value) + return swa_value + def refresh_lru( self, phase: LRURefreshPhase, @@ -1538,29 +1554,19 @@ class SWAComponent(TreeComponent): for full, swa in zip(action.full_indices, action.swa_indices): alloc.set_full_to_swa_mapping(full, swa) return + if isinstance(action, RecoverEvictedSWA): + self._set_swa(action.kept_full, action.incoming_full) + return if isinstance(action, RecoverSWAWithLockedFull): # Keep the locked full; hand the node the INCOMING ids' swa pages, # freeing only the incoming full, then store the swa on the node. + swa_value = self._set_swa(action.kept_full, action.incoming_full) unified = self._unified_allocator() if unified is not None: - # No `full_to_swa_index_mapping` here: the swa sub-pool's v2p IS - # the mapping. Rebind page ownership, then free through the - # composite -- its `swa_v2p_pages > 0` filter skips the - # just-tombstoned swa side, releasing only the full one. - self._transfer_swa_pages( - unified, action.kept_full, action.incoming_full - ) + # The composite skips the detached SWA side and releases Full. unified.free(action.incoming_full) - self.tree_core.set_component_device_value( - action.node_id, - self.component_type, - self._translate_full_to_swa(action.kept_full), - ) - return - swa_value = self._translate_full_to_swa(action.incoming_full) - alloc.set_full_to_swa_mapping(action.kept_full, swa_value) - alloc.clear_full_to_swa_mapping(action.incoming_full) - alloc.free_full_segment(action.incoming_full, start_pos=0) + else: + alloc.free_full_segment(action.incoming_full, start_pos=0) self.tree_core.set_component_device_value( action.node_id, self.component_type, swa_value ) diff --git a/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py b/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py index cbdc9eef60..c7b24cf452 100644 --- a/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py +++ b/test/registered/unit/mem_cache/test_swa_locked_full_recover_unified.py @@ -35,7 +35,10 @@ from test_multi_ended_allocator import _FakeUnifiedSWAKVPool # sibling fixture from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedSWATokenToKVPoolAllocator, ) -from sglang.srt.mem_cache.unified_cache.cache_action import RecoverSWAWithLockedFull +from sglang.srt.mem_cache.unified_cache.cache_action import ( + RecoverEvictedSWA, + RecoverSWAWithLockedFull, +) from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool @@ -292,5 +295,65 @@ class TestStaticPoolPathUnchanged(unittest.TestCase): self.assertEqual((node_id, ct), (3, _SWA)) +class TestRecoverEvictedSWAHandler(_RecoverTestBase): + """`RecoverEvictedSWA` is the mapping-only step of locked-full recovery: + the node's ids adopt the incoming swa pages, while the incoming FULL ids + and the node's device value stay with the caller. Folding either into the + shared step would double-free under a tree core that still owns them.""" + + def test_unified_pool_moves_ownership_and_frees_nothing(self): + probe, allocator = self._probe() + swa = allocator.swa_attn_allocator + kept, incoming = self._two_ranges(allocator) + allocator.free_swa(kept) # what eviction does when it tombstones + swa_live = swa.allocated_count() + full_avail = allocator.full_attn_allocator.available_size() + + probe.apply_component_action( + RecoverEvictedSWA( + component_type=_SWA, kept_full=kept, incoming_full=incoming + ) + ) + + self.assertTrue( + bool((allocator.translate_loc_from_full_to_swa(kept) > 0).all()), + "the node's ids must address live swa pages after recovery", + ) + self.assertTrue( + bool((swa.virtual_to_physical[incoming.to(torch.int64)] == -1).all()), + "the incoming ids' swa side must be tombstoned", + ) + self.assertEqual(swa.allocated_count(), swa_live, "ownership only moved") + self.assertEqual( + allocator.full_attn_allocator.available_size(), + full_avail, + "the incoming FULL side belongs to the caller", + ) + self.assertEqual(probe.tree_core.set_calls, [], "publication is the caller's") + + def test_static_allocator_moves_the_mapping_and_frees_nothing(self): + static = _StaticAllocRecorder() + probe = _Probe.__new__(_Probe) + probe.cache = _Cache(static) + probe.tree_core = _StubTreeCore() + + kept = torch.tensor([1, 2], dtype=torch.int64) + incoming = torch.tensor([5, 6], dtype=torch.int64) + donated = static.full_to_swa_index_mapping[incoming].clone() + probe.apply_component_action( + RecoverEvictedSWA( + component_type=_SWA, kept_full=kept, incoming_full=incoming + ) + ) + + self.assertEqual( + static.full_to_swa_index_mapping[kept].tolist(), donated.tolist() + ) + self.assertEqual(len(static.clear_calls), 1, "incoming must be tombstoned") + self.assertEqual(static.freed_full, []) + self.assertEqual(static.freed_via_inner, []) + self.assertEqual(probe.tree_core.set_calls, []) + + if __name__ == "__main__": unittest.main()