[MemCache] Split the SWA mapping move out of locked-full recovery as RecoverEvictedSWA (#43721)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
Jialin
parent
0c52cb1519
commit
e4cfc0c42a
@@ -98,6 +98,15 @@ class RecoverSWAWithLockedFull(ComponentAction, frozen=True):
|
||||
component_type: ComponentType = ComponentType.SWA
|
||||
|
||||
|
||||
class RecoverEvictedSWA(ComponentAction, frozen=True):
|
||||
"""Recover evicted SWA on kept Full IDs by moving the incoming SWA mapping,
|
||||
leaving Full cleanup and tree publication to the caller."""
|
||||
|
||||
component_type: ComponentType
|
||||
kept_full: torch.Tensor
|
||||
incoming_full: torch.Tensor
|
||||
|
||||
|
||||
class SWARebuild(ComponentAction, frozen=True):
|
||||
"""Rebuild a node's SWA value by translating its source full value, then store it."""
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
FreeComponentHostSlot,
|
||||
FreeDeviceKVFullOnly,
|
||||
RebuildFullToSWAMapping,
|
||||
RecoverEvictedSWA,
|
||||
RecoverSWAWithLockedFull,
|
||||
SWARebuild,
|
||||
)
|
||||
@@ -305,6 +306,21 @@ class SWAComponent(TreeComponent):
|
||||
swa.virtual_to_physical.index_fill_(0, incoming_pages, -1)
|
||||
swa.clear_inverse_history()
|
||||
|
||||
def _set_swa(
|
||||
self, kept_full: torch.Tensor, incoming_full: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Move incoming SWA mappings to kept Full IDs and return SWA indices."""
|
||||
unified = self._unified_allocator()
|
||||
if unified is not None:
|
||||
self._transfer_swa_pages(unified, kept_full, incoming_full)
|
||||
return self._translate_full_to_swa(kept_full)
|
||||
|
||||
allocator = self.cache.token_to_kv_pool_allocator
|
||||
swa_value = self._translate_full_to_swa(incoming_full)
|
||||
allocator.clear_full_to_swa_mapping(incoming_full)
|
||||
allocator.set_full_to_swa_mapping(kept_full, swa_value)
|
||||
return swa_value
|
||||
|
||||
def refresh_lru(
|
||||
self,
|
||||
phase: LRURefreshPhase,
|
||||
@@ -1538,29 +1554,19 @@ class SWAComponent(TreeComponent):
|
||||
for full, swa in zip(action.full_indices, action.swa_indices):
|
||||
alloc.set_full_to_swa_mapping(full, swa)
|
||||
return
|
||||
if isinstance(action, RecoverEvictedSWA):
|
||||
self._set_swa(action.kept_full, action.incoming_full)
|
||||
return
|
||||
if isinstance(action, RecoverSWAWithLockedFull):
|
||||
# Keep the locked full; hand the node the INCOMING ids' swa pages,
|
||||
# freeing only the incoming full, then store the swa on the node.
|
||||
swa_value = self._set_swa(action.kept_full, action.incoming_full)
|
||||
unified = self._unified_allocator()
|
||||
if unified is not None:
|
||||
# No `full_to_swa_index_mapping` here: the swa sub-pool's v2p IS
|
||||
# the mapping. Rebind page ownership, then free through the
|
||||
# composite -- its `swa_v2p_pages > 0` filter skips the
|
||||
# just-tombstoned swa side, releasing only the full one.
|
||||
self._transfer_swa_pages(
|
||||
unified, action.kept_full, action.incoming_full
|
||||
)
|
||||
# The composite skips the detached SWA side and releases Full.
|
||||
unified.free(action.incoming_full)
|
||||
self.tree_core.set_component_device_value(
|
||||
action.node_id,
|
||||
self.component_type,
|
||||
self._translate_full_to_swa(action.kept_full),
|
||||
)
|
||||
return
|
||||
swa_value = self._translate_full_to_swa(action.incoming_full)
|
||||
alloc.set_full_to_swa_mapping(action.kept_full, swa_value)
|
||||
alloc.clear_full_to_swa_mapping(action.incoming_full)
|
||||
alloc.free_full_segment(action.incoming_full, start_pos=0)
|
||||
else:
|
||||
alloc.free_full_segment(action.incoming_full, start_pos=0)
|
||||
self.tree_core.set_component_device_value(
|
||||
action.node_id, self.component_type, swa_value
|
||||
)
|
||||
|
||||
@@ -35,7 +35,10 @@ from test_multi_ended_allocator import _FakeUnifiedSWAKVPool # sibling fixture
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWATokenToKVPoolAllocator,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import RecoverSWAWithLockedFull
|
||||
from sglang.srt.mem_cache.unified_cache.cache_action import (
|
||||
RecoverEvictedSWA,
|
||||
RecoverSWAWithLockedFull,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.mem_cache.unified_cache.components.swa import SWAComponent
|
||||
from sglang.srt.mem_cache.unified_memory_pool import MHASubPoolSpec, UnifiedKVPool
|
||||
@@ -292,5 +295,65 @@ class TestStaticPoolPathUnchanged(unittest.TestCase):
|
||||
self.assertEqual((node_id, ct), (3, _SWA))
|
||||
|
||||
|
||||
class TestRecoverEvictedSWAHandler(_RecoverTestBase):
|
||||
"""`RecoverEvictedSWA` is the mapping-only step of locked-full recovery:
|
||||
the node's ids adopt the incoming swa pages, while the incoming FULL ids
|
||||
and the node's device value stay with the caller. Folding either into the
|
||||
shared step would double-free under a tree core that still owns them."""
|
||||
|
||||
def test_unified_pool_moves_ownership_and_frees_nothing(self):
|
||||
probe, allocator = self._probe()
|
||||
swa = allocator.swa_attn_allocator
|
||||
kept, incoming = self._two_ranges(allocator)
|
||||
allocator.free_swa(kept) # what eviction does when it tombstones
|
||||
swa_live = swa.allocated_count()
|
||||
full_avail = allocator.full_attn_allocator.available_size()
|
||||
|
||||
probe.apply_component_action(
|
||||
RecoverEvictedSWA(
|
||||
component_type=_SWA, kept_full=kept, incoming_full=incoming
|
||||
)
|
||||
)
|
||||
|
||||
self.assertTrue(
|
||||
bool((allocator.translate_loc_from_full_to_swa(kept) > 0).all()),
|
||||
"the node's ids must address live swa pages after recovery",
|
||||
)
|
||||
self.assertTrue(
|
||||
bool((swa.virtual_to_physical[incoming.to(torch.int64)] == -1).all()),
|
||||
"the incoming ids' swa side must be tombstoned",
|
||||
)
|
||||
self.assertEqual(swa.allocated_count(), swa_live, "ownership only moved")
|
||||
self.assertEqual(
|
||||
allocator.full_attn_allocator.available_size(),
|
||||
full_avail,
|
||||
"the incoming FULL side belongs to the caller",
|
||||
)
|
||||
self.assertEqual(probe.tree_core.set_calls, [], "publication is the caller's")
|
||||
|
||||
def test_static_allocator_moves_the_mapping_and_frees_nothing(self):
|
||||
static = _StaticAllocRecorder()
|
||||
probe = _Probe.__new__(_Probe)
|
||||
probe.cache = _Cache(static)
|
||||
probe.tree_core = _StubTreeCore()
|
||||
|
||||
kept = torch.tensor([1, 2], dtype=torch.int64)
|
||||
incoming = torch.tensor([5, 6], dtype=torch.int64)
|
||||
donated = static.full_to_swa_index_mapping[incoming].clone()
|
||||
probe.apply_component_action(
|
||||
RecoverEvictedSWA(
|
||||
component_type=_SWA, kept_full=kept, incoming_full=incoming
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
static.full_to_swa_index_mapping[kept].tolist(), donated.tolist()
|
||||
)
|
||||
self.assertEqual(len(static.clear_calls), 1, "incoming must be tombstoned")
|
||||
self.assertEqual(static.freed_full, [])
|
||||
self.assertEqual(static.freed_via_inner, [])
|
||||
self.assertEqual(probe.tree_core.set_calls, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user