[Perf] unified-memory: Free SWA segments without a host sync (#43170)

Co-authored-by: Caihua Li <caihua.li@bytedance.com>
This commit is contained in:
caihuali95
2026-10-18 05:36:56 -07:00
committed by GitHub
co-authored by Caihua Li
parent 47eeddcd18
commit ba50ea3d77
5 changed files with 190 additions and 50 deletions
@@ -29,6 +29,13 @@ if _is_npu:
_SWA_PEER_MAPPED = Invariant("swa.peer_mapped", Bucket.FATAL_UNCONTAINABLE, IsTrue())
# free_full leaves the mapping alone, so a live entry would strand its SWA peer.
_SWA_PEER_RELEASED = Invariant("swa.peer_released", Bucket.GUARD, IsTrue())
# The unified allocator's segment frees trust the caller's bound/released split.
UNIFIED_SWA_SEGMENT_BOUND = Invariant(
"unified_swa.segment_bound", Bucket.FATAL_UNCONTAINABLE, IsTrue()
)
UNIFIED_SWA_SEGMENT_RELEASED = Invariant(
"unified_swa.segment_released", Bucket.FATAL_UNCONTAINABLE, IsTrue()
)
class SWATokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
@@ -26,7 +26,11 @@ from torch.profiler import record_function
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.allocator.base import MambaFullCacheDonor
from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator
from sglang.srt.mem_cache.allocator.swa import (
UNIFIED_SWA_SEGMENT_BOUND,
UNIFIED_SWA_SEGMENT_RELEASED,
SWATokenToKVPoolAllocator,
)
from sglang.srt.mem_cache.allocator.unified_sub_pool import (
FloatMultiEndedAllocator,
MultiEndedAllocator,
@@ -41,6 +45,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import (
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
from sglang.srt.runtime_context import get_disagg, get_memory
from sglang.srt.utils.common import get_num_new_pages
from sglang.srt.utils.invariants import expect
from sglang.srt.utils.rank_consensus_checker import rank_consensus
logger = logging.getLogger(__name__)
@@ -172,6 +177,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
self.free_group = None
self.free_page_reps_group: Optional[List[torch.Tensor]] = None
self.full_page_reps_group: List[torch.Tensor] = []
self.full_free_group: List[torch.Tensor] = []
# Empty (not None) for the leak checker.
self.free_pages = torch.empty(0, dtype=torch.int64, device=device)
@@ -612,27 +618,25 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
self.swa_attn_allocator.clear_inverse_history()
def free_swa_segment(self, free_index: torch.Tensor, *, start_pos: int) -> None:
"""free_swa() for a kv-row segment: `start_pos` promises a contiguous
ascending range, so page reps come from stride arithmetic, not `torch.unique`."""
"""free_swa() for a kv-row segment whose swa pages are all bound: page reps
come from `start_pos`, so it is fixed-shape at every page size."""
if free_index is None or free_index.numel() == 0:
return
if self.page_size == 1:
self.free_swa(free_index)
return
ps = self.page_size
reps = self.swa_attn_allocator._page_reps(
free_index.detach().to(torch.int64), start_pos
)
# Keep only pages still bound on swa; freeing a tombstoned one would
# corrupt the hole list. `> 0` strict: -1 tombstoned, 0 padding sink.
rep_pages = reps // ps
swa_v2p_pages = self.swa_attn_allocator.virtual_to_physical[rep_pages]
live_reps = reps[swa_v2p_pages > 0]
if live_reps.numel() == 0:
return
self.swa_attn_allocator.free(live_reps, _pages=live_reps // ps)
self._free_swa_page_reps(reps)
self.swa_attn_allocator.clear_inverse_history()
def _free_swa_page_reps(self, reps: torch.Tensor) -> None:
v_pages = reps // self.page_size
expect(
UNIFIED_SWA_SEGMENT_BOUND,
self.swa_attn_allocator.virtual_to_physical[v_pages] > 0,
msg="caller wants free_full_segment",
)
self.swa_attn_allocator.free(reps, _pages=v_pages)
def free_full(self, free_index: torch.Tensor) -> None:
"""Release the full-physical page and the virtual id, leaving the swa
side alone -- the caller already tombstoned it (`swa.v2p_page == -1`)."""
@@ -645,15 +649,21 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
self.full_attn_allocator.clear_inverse_history()
def free_full_segment(self, free_index: torch.Tensor, *, start_pos: int) -> None:
"""Release a kv-row segment's full side; its swa side is already tombstoned."""
if free_index is None or free_index.numel() == 0:
return
if self.page_size == 1:
# token == page: free_full already frees by exact ids, no dedup.
self.free_full(free_index)
return
# The swa v2p is the mapping, so a tombstoned swa page drops out of the
# two-sided segment path by itself; full-only is the same call.
self.free_segment(free_index, start_pos=start_pos)
reps = self.full_attn_allocator._page_reps(
free_index.detach().to(torch.int64), start_pos
)
expect(
UNIFIED_SWA_SEGMENT_RELEASED,
self.swa_attn_allocator.virtual_to_physical[reps // self.page_size] <= 0,
msg="caller wants free_segment",
)
if self.free_page_reps_group is None:
self._release_page_reps((), (reps,))
else:
self.full_page_reps_group.append(reps)
def set_full_to_swa_mapping(
self, full_indices: torch.Tensor, swa_indices: torch.Tensor
@@ -676,44 +686,44 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
def free_group_begin(self) -> None:
BaseTokenToKVPoolAllocator.free_group_begin(self)
self.free_page_reps_group = []
self.full_page_reps_group = []
self.full_free_group = []
def free_group_end(self) -> None:
pending, self.free_page_reps_group = self.free_page_reps_group, None
full_pending, self.full_page_reps_group = self.full_page_reps_group, []
full_free_group, self.full_free_group = self.full_free_group, []
BaseTokenToKVPoolAllocator.free_group_end(self)
if full_free_group:
self.full_attn_allocator.free(torch.cat(full_free_group))
self.full_attn_allocator.clear_inverse_history()
if pending:
self._release_page_reps(pending)
if pending or full_pending:
self._release_page_reps(pending, full_pending)
def free_segment(self, free_index: torch.Tensor, *, start_pos: int) -> None:
"""Fixed-shape counterpart of `free()`; see `MultiEndedAllocator._page_reps`.
Both sides share one page-rep derivation instead of dedup'ing twice."""
"""Fixed-shape counterpart of `free()` for a segment whose swa pages are all
bound; see `MultiEndedAllocator._page_reps`."""
if free_index is None or free_index.numel() == 0:
return
if self.page_size == 1:
self.free(free_index)
return
reps = self.full_attn_allocator._page_reps(
free_index.detach().to(torch.int64), start_pos
)
if self.free_page_reps_group is None:
self._release_page_reps((reps,))
self._release_page_reps((reps,), ())
else:
self.free_page_reps_group.append(reps)
def _release_page_reps(self, pieces: Sequence[torch.Tensor]) -> None:
reps = pieces[0] if len(pieces) == 1 else torch.cat(tuple(pieces))
v_pages = reps // self.page_size
# Same tombstone filter as `free`, but at PAGE granularity (page_size
# times smaller): `> 0` strict -- -1 = tombstoned, 0 = padding sink.
swa_v2p_pages = self.swa_attn_allocator.virtual_to_physical[v_pages]
live_pages = v_pages[swa_v2p_pages > 0]
if live_pages.numel() > 0:
self.swa_attn_allocator.free(live_pages * self.page_size, _pages=live_pages)
self.full_attn_allocator.free(reps, _pages=v_pages)
def _release_page_reps(
self, both: Sequence[torch.Tensor], full_only: Sequence[torch.Tensor]
) -> None:
"""Free page reps: `both` on the swa and full sides, `full_only` on full."""
if both:
self._free_swa_page_reps(
both[0] if len(both) == 1 else torch.cat(tuple(both))
)
pieces = (*both, *full_only)
reps = pieces[0] if len(pieces) == 1 else torch.cat(pieces)
self.full_attn_allocator.free(reps, _pages=reps // self.page_size)
self.full_attn_allocator.clear_inverse_history()
self.swa_attn_allocator.clear_inverse_history()
@@ -738,6 +748,7 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
self.swa_attn_allocator.clear()
self.free_group = None
self.free_page_reps_group = None
self.full_page_reps_group = []
self.full_free_group = []
# -- Lazy compaction hooks --
@@ -1217,7 +1228,12 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
required_swa = num_tokens if swa_num_tokens is None else swa_num_tokens
_flush_deferred_free_group(
self,
(self.free_group, self.free_page_reps_group, self.full_free_group),
(
self.free_group,
self.free_page_reps_group,
self.full_free_group,
self.full_page_reps_group,
),
)
reclaim_plan = self.reclaim_plan(
num_tokens,
@@ -1724,7 +1740,12 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
"""Expose grouped composite frees while preserving the group scope."""
_flush_deferred_free_group(
self,
(self.free_group, self.free_page_reps_group, self.full_free_group),
(
self.free_group,
self.free_page_reps_group,
self.full_page_reps_group,
self.full_free_group,
),
)
def full_tokens_before_mamba_recheck(self, target_size: int) -> int:
@@ -25,6 +25,7 @@ Mirrors `test_paged_free_segment.py`, which pins the same properties for
"""
import ast
import importlib.util
import inspect
import textwrap
import unittest
@@ -32,10 +33,14 @@ from unittest import mock
import torch
from test_multi_ended_allocator import TestPagedMultiEndedAllocator as _PagedFixture
from torch.utils._python_dispatch import TorchDispatchMode
from sglang.srt.environ import InvariantCheckLevel, envs
from sglang.srt.mem_cache.allocator import unified_hybrid_swa, unified_mamba
from sglang.srt.mem_cache.allocator import unified_sub_pool as mea
from sglang.srt.mem_cache.allocator.base import BaseTokenToKVPoolAllocator
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
@@ -51,6 +56,39 @@ def _paged_allocator(lazy: bool):
return full
_aten = torch.ops.aten
# Output shape depends on tensor values: on CUDA each one copies a count to the
# host and blocks it until the stream drains.
_SHAPE_SYNC_OPS = {
_aten.nonzero.default,
_aten.masked_select.default,
_aten._local_scalar_dense.default,
_aten._unique2.default,
_aten.unique_dim.default,
_aten.unique_consecutive.default,
}
_INDEX_OPS = {_aten.index.Tensor, _aten.index_put.default, _aten.index_put_.default}
class _ShapeSyncRecorder(TorchDispatchMode):
"""Records every host-syncing op a call dispatches. A boolean-mask index is
one: its `nonzero` runs inside the index kernel, below this mode."""
def __init__(self):
super().__init__()
self.hits = []
def __torch_dispatch__(self, func, types, args=(), kwargs=None):
kwargs = kwargs or {}
if func in _SHAPE_SYNC_OPS or (
func in _INDEX_OPS
and any(i is not None and i.dtype == torch.bool for i in args[1])
):
self.hits.append(str(func))
return func(*args, **kwargs)
# --------------------------------------------------------------------------
# 1. tombstone scatters
# --------------------------------------------------------------------------
@@ -331,6 +369,33 @@ class TestUnifiedSwaFullSideGroup(unittest.TestCase):
"""`free_full` inside a free group must defer and land at `free_group_end`;
the composite carries that pile itself, not through the SWA parent's hooks."""
def setUp(self):
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
def test_allocation_reclaim_flushes_full_only_segment_frees(self):
"""FULL-only queued pages already satisfy this allocation. Ignoring
their queue makes two-pool reclaim report insufficient capacity."""
from test_unified_swa_shared_virtual_ids import _build
alloc = _build(64, 32, 2, 4)
indices = alloc.alloc(8)
alloc.free_swa(indices)
before = alloc.full_available_size()
alloc.free_group_begin()
self.addCleanup(alloc.free_group_end)
alloc.free_full_segment(indices, start_pos=0)
self.assertEqual(alloc.full_available_size(), before)
cache = mock.Mock()
cache.supports_prefix_sharing.return_value = True
cache.full_evictable_size.return_value = 0
cache.swa_evictable_size.return_value = 0
cache.evict_for_alloc.side_effect = AssertionError("no tree eviction needed")
self.assertTrue(alloc.evict_to_free_tokens(cache, before + 4, swa_num_tokens=0))
self.assertEqual(alloc.full_available_size(), before + 8)
def test_full_only_frees_defer_until_group_end(self):
from test_unified_swa_shared_virtual_ids import _build
@@ -356,7 +421,7 @@ class TestFreeSwaWindowRatchetNoHostSync(unittest.TestCase):
PS = 4
def _swa_composite(self, lazy=True):
def _swa_composite(self, lazy=True, ps=None):
from test_multi_ended_allocator import _FakeKVCache, _make_mha_spec
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
@@ -369,7 +434,7 @@ class TestFreeSwaWindowRatchetNoHostSync(unittest.TestCase):
sub_pool_specs=[full, swa],
device="cpu",
enable_memory_saver=False,
page_size=self.PS,
page_size=ps or self.PS,
)
class _KV:
@@ -386,7 +451,7 @@ class TestFreeSwaWindowRatchetNoHostSync(unittest.TestCase):
device="cpu",
full_max_total_num_tokens=64,
swa_max_total_num_tokens=64,
page_size=self.PS,
page_size=ps or self.PS,
need_sort=False,
forward_stream=None,
lazy_compaction=lazy,
@@ -458,13 +523,58 @@ class TestFreeSwaWindowRatchetNoHostSync(unittest.TestCase):
a2.swa_attn_allocator.schedulable_available_size(),
)
def test_double_ratchet_is_filtered_not_crashed(self):
"""Freeing an already-tombstoned range again must no-op through the
liveness filter (radix eviction and the ratchet can overlap)."""
def test_segment_frees_dispatch_no_data_dependent_shape(self):
"""Bug regression: the ratchet, a request's release and a tree eviction
must not filter by liveness on the device; a boolean-mask index blocks
the scheduler on the in-flight forward at every prefill."""
recorder = _ShapeSyncRecorder()
with recorder:
torch.arange(4)[torch.arange(4) > 1]
self.assertEqual(len(recorder.hits), 1, "the recorder went blind")
for ps in (1, self.PS):
with self.subTest(page_size=ps):
alloc = self._swa_composite(lazy=True, ps=ps)
full_before = alloc.full_available_size()
swa_before = alloc.swa_available_size()
v = alloc.alloc(8 * ps)
recorder = _ShapeSyncRecorder()
with recorder:
alloc.free_swa_segment(v[: 2 * ps], start_pos=0)
alloc.free_group_begin()
alloc.free_full_segment(v[: 2 * ps], start_pos=0)
alloc.free_segment(v[2 * ps : 5 * ps], start_pos=2 * ps)
alloc.free_group_end()
alloc.free_segment(v[5 * ps :], start_pos=5 * ps)
self.assertEqual(recorder.hits, [])
self.assertEqual(alloc.full_available_size(), full_before)
self.assertEqual(alloc.swa_available_size(), swa_before)
def test_a_double_ratchet_is_reported(self):
"""A segment free trusts its caller that the swa pages are bound (the
ratchet's cursor only moves up); a second free must be loud under
strict checks, not corrupt the swa hole list."""
alloc = self._swa_composite(lazy=True)
v = alloc.alloc(4 * self.PS)
alloc.free_swa_segment(v, start_pos=0)
alloc.free_swa_segment(v, start_pos=0) # all tombstoned -> filtered to empty
with envs.SGLANG_INVARIANT_CHECK.override(int(InvariantCheckLevel.STRICT)):
with self.assertRaises(RuntimeError):
alloc.free_swa_segment(v, start_pos=0)
def test_a_full_only_free_of_a_bound_swa_page_is_reported(self):
"""`free_full_segment` releases the full side only; a swa page still
bound under it would leak, so the caller wanted `free_segment`."""
alloc = self._swa_composite(lazy=True)
v = alloc.alloc(4 * self.PS)
with envs.SGLANG_INVARIANT_CHECK.override(int(InvariantCheckLevel.STRICT)):
with self.assertRaises(RuntimeError):
alloc.free_full_segment(v, start_pos=0)
def test_importing_the_module_again_registers_nothing_twice(self):
"""A fixture that patches `sys.modules` drops the modules first imported
under the patch, so the next test imports them again; the invariant
registry refuses a name it already holds."""
spec = importlib.util.find_spec(unified_hybrid_swa.__name__)
spec.loader.exec_module(importlib.util.module_from_spec(spec))
@unittest.skipUnless(
@@ -88,6 +88,7 @@ class TestUnifiedRadixAllocationEviction(CustomTestCase):
allocator.free_group = None
allocator.free_page_reps_group = None
if tri_pool:
allocator.full_page_reps_group = []
allocator.full_free_group = []
allocator.full_attn_allocator = MagicMock()
allocator.full_attn_allocator.schedulable_available_size.return_value = 100
@@ -494,7 +494,7 @@ class TestTriPagedFreeGroup(unittest.TestCase):
allocator.free_group_begin()
allocator.free_full_segment(full_indices, start_pos=0)
self.assertTrue(allocator.full_free_group)
self.assertTrue(allocator.full_page_reps_group)
donor = allocator.mamba_full_cache_donor()
self.assertIsNotNone(donor)
@@ -502,6 +502,7 @@ class TestTriPagedFreeGroup(unittest.TestCase):
self.assertEqual(allocator.free_group, [])
self.assertEqual(allocator.free_page_reps_group, [])
self.assertEqual(allocator.full_page_reps_group, [])
self.assertEqual(allocator.full_free_group, [])
self.assertLess(
allocator.full_attn_allocator.allocated_count(), allocated_before