[Perf] unified-memory: Reduce allocation and SWA retirement overhead (#44970)

This commit is contained in:
Cheng Wan
2026-10-18 05:40:25 -07:00
committed by GitHub
parent a5b21145e1
commit d318a546da
7 changed files with 226 additions and 34 deletions
@@ -28,7 +28,7 @@ import triton.language as tl
# `.item()`, no tensor branching); runs on the scheduler thread.
@triton.jit
@triton.jit(do_not_specialize=["v_pages_ptr"])
def alloc_bind_inplace_kernel(
v_pages_ptr, # in: [N] int64 — virtual page ids
v2p_ptr, # in/out: int64 — virtual_to_physical table
@@ -126,7 +126,7 @@ def free_unbind_inplace_kernel(
tl.store(p2v_ptr + p, -1, mask=mask)
@triton.jit
@triton.jit(do_not_specialize=["v_pages_ptr", "p_pages_ptr"])
def bind_inplace_kernel(
v_pages_ptr, # in: [N] int64 — virtual page ids
p_pages_ptr, # in: [N] int64 — physical page ids to bind them to
@@ -24,6 +24,7 @@ from typing import Callable, Hashable, List, Optional, Sequence, Tuple
import torch
from torch.profiler import record_function
from sglang.srt.environ import InvariantCheckLevel
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.allocator.base import MambaFullCacheDonor
from sglang.srt.mem_cache.allocator.swa import (
@@ -45,7 +46,7 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import (
from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool
from sglang.srt.runtime_context import get_disagg, get_memory
from sglang.srt.utils.common import get_num_new_pages
from sglang.srt.utils.invariants import expect
from sglang.srt.utils.invariants import expect, resolve_level
from sglang.srt.utils.rank_consensus_checker import rank_consensus
logger = logging.getLogger(__name__)
@@ -426,10 +427,10 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
if not self.ensure_capacity(need_tokens, swa_tokens):
return None
# Snapshot the virtual PAGES the kernel will consume; clone so swa keeps
# its view after the slice is consumed.
# free_virtual_ids is rebound (slice/cat), never mutated in place,
# so this prefix view stays valid for SWA after FULL consumes it.
fa = self.full_attn_allocator
new_virtual_pages = fa.free_virtual_ids[:num_new_pages].clone()
new_virtual_pages = fa.free_virtual_ids[:num_new_pages]
out_indices = fa.alloc_extend(
prefix_lens,
@@ -630,11 +631,14 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
def _free_swa_page_reps(self, reps: torch.Tensor) -> None:
v_pages = reps // self.page_size
expect(
UNIFIED_SWA_SEGMENT_BOUND,
self.swa_attn_allocator.virtual_to_physical[v_pages] > 0,
msg="caller wants free_full_segment",
)
# Signal-only invariant: avoid launching its lookup and comparison
# when detection is disabled. This invariant has no recovery action.
if resolve_level() >= InvariantCheckLevel.WARN:
expect(
UNIFIED_SWA_SEGMENT_BOUND,
self.swa_attn_allocator.virtual_to_physical[v_pages] > 0,
msg="caller wants free_full_segment",
)
self.swa_attn_allocator.free(reps, _pages=v_pages)
def free_full(self, free_index: torch.Tensor) -> None:
@@ -655,11 +659,15 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
reps = self.full_attn_allocator._page_reps(
free_index.detach().to(torch.int64), start_pos
)
expect(
UNIFIED_SWA_SEGMENT_RELEASED,
self.swa_attn_allocator.virtual_to_physical[reps // self.page_size] <= 0,
msg="caller wants free_segment",
)
# Signal-only invariant: avoid launching its lookup and comparison
# when detection is disabled. This invariant has no recovery action.
if resolve_level() >= InvariantCheckLevel.WARN:
expect(
UNIFIED_SWA_SEGMENT_RELEASED,
self.swa_attn_allocator.virtual_to_physical[reps // self.page_size]
<= 0,
msg="caller wants free_segment",
)
if self.free_page_reps_group is None:
self._release_page_reps((), (reps,))
else:
@@ -1685,10 +1693,32 @@ class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
h_s = sa._hole_pages()
r_f = fa.num_pages - fa.min_page_index - fa._allocated_pages()
r_s = sa.num_pages - sa.min_page_index - sa._allocated_pages()
# Allocator state is stable throughout this query. Snapshot the band
# geometry once instead of walking the chain at every binary-search step.
full_gap = fa._current_gap_bytes()
full_low = fa._byte_low_frontier()
full_page_bytes = fa.entry_bytes_per_page
swa_chain_low = sa._chain_high_frontier_below_bytes()
if sa._is_frontier_transparent():
swa_high_band_low = swa_chain_low
swa_low_room = 0
else:
swa_high_band_low = sa._byte_high_frontier()
swa_low_room = sa.pages_in_band(
low_byte=swa_chain_low, high_byte=sa._byte_low_frontier()
)
lo_n, hi_n = 0, min(h_f + r_f, h_s + r_s)
while lo_n < hi_n:
mid = (lo_n + hi_n + 1) // 2
if self._fits_page_demand(mid, mid):
full_bytes = max(0, mid - h_f) * full_page_bytes
swa_room = max(
swa_low_room,
sa.pages_in_band(
low_byte=swa_high_band_low, high_byte=full_low - full_bytes
),
)
if full_bytes <= full_gap and max(0, mid - h_s) <= swa_room:
lo_n = mid
else:
hi_n = mid - 1
@@ -1187,10 +1187,10 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
if not _relieve_for_alloc(self, need_tokens):
return None
bs = len(prefix_lens)
# Snapshot the virtual pages the kernel will consume, to bind them
# to physical pages afterward.
# The free list is only rebound, not mutated in place. Retain its
# consumed slice for binding without launching a redundant copy.
if num_new_pages > 0:
new_virtual_pages = self.free_virtual_ids[:num_new_pages].clone()
new_virtual_pages = self.free_virtual_ids[:num_new_pages]
else:
new_virtual_pages = None
@@ -2424,10 +2424,13 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
free_v_pages = free_v_pages_raw
else:
free_v_pages = torch.unique(free_v_pages_raw // self.page_size)
freed_p_pages = self.virtual_to_physical[free_v_pages]
# `index_fill_`, never `t[idx] = -1`: see the END free path.
self.virtual_to_physical.index_fill_(0, free_v_pages, -1)
self.physical_to_virtual.index_fill_(0, freed_p_pages, -1)
# The same fused lookup + tombstones as the END free path avoids
# three separate launches on each sliding-window retirement.
freed_p_pages = free_unbind_inplace(
free_v_pages.contiguous(),
self.virtual_to_physical,
self.physical_to_virtual,
)
if self.is_id_owner:
self.free_virtual_ids = torch.cat([self.free_virtual_ids, free_v_pages])
self._free_phys_pages = torch.cat([self._free_phys_pages, freed_p_pages])