[Perf] unified-memory: Prewarm the compaction move at startup and copy whole page envelopes by page id (#44592)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-16 16:47:50 -07:00
committed by GitHub
co-authored by metamergebot ZYHowell
parent aaf5f397a5
commit 36cf028346
15 changed files with 177 additions and 64 deletions
@@ -7,7 +7,10 @@ import triton
import triton.language as tl
@triton.jit
# Callers pass page-id slices at any element offset (compaction launches a long
# move in chunks); each program loads only one scalar id, so specializing on
# their alignment would only compile more variants, and those mid-serving.
@triton.jit(do_not_specialize_on_alignment=["src_pages", "dst_pages"])
def _copy_pages_kernel(
buf,
src_pages,
+4
View File
@@ -572,6 +572,10 @@ class Envs:
# Per-call move cap on a non-urgent lazy-compaction flush, so a large
# backlog cannot stall the scheduler loop; urgent flushes are uncapped.
SGLANG_LAZY_COMPACTION_MAX_MOVES_PER_CALL = EnvInt(4096)
# Max pages one compaction move-kernel launch copies (eager or lazy, urgent
# or not); a larger move is issued as consecutive launches of this size
# within the same compaction call.
SGLANG_COMPACTION_MAX_PAGES_PER_MOVE = EnvInt(4096)
# HND KV layout folds (page, head) into one paged index for per-kv-head sparse
# page tables (DP attn); paged backends like trtllm_mha consume it directly.
SGLANG_USE_HND_KVCACHE = EnvBool(False)
+5
View File
@@ -1106,6 +1106,11 @@ class Scheduler(
tic = time.perf_counter()
self.init_memory_pools()
self.kv_cache_allocation_time = time.perf_counter() - tic
if self.enable_unified_memory:
# The first compaction move must not happen while serving with a
# full pool (see MultiEndedAllocator.prewarm_compaction); prewarm it
# before graph capture and KV sizing account for its memory.
self.tp_worker.model_runner.token_to_kv_pool_allocator.prewarm_compaction()
self.init_all_attention_backends()
self.init_all_cuda_graphs()
@@ -346,6 +346,11 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
lazy_compaction=self.lazy_compaction,
)
def prewarm_compaction(self) -> None:
"""Prewarm the page-move path of every member that compacts."""
for allocator in self._move_gate_targets():
allocator.prewarm_compaction()
def translate_write_loc(
self,
loc: torch.Tensor,
@@ -349,6 +349,11 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
lazy_compaction=self.lazy_compaction,
)
def prewarm_compaction(self) -> None:
"""Prewarm the page-move path of every member that compacts."""
for allocator in self._move_gate_targets():
allocator.prewarm_compaction()
def is_slot_allocated(self, slot: int) -> bool:
return self.full_attn_allocator.is_slot_allocated(slot)
@@ -446,6 +446,15 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self._lazy_max_moves_per_call = (
envs.SGLANG_LAZY_COMPACTION_MAX_MOVES_PER_CALL.get()
)
# A move-kernel launch copies at most this many pages; `_move_pages`
# issues a larger move as consecutive launches within the same call
# (see there). Bounded by the pool, which never has more pages.
max_pages_per_move = envs.SGLANG_COMPACTION_MAX_PAGES_PER_MOVE.get()
assert max_pages_per_move > 0, (
"SGLANG_COMPACTION_MAX_PAGES_PER_MOVE must be positive, got "
f"{max_pages_per_move}"
)
self._move_max_pages = min(max_pages_per_move, self.num_pages)
# Epoch-keyed memos for the capacity views: pure between mutations, but
# schedulers read them O(queue) times per step.
@@ -1466,25 +1475,60 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
f"[{self.sub_pool_name}] page copy while a move gate is closed"
)
v_moved = self.physical_to_virtual[src_pages].clone() # read pre-wipe
# Expand to PHYSICAL token granularity (the move kernel is
# token-granular over pool rows).
if self.pool_page_size == 1:
src_t, dst_t = src_pages, dst_pages
else:
ps = self.pool_page_size
offsets = torch.arange(ps, dtype=torch.int64, device=self.device)
src_t = (src_pages[:, None] * ps + offsets).reshape(-1)
dst_t = (dst_pages[:, None] * ps + offsets).reshape(-1)
# Un-translated copy: the public copy_from translates virtual ids,
# which we must NOT do here.
self._kvcache.move_kv_cache(dst_t, src_t)
self._move_pages(src_pages, dst_pages)
self.virtual_to_physical[v_moved] = dst_pages
self.physical_to_virtual[dst_pages] = v_moved
self._inverse_history.append((src_pages, dst_pages, v_moved))
return v_moved
def _move_pages(self, src_pages: torch.Tensor, dst_pages: torch.Tensor) -> None:
# Whole page envelopes by physical page id, at most `_move_max_pages`
# per launch, every launch issued in order on the current stream before
# this returns. Callers pass disjoint src/dst sets, so no launch reads
# a page another launch writes.
for start in range(0, src_pages.numel(), self._move_max_pages):
# Un-translated copy: the public copy_from translates virtual ids,
# which we must NOT do here.
self._kvcache.move_physical_pages(
dst_pages[start : start + self._move_max_pages],
src_pages[start : start + self._move_max_pages],
self.pool_page_size,
)
def prewarm_compaction(self) -> None:
"""Run the compaction commit's first-use work once at startup, on the
reserved sink page 0, so that no serving-time compaction is the first
to compile, load or launch it.
A first use while serving stalls the scheduler on the page-copy kernel's
compile and load and on the first `.all()` reduction and `_assert_async`
of the commit. A kernel's first launch can also take device memory
outside every pool (a spilling kernel grows the context's local-memory
reservation), and an urgent compaction under a full pool then fails
with `Triton Error [CUDA]: out of memory`. Paid here, before KV sizing,
that memory is accounted like any other startup allocation.
The page-copy kernel specializes on neither the move size (a grid
dimension) nor the alignment of the page-id slices, so this launch
selects the variant every serving-time move of this sub-pool uses.
"""
if torch.device(self.device).type != "cuda":
return
assert self.min_page_index >= 1, (
f"MultiEndedAllocator({self.sub_pool_name!r}).prewarm_compaction needs "
"the reserved sink page 0"
)
# The sink page copied onto itself, listed twice: every byte written is
# the byte read, so the coinciding src/dst are harmless, and two entries
# make `.all()` launch the reduction kernel a real commit uses (it skips
# the kernel for one element).
sink = torch.zeros(2, dtype=torch.int64, device=self.device)
self._move_pages(sink, sink)
# The commit's p2v check; its page-table gather and writes are the ones
# alloc and free run, so they are warm before any compaction can exist.
torch._assert_async((sink >= 0).all(), "prewarm_compaction")
torch.cuda.current_stream().synchronize()
# -- lazy compaction primitives --
def set_latest_forward_done_event(self, event: Optional[torch.cuda.Event]) -> None:
@@ -1902,10 +1946,10 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
latest_event: Optional[torch.cuda.Event],
released_fired: List[torch.Tensor],
) -> None:
"""Issue ONE `move_kv_cache` plus ONE bulk v2p/p2v remap for the
accumulated `(src, dst)` pairs. Fired srcs accumulate in `released_fired`
(merged by `_flush` after its dst-slice); event-pending srcs route to
`_pending_reuse` for read-race gating.
"""Issue the page copies (`_move_pages`) plus ONE bulk v2p/p2v remap for
the accumulated `(src, dst)` pairs. Fired srcs accumulate in
`released_fired` (merged by `_flush` after its dst-slice); event-pending
srcs route to `_pending_reuse` for read-race gating.
"""
if not srcs:
return
@@ -1920,17 +1964,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
(v_moveds_t >= 0).all(),
"invalid p2v mapping in MultiEndedAllocator._flush",
)
# Expand to PHYSICAL token granularity (the move kernel is
# token-granular over pool rows).
if self.pool_page_size == 1:
src_t, dst_t = src_pages_t, dst_pages_t
else:
ps = self.pool_page_size
offsets = torch.arange(ps, dtype=torch.int64, device=self.device)
src_t = (src_pages_t[:, None] * ps + offsets).reshape(-1)
dst_t = (dst_pages_t[:, None] * ps + offsets).reshape(-1)
self._kvcache.move_kv_cache(dst_t, src_t)
# ONE bulk remap (single-writer on schedule_stream).
self._move_pages(src_pages_t, dst_pages_t)
# ONE bulk remap (single-writer on schedule_stream), issued after the
# copies on the same stream: work ordered after it sees moved pages.
self.virtual_to_physical[v_moveds_t] = dst_pages_t
self.physical_to_virtual[dst_pages_t] = v_moveds_t
self.physical_to_virtual.index_fill_(0, src_pages_t, -1)
@@ -760,7 +760,7 @@ class UnifiedKVPool:
class UnifiedMHATokenToKVPool(MHATokenToKVPool):
"""MHA KV pool whose per-layer `k_buffer`/`v_buffer` are token-major views
into a `UnifiedKVPool`: `(num_pages * ps, head_num, head_dim)` with the whole
entry as slot stride, indexed by the physical token id. `move_kv_cache`
entry as slot stride, indexed by the physical token id. `move_physical_pages`
relocates whole page envelopes.
"""
@@ -823,17 +823,28 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
"""Relocate slots by whole page envelope (`tgt_loc`/`src_loc` are
physical token ids)."""
physical token ids, in whole page-major runs)."""
if tgt_loc.numel() == 0:
return
ps = self.page_size
self.move_physical_pages(
tgt_loc.view(-1, ps)[:, 0] // ps, src_loc.view(-1, ps)[:, 0] // ps, ps
)
def move_physical_pages(
self, tgt_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
):
"""Copy whole page envelopes `src_pages` -> `tgt_pages`: disjoint sets of
physical page ids, in pages of `page_size` slots (compaction moves)."""
assert page_size == self.page_size, (
f"page ids of {page_size}-slot pages passed to a pool whose envelope "
f"holds {self.page_size} slots"
)
# The envelope view below starts at byte 0, so this sub-pool must be
# anchored there; a non-zero anchor moves another sub-pool's bytes, and
# the ids stay in range so nothing downstream notices.
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
ps = self.page_size
tgt_pages = tgt_loc.view(-1, ps)[:, 0] // ps
src_pages = src_loc.view(-1, ps)[:, 0] // ps
with record_function("UnifiedMHA.move_kv_cache"):
with record_function("UnifiedMHA.move_physical_pages"):
copy_pages(
self._unified_buffer._raw,
tgt_pages,
@@ -894,7 +905,7 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
into a `UnifiedKVPool`: `(num_pages * ps, 1, kv_cache_dim)` with the whole
entry as slot stride, indexed by the physical token id (`translate_kv_loc`
of the virtual id), so the stock `MLATokenToKVPool` read/write methods work
on them unmodified. `move_kv_cache` relocates whole page envelopes.
on them unmodified. `move_physical_pages` relocates whole page envelopes.
"""
requires_physical_write_loc = True
@@ -967,17 +978,28 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
"""Relocate whole page envelopes.
`tgt_loc`/`src_loc` are physical token ids: both compaction paths
expand page ids into page-major-ordered token runs
`tgt_loc`/`src_loc` are physical token ids in whole page-major runs
(`pages[:, None] * ps + offsets`), relied on here to recover the page
lists. One contiguous envelope copy replaces the per-layer strided moves.
lists.
"""
if tgt_loc.numel() == 0:
return
ps = self.page_size
tgt_pages = tgt_loc.view(-1, ps)[:, 0] // ps
src_pages = src_loc.view(-1, ps)[:, 0] // ps
with record_function("UnifiedMLA.move_kv_cache"):
self.move_physical_pages(
tgt_loc.view(-1, ps)[:, 0] // ps, src_loc.view(-1, ps)[:, 0] // ps, ps
)
def move_physical_pages(
self, tgt_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
):
"""Copy whole page envelopes `src_pages` -> `tgt_pages`: disjoint sets of
physical page ids, in pages of `page_size` slots (compaction moves). One
contiguous envelope copy replaces the per-layer strided moves."""
assert page_size == self.page_size, (
f"page ids of {page_size}-slot pages passed to a pool whose envelope "
f"holds {self.page_size} slots"
)
with record_function("UnifiedMLA.move_physical_pages"):
copy_pages(
self._unified_buffer._raw,
tgt_pages,
@@ -1113,11 +1135,18 @@ class UnifiedMambaPool(MambaPool):
# take PHYSICAL slot ids; callers translate via the slot allocator first.
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
# Cross-pool physical-move contract, implemented by every pool the
# MultiEndedAllocator wraps. Ids are PHYSICAL slots; `MambaPool.copy_from`
# takes (src, dst), hence the swap.
# Ids are PHYSICAL slots; `MambaPool.copy_from` takes (src, dst), hence
# the swap.
MambaPool.copy_from(self, src_loc, tgt_loc)
def move_physical_pages(
self, tgt_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
):
# Cross-pool physical-move contract, implemented by every pool the
# MultiEndedAllocator wraps. A state slot is one page.
assert page_size == 1, f"mamba state pages are single slots, got {page_size}"
self.move_kv_cache(tgt_pages, src_pages)
# -- PD state transfer (StateType.MAMBA) --
# The transfer item is the whole per-slot envelope, addressed as
# `raw_ptr + physical_slot * entry_bytes`. An envelope cannot be TP-resliced
@@ -92,7 +92,12 @@ class FakeKVCache:
def __init__(self, max_slots: int):
self.buf = torch.full((max_slots,), -1, dtype=torch.int64)
def move_kv_cache(self, dst_loc: torch.Tensor, src_loc: torch.Tensor):
def move_physical_pages(
self, dst_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
):
slots = torch.arange(page_size, device=dst_pages.device)
dst_loc = (dst_pages[:, None] * page_size + slots).flatten()
src_loc = (src_pages[:, None] * page_size + slots).flatten()
self.buf[dst_loc] = self.buf[src_loc].clone()