[Perf] unified-memory: Prewarm the compaction move at startup and copy whole page envelopes by page id (#44592)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
ZYHowell
parent
aaf5f397a5
commit
36cf028346
@@ -7,7 +7,10 @@ import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
# Callers pass page-id slices at any element offset (compaction launches a long
|
||||
# move in chunks); each program loads only one scalar id, so specializing on
|
||||
# their alignment would only compile more variants, and those mid-serving.
|
||||
@triton.jit(do_not_specialize_on_alignment=["src_pages", "dst_pages"])
|
||||
def _copy_pages_kernel(
|
||||
buf,
|
||||
src_pages,
|
||||
|
||||
@@ -572,6 +572,10 @@ class Envs:
|
||||
# Per-call move cap on a non-urgent lazy-compaction flush, so a large
|
||||
# backlog cannot stall the scheduler loop; urgent flushes are uncapped.
|
||||
SGLANG_LAZY_COMPACTION_MAX_MOVES_PER_CALL = EnvInt(4096)
|
||||
# Max pages one compaction move-kernel launch copies (eager or lazy, urgent
|
||||
# or not); a larger move is issued as consecutive launches of this size
|
||||
# within the same compaction call.
|
||||
SGLANG_COMPACTION_MAX_PAGES_PER_MOVE = EnvInt(4096)
|
||||
# HND KV layout folds (page, head) into one paged index for per-kv-head sparse
|
||||
# page tables (DP attn); paged backends like trtllm_mha consume it directly.
|
||||
SGLANG_USE_HND_KVCACHE = EnvBool(False)
|
||||
|
||||
@@ -1106,6 +1106,11 @@ class Scheduler(
|
||||
tic = time.perf_counter()
|
||||
self.init_memory_pools()
|
||||
self.kv_cache_allocation_time = time.perf_counter() - tic
|
||||
if self.enable_unified_memory:
|
||||
# The first compaction move must not happen while serving with a
|
||||
# full pool (see MultiEndedAllocator.prewarm_compaction); prewarm it
|
||||
# before graph capture and KV sizing account for its memory.
|
||||
self.tp_worker.model_runner.token_to_kv_pool_allocator.prewarm_compaction()
|
||||
|
||||
self.init_all_attention_backends()
|
||||
self.init_all_cuda_graphs()
|
||||
|
||||
@@ -346,6 +346,11 @@ class UnifiedSWAAllocatorBase(SWATokenToKVPoolAllocator):
|
||||
lazy_compaction=self.lazy_compaction,
|
||||
)
|
||||
|
||||
def prewarm_compaction(self) -> None:
|
||||
"""Prewarm the page-move path of every member that compacts."""
|
||||
for allocator in self._move_gate_targets():
|
||||
allocator.prewarm_compaction()
|
||||
|
||||
def translate_write_loc(
|
||||
self,
|
||||
loc: torch.Tensor,
|
||||
|
||||
@@ -349,6 +349,11 @@ class UnifiedMambaTokenToKVPoolAllocator(BaseTokenToKVPoolAllocator):
|
||||
lazy_compaction=self.lazy_compaction,
|
||||
)
|
||||
|
||||
def prewarm_compaction(self) -> None:
|
||||
"""Prewarm the page-move path of every member that compacts."""
|
||||
for allocator in self._move_gate_targets():
|
||||
allocator.prewarm_compaction()
|
||||
|
||||
def is_slot_allocated(self, slot: int) -> bool:
|
||||
return self.full_attn_allocator.is_slot_allocated(slot)
|
||||
|
||||
|
||||
@@ -446,6 +446,15 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self._lazy_max_moves_per_call = (
|
||||
envs.SGLANG_LAZY_COMPACTION_MAX_MOVES_PER_CALL.get()
|
||||
)
|
||||
# A move-kernel launch copies at most this many pages; `_move_pages`
|
||||
# issues a larger move as consecutive launches within the same call
|
||||
# (see there). Bounded by the pool, which never has more pages.
|
||||
max_pages_per_move = envs.SGLANG_COMPACTION_MAX_PAGES_PER_MOVE.get()
|
||||
assert max_pages_per_move > 0, (
|
||||
"SGLANG_COMPACTION_MAX_PAGES_PER_MOVE must be positive, got "
|
||||
f"{max_pages_per_move}"
|
||||
)
|
||||
self._move_max_pages = min(max_pages_per_move, self.num_pages)
|
||||
|
||||
# Epoch-keyed memos for the capacity views: pure between mutations, but
|
||||
# schedulers read them O(queue) times per step.
|
||||
@@ -1466,25 +1475,60 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
f"[{self.sub_pool_name}] page copy while a move gate is closed"
|
||||
)
|
||||
v_moved = self.physical_to_virtual[src_pages].clone() # read pre-wipe
|
||||
|
||||
# Expand to PHYSICAL token granularity (the move kernel is
|
||||
# token-granular over pool rows).
|
||||
if self.pool_page_size == 1:
|
||||
src_t, dst_t = src_pages, dst_pages
|
||||
else:
|
||||
ps = self.pool_page_size
|
||||
offsets = torch.arange(ps, dtype=torch.int64, device=self.device)
|
||||
src_t = (src_pages[:, None] * ps + offsets).reshape(-1)
|
||||
dst_t = (dst_pages[:, None] * ps + offsets).reshape(-1)
|
||||
|
||||
# Un-translated copy: the public copy_from translates virtual ids,
|
||||
# which we must NOT do here.
|
||||
self._kvcache.move_kv_cache(dst_t, src_t)
|
||||
self._move_pages(src_pages, dst_pages)
|
||||
self.virtual_to_physical[v_moved] = dst_pages
|
||||
self.physical_to_virtual[dst_pages] = v_moved
|
||||
self._inverse_history.append((src_pages, dst_pages, v_moved))
|
||||
return v_moved
|
||||
|
||||
def _move_pages(self, src_pages: torch.Tensor, dst_pages: torch.Tensor) -> None:
|
||||
# Whole page envelopes by physical page id, at most `_move_max_pages`
|
||||
# per launch, every launch issued in order on the current stream before
|
||||
# this returns. Callers pass disjoint src/dst sets, so no launch reads
|
||||
# a page another launch writes.
|
||||
for start in range(0, src_pages.numel(), self._move_max_pages):
|
||||
# Un-translated copy: the public copy_from translates virtual ids,
|
||||
# which we must NOT do here.
|
||||
self._kvcache.move_physical_pages(
|
||||
dst_pages[start : start + self._move_max_pages],
|
||||
src_pages[start : start + self._move_max_pages],
|
||||
self.pool_page_size,
|
||||
)
|
||||
|
||||
def prewarm_compaction(self) -> None:
|
||||
"""Run the compaction commit's first-use work once at startup, on the
|
||||
reserved sink page 0, so that no serving-time compaction is the first
|
||||
to compile, load or launch it.
|
||||
|
||||
A first use while serving stalls the scheduler on the page-copy kernel's
|
||||
compile and load and on the first `.all()` reduction and `_assert_async`
|
||||
of the commit. A kernel's first launch can also take device memory
|
||||
outside every pool (a spilling kernel grows the context's local-memory
|
||||
reservation), and an urgent compaction under a full pool then fails
|
||||
with `Triton Error [CUDA]: out of memory`. Paid here, before KV sizing,
|
||||
that memory is accounted like any other startup allocation.
|
||||
|
||||
The page-copy kernel specializes on neither the move size (a grid
|
||||
dimension) nor the alignment of the page-id slices, so this launch
|
||||
selects the variant every serving-time move of this sub-pool uses.
|
||||
"""
|
||||
if torch.device(self.device).type != "cuda":
|
||||
return
|
||||
assert self.min_page_index >= 1, (
|
||||
f"MultiEndedAllocator({self.sub_pool_name!r}).prewarm_compaction needs "
|
||||
"the reserved sink page 0"
|
||||
)
|
||||
# The sink page copied onto itself, listed twice: every byte written is
|
||||
# the byte read, so the coinciding src/dst are harmless, and two entries
|
||||
# make `.all()` launch the reduction kernel a real commit uses (it skips
|
||||
# the kernel for one element).
|
||||
sink = torch.zeros(2, dtype=torch.int64, device=self.device)
|
||||
self._move_pages(sink, sink)
|
||||
# The commit's p2v check; its page-table gather and writes are the ones
|
||||
# alloc and free run, so they are warm before any compaction can exist.
|
||||
torch._assert_async((sink >= 0).all(), "prewarm_compaction")
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
# -- lazy compaction primitives --
|
||||
|
||||
def set_latest_forward_done_event(self, event: Optional[torch.cuda.Event]) -> None:
|
||||
@@ -1902,10 +1946,10 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
latest_event: Optional[torch.cuda.Event],
|
||||
released_fired: List[torch.Tensor],
|
||||
) -> None:
|
||||
"""Issue ONE `move_kv_cache` plus ONE bulk v2p/p2v remap for the
|
||||
accumulated `(src, dst)` pairs. Fired srcs accumulate in `released_fired`
|
||||
(merged by `_flush` after its dst-slice); event-pending srcs route to
|
||||
`_pending_reuse` for read-race gating.
|
||||
"""Issue the page copies (`_move_pages`) plus ONE bulk v2p/p2v remap for
|
||||
the accumulated `(src, dst)` pairs. Fired srcs accumulate in
|
||||
`released_fired` (merged by `_flush` after its dst-slice); event-pending
|
||||
srcs route to `_pending_reuse` for read-race gating.
|
||||
"""
|
||||
if not srcs:
|
||||
return
|
||||
@@ -1920,17 +1964,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
(v_moveds_t >= 0).all(),
|
||||
"invalid p2v mapping in MultiEndedAllocator._flush",
|
||||
)
|
||||
# Expand to PHYSICAL token granularity (the move kernel is
|
||||
# token-granular over pool rows).
|
||||
if self.pool_page_size == 1:
|
||||
src_t, dst_t = src_pages_t, dst_pages_t
|
||||
else:
|
||||
ps = self.pool_page_size
|
||||
offsets = torch.arange(ps, dtype=torch.int64, device=self.device)
|
||||
src_t = (src_pages_t[:, None] * ps + offsets).reshape(-1)
|
||||
dst_t = (dst_pages_t[:, None] * ps + offsets).reshape(-1)
|
||||
self._kvcache.move_kv_cache(dst_t, src_t)
|
||||
# ONE bulk remap (single-writer on schedule_stream).
|
||||
self._move_pages(src_pages_t, dst_pages_t)
|
||||
# ONE bulk remap (single-writer on schedule_stream), issued after the
|
||||
# copies on the same stream: work ordered after it sees moved pages.
|
||||
self.virtual_to_physical[v_moveds_t] = dst_pages_t
|
||||
self.physical_to_virtual[dst_pages_t] = v_moveds_t
|
||||
self.physical_to_virtual.index_fill_(0, src_pages_t, -1)
|
||||
|
||||
@@ -760,7 +760,7 @@ class UnifiedKVPool:
|
||||
class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
"""MHA KV pool whose per-layer `k_buffer`/`v_buffer` are token-major views
|
||||
into a `UnifiedKVPool`: `(num_pages * ps, head_num, head_dim)` with the whole
|
||||
entry as slot stride, indexed by the physical token id. `move_kv_cache`
|
||||
entry as slot stride, indexed by the physical token id. `move_physical_pages`
|
||||
relocates whole page envelopes.
|
||||
"""
|
||||
|
||||
@@ -823,17 +823,28 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
|
||||
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
||||
"""Relocate slots by whole page envelope (`tgt_loc`/`src_loc` are
|
||||
physical token ids)."""
|
||||
physical token ids, in whole page-major runs)."""
|
||||
if tgt_loc.numel() == 0:
|
||||
return
|
||||
ps = self.page_size
|
||||
self.move_physical_pages(
|
||||
tgt_loc.view(-1, ps)[:, 0] // ps, src_loc.view(-1, ps)[:, 0] // ps, ps
|
||||
)
|
||||
|
||||
def move_physical_pages(
|
||||
self, tgt_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
|
||||
):
|
||||
"""Copy whole page envelopes `src_pages` -> `tgt_pages`: disjoint sets of
|
||||
physical page ids, in pages of `page_size` slots (compaction moves)."""
|
||||
assert page_size == self.page_size, (
|
||||
f"page ids of {page_size}-slot pages passed to a pool whose envelope "
|
||||
f"holds {self.page_size} slots"
|
||||
)
|
||||
# The envelope view below starts at byte 0, so this sub-pool must be
|
||||
# anchored there; a non-zero anchor moves another sub-pool's bytes, and
|
||||
# the ids stay in range so nothing downstream notices.
|
||||
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
||||
ps = self.page_size
|
||||
tgt_pages = tgt_loc.view(-1, ps)[:, 0] // ps
|
||||
src_pages = src_loc.view(-1, ps)[:, 0] // ps
|
||||
with record_function("UnifiedMHA.move_kv_cache"):
|
||||
with record_function("UnifiedMHA.move_physical_pages"):
|
||||
copy_pages(
|
||||
self._unified_buffer._raw,
|
||||
tgt_pages,
|
||||
@@ -894,7 +905,7 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
|
||||
into a `UnifiedKVPool`: `(num_pages * ps, 1, kv_cache_dim)` with the whole
|
||||
entry as slot stride, indexed by the physical token id (`translate_kv_loc`
|
||||
of the virtual id), so the stock `MLATokenToKVPool` read/write methods work
|
||||
on them unmodified. `move_kv_cache` relocates whole page envelopes.
|
||||
on them unmodified. `move_physical_pages` relocates whole page envelopes.
|
||||
"""
|
||||
|
||||
requires_physical_write_loc = True
|
||||
@@ -967,17 +978,28 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
|
||||
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
||||
"""Relocate whole page envelopes.
|
||||
|
||||
`tgt_loc`/`src_loc` are physical token ids: both compaction paths
|
||||
expand page ids into page-major-ordered token runs
|
||||
`tgt_loc`/`src_loc` are physical token ids in whole page-major runs
|
||||
(`pages[:, None] * ps + offsets`), relied on here to recover the page
|
||||
lists. One contiguous envelope copy replaces the per-layer strided moves.
|
||||
lists.
|
||||
"""
|
||||
if tgt_loc.numel() == 0:
|
||||
return
|
||||
ps = self.page_size
|
||||
tgt_pages = tgt_loc.view(-1, ps)[:, 0] // ps
|
||||
src_pages = src_loc.view(-1, ps)[:, 0] // ps
|
||||
with record_function("UnifiedMLA.move_kv_cache"):
|
||||
self.move_physical_pages(
|
||||
tgt_loc.view(-1, ps)[:, 0] // ps, src_loc.view(-1, ps)[:, 0] // ps, ps
|
||||
)
|
||||
|
||||
def move_physical_pages(
|
||||
self, tgt_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
|
||||
):
|
||||
"""Copy whole page envelopes `src_pages` -> `tgt_pages`: disjoint sets of
|
||||
physical page ids, in pages of `page_size` slots (compaction moves). One
|
||||
contiguous envelope copy replaces the per-layer strided moves."""
|
||||
assert page_size == self.page_size, (
|
||||
f"page ids of {page_size}-slot pages passed to a pool whose envelope "
|
||||
f"holds {self.page_size} slots"
|
||||
)
|
||||
with record_function("UnifiedMLA.move_physical_pages"):
|
||||
copy_pages(
|
||||
self._unified_buffer._raw,
|
||||
tgt_pages,
|
||||
@@ -1113,11 +1135,18 @@ class UnifiedMambaPool(MambaPool):
|
||||
# take PHYSICAL slot ids; callers translate via the slot allocator first.
|
||||
|
||||
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
||||
# Cross-pool physical-move contract, implemented by every pool the
|
||||
# MultiEndedAllocator wraps. Ids are PHYSICAL slots; `MambaPool.copy_from`
|
||||
# takes (src, dst), hence the swap.
|
||||
# Ids are PHYSICAL slots; `MambaPool.copy_from` takes (src, dst), hence
|
||||
# the swap.
|
||||
MambaPool.copy_from(self, src_loc, tgt_loc)
|
||||
|
||||
def move_physical_pages(
|
||||
self, tgt_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
|
||||
):
|
||||
# Cross-pool physical-move contract, implemented by every pool the
|
||||
# MultiEndedAllocator wraps. A state slot is one page.
|
||||
assert page_size == 1, f"mamba state pages are single slots, got {page_size}"
|
||||
self.move_kv_cache(tgt_pages, src_pages)
|
||||
|
||||
# -- PD state transfer (StateType.MAMBA) --
|
||||
# The transfer item is the whole per-slot envelope, addressed as
|
||||
# `raw_ptr + physical_slot * entry_bytes`. An envelope cannot be TP-resliced
|
||||
|
||||
@@ -92,7 +92,12 @@ class FakeKVCache:
|
||||
def __init__(self, max_slots: int):
|
||||
self.buf = torch.full((max_slots,), -1, dtype=torch.int64)
|
||||
|
||||
def move_kv_cache(self, dst_loc: torch.Tensor, src_loc: torch.Tensor):
|
||||
def move_physical_pages(
|
||||
self, dst_pages: torch.Tensor, src_pages: torch.Tensor, page_size: int
|
||||
):
|
||||
slots = torch.arange(page_size, device=dst_pages.device)
|
||||
dst_loc = (dst_pages[:, None] * page_size + slots).flatten()
|
||||
src_loc = (src_pages[:, None] * page_size + slots).flatten()
|
||||
self.buf[dst_loc] = self.buf[src_loc].clone()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user