[unified-memory] Copy page envelopes in place during compaction (#42252)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-02 15:12:14 -07:00
committed by GitHub
co-authored by merrymercy ZYHowell
parent e8fab85a02
commit fa7784d140
2 changed files with 79 additions and 6 deletions
@@ -0,0 +1,66 @@
"""Copy disjoint physical page envelopes without a gathered KV temporary."""
from __future__ import annotations
import torch
import triton
import triton.language as tl
@triton.jit
def _copy_pages_kernel(
buf,
src_pages,
dst_pages,
page_words,
SRC_STRIDE: tl.constexpr,
DST_STRIDE: tl.constexpr,
BLOCK: tl.constexpr,
):
pair = tl.program_id(0)
src = tl.load(src_pages + pair * SRC_STRIDE).to(tl.int64)
dst = tl.load(dst_pages + pair * DST_STRIDE).to(tl.int64)
offsets = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < page_words
values = tl.load(buf + src * page_words + offsets, mask=mask)
tl.store(buf + dst * page_words + offsets, values, mask=mask)
_BLOCK = 2048
def copy_pages(
raw: torch.Tensor,
dst_pages: torch.Tensor,
src_pages: torch.Tensor,
num_pages: int,
page_bytes: int,
) -> None:
"""Copy src -> dst on the current stream; dst and src sets must be disjoint.
The allocator supplies unique destinations in free space. Overlapping float
shifts are already split into ordered singleton calls by the allocator.
Reinterpreting the buffer is a view; the GPU copy has no KV-sized scratch.
"""
count = dst_pages.numel()
assert count == src_pages.numel()
if count == 0:
return
assert raw.dtype == torch.uint8 and raw.is_contiguous()
if not raw.is_cuda:
env = raw[: num_pages * page_bytes].view(num_pages, page_bytes)
env[dst_pages] = env[src_pages]
return
# Actual MHA/MLA pages are 8-byte aligned. Keep unaligned layouts valid.
wide = page_bytes % 8 == 0 and raw.storage_offset() % 8 == 0
words = raw[: num_pages * page_bytes].view(torch.int64 if wide else torch.uint8)
page_words = page_bytes // (8 if wide else 1)
_copy_pages_kernel[(count, triton.cdiv(page_words, _BLOCK))](
words,
src_pages,
dst_pages,
page_words,
SRC_STRIDE=src_pages.stride(0),
DST_STRIDE=dst_pages.stride(0),
BLOCK=_BLOCK,
)
@@ -33,6 +33,7 @@ from typing import ClassVar, Dict, List, NamedTuple, Optional, Tuple
import torch
from torch.profiler import record_function
from sglang.kernels.ops.kvcache.copy_pages import copy_pages
from sglang.kernels.ops.kvcache.zero_pages import zero_pages
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.environ import envs
@@ -642,10 +643,13 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
tgt_pages = tgt_loc.view(-1, ps)[:, 0] // ps
src_pages = src_loc.view(-1, ps)[:, 0] // ps
with record_function("UnifiedMHA.move_kv_cache"):
env = self._unified_buffer._raw[: self._num_pages * self._page_bytes].view(
self._num_pages, self._page_bytes
copy_pages(
self._unified_buffer._raw,
tgt_pages,
src_pages,
self._num_pages,
self._page_bytes,
)
env[tgt_pages] = env[src_pages]
def get_contiguous_buf_infos(self):
"""Register the raw buffer as physical page envelopes for PD transfer.
@@ -767,10 +771,13 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
tgt_pages = tgt_loc.view(-1, ps)[:, 0] // ps
src_pages = src_loc.view(-1, ps)[:, 0] // ps
with record_function("UnifiedMLA.move_kv_cache"):
env = self._unified_buffer._raw[: self._num_pages * self._page_bytes].view(
self._num_pages, self._page_bytes
copy_pages(
self._unified_buffer._raw,
tgt_pages,
src_pages,
self._num_pages,
self._page_bytes,
)
env[tgt_pages] = env[src_pages]
def zero_physical_pages(self, phys_pages: torch.Tensor) -> None:
"""Zero whole page envelopes (PHYSICAL page ids) on allocator