From fa7784d140eef70223cabf942ab81d53a48fdb0a Mon Sep 17 00:00:00 2001 From: metamergebot Date: Fri, 2 Oct 2026 15:12:14 -0700 Subject: [PATCH] [unified-memory] Copy page envelopes in place during compaction (#42252) Co-authored-by: merrymercy Co-authored-by: ZYHowell --- .../sglang/kernels/ops/kvcache/copy_pages.py | 66 +++++++++++++++++++ .../srt/mem_cache/unified_memory_pool.py | 19 ++++-- 2 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 python/sglang/kernels/ops/kvcache/copy_pages.py diff --git a/python/sglang/kernels/ops/kvcache/copy_pages.py b/python/sglang/kernels/ops/kvcache/copy_pages.py new file mode 100644 index 0000000000..dcdc036231 --- /dev/null +++ b/python/sglang/kernels/ops/kvcache/copy_pages.py @@ -0,0 +1,66 @@ +"""Copy disjoint physical page envelopes without a gathered KV temporary.""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _copy_pages_kernel( + buf, + src_pages, + dst_pages, + page_words, + SRC_STRIDE: tl.constexpr, + DST_STRIDE: tl.constexpr, + BLOCK: tl.constexpr, +): + pair = tl.program_id(0) + src = tl.load(src_pages + pair * SRC_STRIDE).to(tl.int64) + dst = tl.load(dst_pages + pair * DST_STRIDE).to(tl.int64) + offsets = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK) + mask = offsets < page_words + values = tl.load(buf + src * page_words + offsets, mask=mask) + tl.store(buf + dst * page_words + offsets, values, mask=mask) + + +_BLOCK = 2048 + + +def copy_pages( + raw: torch.Tensor, + dst_pages: torch.Tensor, + src_pages: torch.Tensor, + num_pages: int, + page_bytes: int, +) -> None: + """Copy src -> dst on the current stream; dst and src sets must be disjoint. + + The allocator supplies unique destinations in free space. Overlapping float + shifts are already split into ordered singleton calls by the allocator. + Reinterpreting the buffer is a view; the GPU copy has no KV-sized scratch. + """ + count = dst_pages.numel() + assert count == src_pages.numel() + if count == 0: + return + assert raw.dtype == torch.uint8 and raw.is_contiguous() + if not raw.is_cuda: + env = raw[: num_pages * page_bytes].view(num_pages, page_bytes) + env[dst_pages] = env[src_pages] + return + # Actual MHA/MLA pages are 8-byte aligned. Keep unaligned layouts valid. + wide = page_bytes % 8 == 0 and raw.storage_offset() % 8 == 0 + words = raw[: num_pages * page_bytes].view(torch.int64 if wide else torch.uint8) + page_words = page_bytes // (8 if wide else 1) + _copy_pages_kernel[(count, triton.cdiv(page_words, _BLOCK))]( + words, + src_pages, + dst_pages, + page_words, + SRC_STRIDE=src_pages.stride(0), + DST_STRIDE=dst_pages.stride(0), + BLOCK=_BLOCK, + ) diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index 8f2d66ff58..24274fb7ef 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -33,6 +33,7 @@ from typing import ClassVar, Dict, List, NamedTuple, Optional, Tuple import torch from torch.profiler import record_function +from sglang.kernels.ops.kvcache.copy_pages import copy_pages from sglang.kernels.ops.kvcache.zero_pages import zero_pages from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.environ import envs @@ -642,10 +643,13 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool): tgt_pages = tgt_loc.view(-1, ps)[:, 0] // ps src_pages = src_loc.view(-1, ps)[:, 0] // ps with record_function("UnifiedMHA.move_kv_cache"): - env = self._unified_buffer._raw[: self._num_pages * self._page_bytes].view( - self._num_pages, self._page_bytes + copy_pages( + self._unified_buffer._raw, + tgt_pages, + src_pages, + self._num_pages, + self._page_bytes, ) - env[tgt_pages] = env[src_pages] def get_contiguous_buf_infos(self): """Register the raw buffer as physical page envelopes for PD transfer. @@ -767,10 +771,13 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool): tgt_pages = tgt_loc.view(-1, ps)[:, 0] // ps src_pages = src_loc.view(-1, ps)[:, 0] // ps with record_function("UnifiedMLA.move_kv_cache"): - env = self._unified_buffer._raw[: self._num_pages * self._page_bytes].view( - self._num_pages, self._page_bytes + copy_pages( + self._unified_buffer._raw, + tgt_pages, + src_pages, + self._num_pages, + self._page_bytes, ) - env[tgt_pages] = env[src_pages] def zero_physical_pages(self, phys_pages: torch.Tensor) -> None: """Zero whole page envelopes (PHYSICAL page ids) on allocator