[unified-memory] Copy page envelopes in place during compaction (#42252)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com> Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
co-authored by
merrymercy
ZYHowell
parent
e8fab85a02
commit
fa7784d140
@@ -0,0 +1,66 @@
|
||||
"""Copy disjoint physical page envelopes without a gathered KV temporary."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _copy_pages_kernel(
|
||||
buf,
|
||||
src_pages,
|
||||
dst_pages,
|
||||
page_words,
|
||||
SRC_STRIDE: tl.constexpr,
|
||||
DST_STRIDE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
pair = tl.program_id(0)
|
||||
src = tl.load(src_pages + pair * SRC_STRIDE).to(tl.int64)
|
||||
dst = tl.load(dst_pages + pair * DST_STRIDE).to(tl.int64)
|
||||
offsets = tl.program_id(1) * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offsets < page_words
|
||||
values = tl.load(buf + src * page_words + offsets, mask=mask)
|
||||
tl.store(buf + dst * page_words + offsets, values, mask=mask)
|
||||
|
||||
|
||||
_BLOCK = 2048
|
||||
|
||||
|
||||
def copy_pages(
|
||||
raw: torch.Tensor,
|
||||
dst_pages: torch.Tensor,
|
||||
src_pages: torch.Tensor,
|
||||
num_pages: int,
|
||||
page_bytes: int,
|
||||
) -> None:
|
||||
"""Copy src -> dst on the current stream; dst and src sets must be disjoint.
|
||||
|
||||
The allocator supplies unique destinations in free space. Overlapping float
|
||||
shifts are already split into ordered singleton calls by the allocator.
|
||||
Reinterpreting the buffer is a view; the GPU copy has no KV-sized scratch.
|
||||
"""
|
||||
count = dst_pages.numel()
|
||||
assert count == src_pages.numel()
|
||||
if count == 0:
|
||||
return
|
||||
assert raw.dtype == torch.uint8 and raw.is_contiguous()
|
||||
if not raw.is_cuda:
|
||||
env = raw[: num_pages * page_bytes].view(num_pages, page_bytes)
|
||||
env[dst_pages] = env[src_pages]
|
||||
return
|
||||
# Actual MHA/MLA pages are 8-byte aligned. Keep unaligned layouts valid.
|
||||
wide = page_bytes % 8 == 0 and raw.storage_offset() % 8 == 0
|
||||
words = raw[: num_pages * page_bytes].view(torch.int64 if wide else torch.uint8)
|
||||
page_words = page_bytes // (8 if wide else 1)
|
||||
_copy_pages_kernel[(count, triton.cdiv(page_words, _BLOCK))](
|
||||
words,
|
||||
src_pages,
|
||||
dst_pages,
|
||||
page_words,
|
||||
SRC_STRIDE=src_pages.stride(0),
|
||||
DST_STRIDE=dst_pages.stride(0),
|
||||
BLOCK=_BLOCK,
|
||||
)
|
||||
@@ -33,6 +33,7 @@ from typing import ClassVar, Dict, List, NamedTuple, Optional, Tuple
|
||||
import torch
|
||||
from torch.profiler import record_function
|
||||
|
||||
from sglang.kernels.ops.kvcache.copy_pages import copy_pages
|
||||
from sglang.kernels.ops.kvcache.zero_pages import zero_pages
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.environ import envs
|
||||
@@ -642,10 +643,13 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
tgt_pages = tgt_loc.view(-1, ps)[:, 0] // ps
|
||||
src_pages = src_loc.view(-1, ps)[:, 0] // ps
|
||||
with record_function("UnifiedMHA.move_kv_cache"):
|
||||
env = self._unified_buffer._raw[: self._num_pages * self._page_bytes].view(
|
||||
self._num_pages, self._page_bytes
|
||||
copy_pages(
|
||||
self._unified_buffer._raw,
|
||||
tgt_pages,
|
||||
src_pages,
|
||||
self._num_pages,
|
||||
self._page_bytes,
|
||||
)
|
||||
env[tgt_pages] = env[src_pages]
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
"""Register the raw buffer as physical page envelopes for PD transfer.
|
||||
@@ -767,10 +771,13 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
|
||||
tgt_pages = tgt_loc.view(-1, ps)[:, 0] // ps
|
||||
src_pages = src_loc.view(-1, ps)[:, 0] // ps
|
||||
with record_function("UnifiedMLA.move_kv_cache"):
|
||||
env = self._unified_buffer._raw[: self._num_pages * self._page_bytes].view(
|
||||
self._num_pages, self._page_bytes
|
||||
copy_pages(
|
||||
self._unified_buffer._raw,
|
||||
tgt_pages,
|
||||
src_pages,
|
||||
self._num_pages,
|
||||
self._page_bytes,
|
||||
)
|
||||
env[tgt_pages] = env[src_pages]
|
||||
|
||||
def zero_physical_pages(self, phys_pages: torch.Tensor) -> None:
|
||||
"""Zero whole page envelopes (PHYSICAL page ids) on allocator
|
||||
|
||||
Reference in New Issue
Block a user