Fix K-only host buffer cleanup (#43101)

This commit is contained in:
Ke Bao
2026-10-09 00:21:27 +08:00
committed by GitHub
parent 4ab4d90d78
commit 46ad9eb439
2 changed files with 61 additions and 0 deletions
@@ -40,6 +40,7 @@ from sglang.srt.mem_cache.pool_host.base import (
)
from sglang.srt.mem_cache.pool_host.common import (
ALLOC_MEMORY_FUNCS,
_cuda_host_unregister,
get_allocator_from_storage,
make_kernel_ptr_table,
)
@@ -822,6 +823,7 @@ class MHATokenToKOnlyPoolHost(HostKVCache):
device: str = "cpu",
allocator_type: str = "default",
):
self._destroyed = False
self.device_pool = device_pool
self.page_size = anchor_host.page_size
self.layout = layout
@@ -884,6 +886,15 @@ class MHATokenToKOnlyPoolHost(HostKVCache):
host_memory_registered=self.pin_memory,
)
def destroy(self):
if self._destroyed:
return
if self.pin_memory and (_is_cuda or _is_hip):
_cuda_host_unregister(self.k_buffer)
self.k_buffer = None
self.k_data_refs = []
super().destroy()
def get_size_per_token(self):
return self.head_dim * self.head_num * self.layer_num * self.dtype.itemsize