From 46ad9eb43919e86d21bd24a5b6f76712c3219fb7 Mon Sep 17 00:00:00 2001 From: Ke Bao Date: Fri, 9 Oct 2026 00:21:27 +0800 Subject: [PATCH] Fix K-only host buffer cleanup (#43101) --- python/sglang/srt/mem_cache/pool_host/mha.py | 11 ++++ .../hicache/test_k_only_host_release.py | 50 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 test/registered/hicache/test_k_only_host_release.py diff --git a/python/sglang/srt/mem_cache/pool_host/mha.py b/python/sglang/srt/mem_cache/pool_host/mha.py index aa5f69bbbe..f612e57115 100644 --- a/python/sglang/srt/mem_cache/pool_host/mha.py +++ b/python/sglang/srt/mem_cache/pool_host/mha.py @@ -40,6 +40,7 @@ from sglang.srt.mem_cache.pool_host.base import ( ) from sglang.srt.mem_cache.pool_host.common import ( ALLOC_MEMORY_FUNCS, + _cuda_host_unregister, get_allocator_from_storage, make_kernel_ptr_table, ) @@ -822,6 +823,7 @@ class MHATokenToKOnlyPoolHost(HostKVCache): device: str = "cpu", allocator_type: str = "default", ): + self._destroyed = False self.device_pool = device_pool self.page_size = anchor_host.page_size self.layout = layout @@ -884,6 +886,15 @@ class MHATokenToKOnlyPoolHost(HostKVCache): host_memory_registered=self.pin_memory, ) + def destroy(self): + if self._destroyed: + return + if self.pin_memory and (_is_cuda or _is_hip): + _cuda_host_unregister(self.k_buffer) + self.k_buffer = None + self.k_data_refs = [] + super().destroy() + def get_size_per_token(self): return self.head_dim * self.head_num * self.layer_num * self.dtype.itemsize diff --git a/test/registered/hicache/test_k_only_host_release.py b/test/registered/hicache/test_k_only_host_release.py new file mode 100644 index 0000000000..33242e2d0f --- /dev/null +++ b/test/registered/hicache/test_k_only_host_release.py @@ -0,0 +1,50 @@ +import unittest + +import torch + +from sglang.srt.mem_cache.memory_pool import MHATokenToKOnlyPool, MHATokenToKVPool +from sglang.srt.mem_cache.pool_host.common import ( + _cuda_host_register, + _cuda_host_unregister, +) +from sglang.srt.mem_cache.pool_host.mha import ( + MHATokenToKOnlyPoolHost, + MHATokenToKVPoolHost, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-small") + + +class TestKOnlyHostRelease(CustomTestCase): + def test_destroy_releases_registered_key_pages(self): + common = dict( + size=256, + page_size=64, + dtype=torch.bfloat16, + head_num=1, + head_dim=128, + layer_num=2, + device="cuda", + enable_memory_saver=False, + ) + main_pool = MHATokenToKVPool(**common) + index_pool = MHATokenToKOnlyPool(**common) + anchor = MHATokenToKVPoolHost( + main_pool, 2, 0, 64, "layer_first", pin_memory=False + ) + self.addCleanup(anchor.destroy) + host = MHATokenToKOnlyPoolHost(index_pool, anchor, "layer_first") + buffer = host.k_buffer + self.addCleanup(_cuda_host_unregister, buffer) + host.destroy() + host.destroy() + _cuda_host_register(buffer) + self.assertIsNone(host.k_buffer) + self.assertEqual(host.k_data_refs, []) + torch.cuda.synchronize() + + +if __name__ == "__main__": + unittest.main()