From 642e81fbb8e91d30a970491e5193e07c166f2617 Mon Sep 17 00:00:00 2001 From: Ke Bao Date: Fri, 9 Oct 2026 00:21:38 +0800 Subject: [PATCH] Fix DSA index host buffer cleanup (#43100) --- python/sglang/srt/mem_cache/pool_host/dsa.py | 11 ++++ .../hicache/test_dsa_host_release.py | 58 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 test/registered/hicache/test_dsa_host_release.py diff --git a/python/sglang/srt/mem_cache/pool_host/dsa.py b/python/sglang/srt/mem_cache/pool_host/dsa.py index 556510ffb9..b27d7d1b7b 100644 --- a/python/sglang/srt/mem_cache/pool_host/dsa.py +++ b/python/sglang/srt/mem_cache/pool_host/dsa.py @@ -24,6 +24,7 @@ from sglang.srt.mem_cache.pool_host.base import ( ) from sglang.srt.mem_cache.pool_host.common import ( ALLOC_MEMORY_FUNCS, + _cuda_host_unregister, get_allocator_from_storage, make_kernel_ptr_table, ) @@ -142,6 +143,7 @@ class DSAIndexerPoolHost(HostKVCache): is_dummy: bool = False, ): self._is_dummy = is_dummy + self._destroyed = False self.decl = decl storage_info = decl.storage_info device_pool = decl.device_pool @@ -227,6 +229,15 @@ class DSAIndexerPoolHost(HostKVCache): self.lock = threading.RLock() self.clear() + def destroy(self): + if self._destroyed: + return + if not self._is_dummy and self.pin_memory and (_is_cuda or _is_hip): + _cuda_host_unregister(self.index_k_with_scale_buffer) + self.index_k_with_scale_buffer = None + self.index_k_data_refs = [] + super().destroy() + def get_size_per_token(self): return self.decl.storage_info.bytes_per_token_per_layer * self.layer_num diff --git a/test/registered/hicache/test_dsa_host_release.py b/test/registered/hicache/test_dsa_host_release.py new file mode 100644 index 0000000000..7bf4b8a4e2 --- /dev/null +++ b/test/registered/hicache/test_dsa_host_release.py @@ -0,0 +1,58 @@ +import unittest + +import torch + +from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool +from sglang.srt.mem_cache.pool_host.common import ( + _cuda_host_register, + _cuda_host_unregister, +) +from sglang.srt.mem_cache.pool_host.dsa import ( + DSAIndexerPoolHost, + make_dsa_indexer_pool_decl, +) +from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=8, stage="base-b", runner_config="1-gpu-small") + + +class TestDSAHostRelease(CustomTestCase): + def test_destroy_releases_index_pages_for_registration_reuse(self): + pool = DSATokenToKVPool( + size=256, + page_size=64, + kv_lora_rank=128, + dtype=torch.bfloat16, + qk_rope_head_dim=32, + layer_num=2, + device="cuda", + enable_memory_saver=False, + kv_cache_dim=160, + index_head_dim=128, + ) + anchor = MLATokenToKVPoolHost( + pool, + host_to_device_ratio=2, + host_size=0, + page_size=64, + layout="layer_first", + pin_memory=False, + ) + self.addCleanup(anchor.destroy) + host = DSAIndexerPoolHost( + decl=make_dsa_indexer_pool_decl(pool), anchor_host=anchor + ) + buffer = host.index_k_with_scale_buffer + self.addCleanup(_cuda_host_unregister, buffer) + host.destroy() + self.assertIsNone(host.index_k_with_scale_buffer) + self.assertEqual(host.index_k_data_refs, []) + host.destroy() + _cuda_host_register(buffer) + torch.cuda.synchronize() + + +if __name__ == "__main__": + unittest.main()