Fix DSA index host buffer cleanup (#43100)
This commit is contained in:
@@ -24,6 +24,7 @@ from sglang.srt.mem_cache.pool_host.base import (
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.common import (
|
||||
ALLOC_MEMORY_FUNCS,
|
||||
_cuda_host_unregister,
|
||||
get_allocator_from_storage,
|
||||
make_kernel_ptr_table,
|
||||
)
|
||||
@@ -142,6 +143,7 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
is_dummy: bool = False,
|
||||
):
|
||||
self._is_dummy = is_dummy
|
||||
self._destroyed = False
|
||||
self.decl = decl
|
||||
storage_info = decl.storage_info
|
||||
device_pool = decl.device_pool
|
||||
@@ -227,6 +229,15 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
self.lock = threading.RLock()
|
||||
self.clear()
|
||||
|
||||
def destroy(self):
|
||||
if self._destroyed:
|
||||
return
|
||||
if not self._is_dummy and self.pin_memory and (_is_cuda or _is_hip):
|
||||
_cuda_host_unregister(self.index_k_with_scale_buffer)
|
||||
self.index_k_with_scale_buffer = None
|
||||
self.index_k_data_refs = []
|
||||
super().destroy()
|
||||
|
||||
def get_size_per_token(self):
|
||||
return self.decl.storage_info.bytes_per_token_per_layer * self.layer_num
|
||||
|
||||
|
||||
Reference in New Issue
Block a user