minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

【hicache】Optimize HiCache prefetch logic: adapt to remaining available memory when memory is insufficient (#16370)

Browse Source 
Signed-off-by: CLFutureX <chenyongqyl@163.com>
Co-authored-by: Zhiqiang Xie <xiezhq@stanford.edu>
This commit is contained in:
PiteXChen
2026-04-27 17:18:35 -07:00
committed by GitHub
co-authored by Zhiqiang Xie
parent 27659ea8c8
commit 4a04a9818e
1 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/mem_cache/hiradix_cache.py
+8
View File
@@ -1287,6 +1287,14 @@ class HiRadixCache(RadixCache):
self.evict_host(prefetch_length) self.evict_host(prefetch_length)
host_indices = self.cache_controller.mem_pool_host.alloc(prefetch_length) host_indices = self.cache_controller.mem_pool_host.alloc(prefetch_length)
if host_indices is None: if host_indices is None:
avaliable_size = self.cache_controller.mem_pool_host.available_size()
prefetch_length = avaliable_size - (avaliable_size % self.page_size)
if prefetch_length >= self.prefetch_threshold:
new_input_tokens = new_input_tokens[:prefetch_length]
host_indices = self.cache_controller.mem_pool_host.alloc(
prefetch_length
)
else:
last_host_node.release_host() last_host_node.release_host()
# no sufficient host memory for prefetch # no sufficient host memory for prefetch
return return