minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[HiCache] fix: truncate prefetch key on degraded allocation (#25991)

Browse Source
This commit is contained in:
shuwenn
2026-06-03 22:06:18 +08:00
committed by GitHub
parent 33f943fbf5
commit f65aae8493
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/mem_cache/hiradix_cache.py
+6 -3
View File
@@ -1381,13 +1381,16 @@ class HiRadixCache(RadixCache):
self.evict_host(prefetch_length)
host_indices = self.cache_controller.mem_pool_host.alloc(prefetch_length)
if host_indices is None:
avaliable_size = self.cache_controller.mem_pool_host.available_size()
prefetch_length = avaliable_size - (avaliable_size % self.page_size)
available_size = self.cache_controller.mem_pool_host.available_size()
prefetch_length = available_size - (available_size % self.page_size)
if prefetch_length >= self.prefetch_threshold:
new_input_tokens = new_input_tokens[:prefetch_length]
prefetch_key = prefetch_key[:prefetch_length]
host_indices = self.cache_controller.mem_pool_host.alloc(
prefetch_length
)
if host_indices is None:
last_host_node.release_host()
return
else:
last_host_node.release_host()
# no sufficient host memory for prefetch