From f65aae849366cfab82cd027a722a67ec7193f454 Mon Sep 17 00:00:00 2001 From: shuwenn <47200617+alphabetc1@users.noreply.github.com> Date: Wed, 3 Jun 2026 22:06:18 +0800 Subject: [PATCH] [HiCache] fix: truncate prefetch key on degraded allocation (#25991) --- python/sglang/srt/mem_cache/hiradix_cache.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index 37a8a3698..aef02c2ca 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -1381,13 +1381,16 @@ class HiRadixCache(RadixCache): self.evict_host(prefetch_length) host_indices = self.cache_controller.mem_pool_host.alloc(prefetch_length) if host_indices is None: - avaliable_size = self.cache_controller.mem_pool_host.available_size() - prefetch_length = avaliable_size - (avaliable_size % self.page_size) + available_size = self.cache_controller.mem_pool_host.available_size() + prefetch_length = available_size - (available_size % self.page_size) if prefetch_length >= self.prefetch_threshold: - new_input_tokens = new_input_tokens[:prefetch_length] + prefetch_key = prefetch_key[:prefetch_length] host_indices = self.cache_controller.mem_pool_host.alloc( prefetch_length ) + if host_indices is None: + last_host_node.release_host() + return else: last_host_node.release_host() # no sufficient host memory for prefetch