From 4a04a9818e335a4496f6ab8cade377fdc6997fd0 Mon Sep 17 00:00:00 2001 From: PiteXChen <44110731+CLFutureX@users.noreply.github.com> Date: Tue, 28 Apr 2026 08:18:35 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90hicache=E3=80=91Optimize=20HiCache=20p?= =?UTF-8?q?refetch=20logic:=20adapt=20to=20remaining=20available=20memory?= =?UTF-8?q?=20when=20memory=20is=20insufficient=20(#16370)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: CLFutureX Co-authored-by: Zhiqiang Xie --- python/sglang/srt/mem_cache/hiradix_cache.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/mem_cache/hiradix_cache.py b/python/sglang/srt/mem_cache/hiradix_cache.py index 718d186c3..9203e2db2 100644 --- a/python/sglang/srt/mem_cache/hiradix_cache.py +++ b/python/sglang/srt/mem_cache/hiradix_cache.py @@ -1287,9 +1287,17 @@ class HiRadixCache(RadixCache): self.evict_host(prefetch_length) host_indices = self.cache_controller.mem_pool_host.alloc(prefetch_length) if host_indices is None: - last_host_node.release_host() - # no sufficient host memory for prefetch - return + avaliable_size = self.cache_controller.mem_pool_host.available_size() + prefetch_length = avaliable_size - (avaliable_size % self.page_size) + if prefetch_length >= self.prefetch_threshold: + new_input_tokens = new_input_tokens[:prefetch_length] + host_indices = self.cache_controller.mem_pool_host.alloc( + prefetch_length + ) + else: + last_host_node.release_host() + # no sufficient host memory for prefetch + return operation = self.cache_controller.prefetch( req_id, host_indices,