【hicache】Optimize HiCache prefetch logic: adapt to remaining available memory when memory is insufficient (#16370)

Signed-off-by: CLFutureX <chenyongqyl@163.com> Co-authored-by: Zhiqiang Xie <xiezhq@stanford.edu>
2026-04-27 17:18:35 -07:00
parent 27659ea8c8
commit 4a04a9818e
1 changed files with 11 additions and 3 deletions
@@ -1287,9 +1287,17 @@ class HiRadixCache(RadixCache):
            self.evict_host(prefetch_length)
            host_indices = self.cache_controller.mem_pool_host.alloc(prefetch_length)
        if host_indices is None:
-            last_host_node.release_host()
-            # no sufficient host memory for prefetch
-            return
+            avaliable_size = self.cache_controller.mem_pool_host.available_size()
+            prefetch_length = avaliable_size - (avaliable_size % self.page_size)
+            if prefetch_length >= self.prefetch_threshold:
+                new_input_tokens = new_input_tokens[:prefetch_length]
+                host_indices = self.cache_controller.mem_pool_host.alloc(
+                    prefetch_length
+                )
+            else:
+                last_host_node.release_host()
+                # no sufficient host memory for prefetch
+                return
        operation = self.cache_controller.prefetch(
            req_id,
            host_indices,