Introduce global alloc_len_per_decode & clean check decode memory (#15115)

2026-01-26 10:26:20 -08:00
parent 8643fb2f52
commit 85d077f44d
10 changed files with 76 additions and 101 deletions
@@ -235,8 +235,7 @@ def evict_from_tree_cache(tree_cache: BasePrefixCache | None, num_tokens: int):

    allocator = tree_cache.token_to_kv_pool_allocator

-    # Check if this is a hybrid allocator
-    if hasattr(allocator, "full_available_size"):
+    if isinstance(allocator, SWATokenToKVPoolAllocator):
        # Hybrid allocator
        full_available_size = allocator.full_available_size()
        swa_available_size = allocator.swa_available_size()