[RadixTree][3/N Refactor]:Support unified insert/evict params (#17401)
This commit is contained in:
@@ -7,7 +7,7 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache, EvictParams
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWATokenToKVPoolAllocator
|
||||
from sglang.srt.server_args import get_global_server_args
|
||||
@@ -244,11 +244,13 @@ def evict_from_tree_cache(tree_cache: BasePrefixCache | None, num_tokens: int):
|
||||
if full_available_size < num_tokens or swa_available_size < num_tokens:
|
||||
full_num_tokens = max(0, num_tokens - full_available_size)
|
||||
swa_num_tokens = max(0, num_tokens - swa_available_size)
|
||||
tree_cache.evict(full_num_tokens, swa_num_tokens)
|
||||
tree_cache.evict(
|
||||
EvictParams(num_tokens=full_num_tokens, swa_num_tokens=swa_num_tokens)
|
||||
)
|
||||
else:
|
||||
# Standard allocator
|
||||
if allocator.available_size() < num_tokens:
|
||||
tree_cache.evict(num_tokens)
|
||||
tree_cache.evict(EvictParams(num_tokens=num_tokens))
|
||||
|
||||
|
||||
def alloc_paged_token_slots_extend(
|
||||
@@ -311,7 +313,7 @@ def alloc_req_slots(
|
||||
if mamba_available_size < mamba_state_needed:
|
||||
if tree_cache is not None and tree_cache.supports_mamba():
|
||||
mamba_num = max(0, mamba_state_needed - mamba_available_size)
|
||||
tree_cache.evict_mamba(mamba_num)
|
||||
tree_cache.evict(EvictParams(num_tokens=0, mamba_num=mamba_num))
|
||||
req_pool_indices = req_to_token_pool.alloc(num_reqs, reqs)
|
||||
else:
|
||||
req_pool_indices = req_to_token_pool.alloc(num_reqs)
|
||||
|
||||
Reference in New Issue
Block a user