[Auto Sync] Update schedule_batch.py, common.py, eagle_info... (20260105) (#16519)
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: cctry <shiyang@x.ai>
This commit is contained in:
co-authored by
github-actions[bot]
cctry
parent
1e5de05e35
commit
fb04e7e3c8
@@ -484,6 +484,14 @@ def alloc_for_decode(batch: ScheduleBatch, token_per_req: int) -> torch.Tensor:
|
||||
|
||||
def release_kv_cache(req: Req, tree_cache: BasePrefixCache, is_insert: bool = True):
|
||||
tree_cache.cache_finished_req(req, is_insert=is_insert)
|
||||
|
||||
# MambaRadixCache may alloc mamba state before alloc KV cache
|
||||
if req.req_pool_idx is None:
|
||||
assert isinstance(
|
||||
tree_cache, MambaRadixCache
|
||||
), "Only MambaRadixCache can handle abort with prefix cache hit before alloc"
|
||||
return
|
||||
|
||||
start_p, end_p = req.pop_overallocated_kv_cache()
|
||||
|
||||
global_server_args = get_global_server_args()
|
||||
|
||||
Reference in New Issue
Block a user