[Fix] mamba: Release consumed lazy Mamba prefix locks before donation retry (#43559)
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from dataclasses import replace
|
||||
from typing import TYPE_CHECKING, Callable, Optional, Sequence
|
||||
|
||||
import torch
|
||||
@@ -504,10 +505,29 @@ class MambaComponent(TreeComponent):
|
||||
if cd.lock_ref == 0:
|
||||
self.tree_core._update_evictable_leaf_sets(node)
|
||||
|
||||
def _alloc_mamba_slot(self) -> torch.Tensor:
|
||||
def _alloc_mamba_slot(self, req: Optional[Req] = None) -> torch.Tensor:
|
||||
"""Allocate one mamba pool slot, evicting if necessary."""
|
||||
slot = self.cache.req_to_token_pool.mamba_allocator.alloc(1)
|
||||
if slot is None:
|
||||
if req is not None and get_exec().mamba.enable_mamba_extra_buffer_lazy:
|
||||
lock = req.lock
|
||||
ct = self.component_type
|
||||
if (
|
||||
lock is not None
|
||||
and not lock.swa_released
|
||||
and ct not in lock.receipt.skipped_lock_components
|
||||
):
|
||||
# Prefill has consumed the COW source; retain only its KV locks.
|
||||
self.cache.dec_lock_ref(
|
||||
lock.node,
|
||||
replace(
|
||||
lock.receipt,
|
||||
skipped_lock_components=tuple(
|
||||
c for c in self.cache.tree_components if c != ct
|
||||
),
|
||||
),
|
||||
)
|
||||
lock.receipt.skipped_lock_components += (ct,)
|
||||
self.cache.evict_for_alloc(EvictParams(num_tokens=0, mamba_num=1))
|
||||
slot = self.cache.req_to_token_pool.mamba_allocator.alloc(1)
|
||||
assert slot is not None, "Can not alloc mamba cache"
|
||||
@@ -611,7 +631,7 @@ class MambaComponent(TreeComponent):
|
||||
# Donate the mamba index to the radix cache instead of copying.
|
||||
if self.int8_ckpt_pool is not None:
|
||||
if self.cache.enable_mamba_extra_buffer:
|
||||
new_slot = self._alloc_mamba_slot()
|
||||
new_slot = self._alloc_mamba_slot(req)
|
||||
src_active = (
|
||||
self.cache.req_to_token_pool.donate_mamba_ping_pong_slot(
|
||||
req, new_slot
|
||||
@@ -624,14 +644,14 @@ class MambaComponent(TreeComponent):
|
||||
req.kv.mamba_pool_idx.view(-1)
|
||||
)
|
||||
elif self.cache.enable_mamba_extra_buffer:
|
||||
new_slot = self._alloc_mamba_slot()
|
||||
new_slot = self._alloc_mamba_slot(req)
|
||||
mamba_value_donated = (
|
||||
self.cache.req_to_token_pool.donate_mamba_ping_pong_slot(
|
||||
req, new_slot
|
||||
)
|
||||
)
|
||||
else:
|
||||
mamba_value_donated = self._alloc_mamba_slot()
|
||||
mamba_value_donated = self._alloc_mamba_slot(req)
|
||||
# mamba_pool is a pure PHYSICAL store; translate both slot ids
|
||||
# virtual->physical (identity for the non-unified memory pool) first.
|
||||
translate = self.cache.req_to_token_pool.translate_mamba_indices
|
||||
|
||||
Reference in New Issue
Block a user