[HiCache] Fix Mamba track-boundary bookkeeping under overlap scheduling (#29792)
Co-authored-by: Hanming Lu <hanminglu@meta.com>
This commit is contained in:
co-authored by
Hanming Lu
parent
a3bd7d9401
commit
8f3d3a31f4
@@ -1388,14 +1388,8 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
return mamba_next_track_idx
|
||||
|
||||
def get_mamba_ping_pong_keep_idx(self, req: Req) -> int:
|
||||
"""Return the ping-pong index holding the most recent tracked state.
|
||||
|
||||
In lazy mode the valid state stays at next_track_idx (no eager swap).
|
||||
In normal mode it is at the "other" index (swapped after each track).
|
||||
"""
|
||||
if self.enable_mamba_extra_buffer_lazy:
|
||||
return req.mamba_next_track_idx
|
||||
return self.get_mamba_ping_pong_other_idx(req.mamba_next_track_idx)
|
||||
"""Return the ping-pong index holding the most recent tracked state."""
|
||||
return req.mamba_last_track_idx
|
||||
|
||||
def _alloc_ping_pong_buffer(self, req: Req):
|
||||
"""Allocate the ping-pong track buffer for a new request.
|
||||
@@ -1422,6 +1416,11 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
buf[:n] = slots
|
||||
req.mamba_ping_pong_track_buffer = buf
|
||||
req.mamba_next_track_idx = 0
|
||||
req.mamba_last_track_idx = (
|
||||
0
|
||||
if self.enable_mamba_extra_buffer_lazy
|
||||
else self.get_mamba_ping_pong_other_idx(0)
|
||||
)
|
||||
|
||||
def set_mamba_ping_pong_slot(self, req: Req, idx: int, value):
|
||||
"""Update a ping-pong slot value and sync the device-side mapping.
|
||||
@@ -1442,8 +1441,6 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
|
||||
Returns the old slot index (shape [1]) for cache insertion and
|
||||
replaces it with new_slot so the request can continue tracking.
|
||||
In lazy mode the valid state is at next_track_idx; in normal mode
|
||||
it is at the "other" index.
|
||||
"""
|
||||
donate_idx = self.get_mamba_ping_pong_keep_idx(req)
|
||||
mamba_value_donated = (
|
||||
@@ -1512,6 +1509,7 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
# tensor on the req side while the new pool slot leaks).
|
||||
req.mamba_ping_pong_track_buffer = None
|
||||
req.mamba_next_track_idx = None
|
||||
req.mamba_last_track_idx = None
|
||||
|
||||
def clear(self):
|
||||
logger.info("Reset HybridReqToTokenPool")
|
||||
|
||||
Reference in New Issue
Block a user