[Fix] unified_cache: Prevent misaligned Mamba checkpoints after speculative truncation (#43387)
This commit is contained in:
@@ -1422,6 +1422,9 @@ class SchedulerBatchResultProcessor:
|
||||
self.decode_offload_manager.offload_kv_cache(req)
|
||||
|
||||
if req.finished():
|
||||
if not self._mamba_can_cache_finished_req(req, result, i):
|
||||
req.skip_radix_cache_insert = True
|
||||
|
||||
# isinstance narrowing: create_worker may also return plain
|
||||
# TpModelWorker-based drafts, which carry no spec-worker hooks.
|
||||
if isinstance(self.draft_worker, BaseSpecWorker):
|
||||
@@ -1455,6 +1458,30 @@ class SchedulerBatchResultProcessor:
|
||||
|
||||
self._maybe_collect_customized_info(i, req, logits_output)
|
||||
|
||||
def _mamba_can_cache_finished_req(
|
||||
self, req: Req, result: GenerationBatchResult, i: int
|
||||
) -> bool:
|
||||
if (
|
||||
result.num_correct_drafts_per_req_cpu is None
|
||||
or req.kv.mamba_ping_pong_track_buffer is not None
|
||||
or not self.tree_cache.supports_mamba()
|
||||
):
|
||||
return True
|
||||
|
||||
# no_buffer donates the live state, which cannot be shortened with the key.
|
||||
if req.owned_kv_len() != req.kv.kv_committed_len:
|
||||
return False
|
||||
|
||||
retained = result.grammar_retained_tokens
|
||||
if retained is None or retained[i] is None:
|
||||
return True
|
||||
|
||||
num_verified = (
|
||||
result.num_correct_drafts_per_req_cpu[i]
|
||||
+ result.num_non_draft_tokens_per_req
|
||||
)
|
||||
return len(retained[i]) == num_verified
|
||||
|
||||
def _maybe_update_reasoning_tokens(
|
||||
self,
|
||||
req: Req,
|
||||
|
||||
Reference in New Issue
Block a user