[Fix] unified_cache: Prevent misaligned Mamba checkpoints after speculative truncation (#43387)

This commit is contained in:
Shuwen Wang
2026-10-17 20:20:49 +08:00
committed by GitHub
parent 025655068c
commit 86e67f7d93
3 changed files with 241 additions and 34 deletions
@@ -1422,6 +1422,9 @@ class SchedulerBatchResultProcessor:
self.decode_offload_manager.offload_kv_cache(req)
if req.finished():
if not self._mamba_can_cache_finished_req(req, result, i):
req.skip_radix_cache_insert = True
# isinstance narrowing: create_worker may also return plain
# TpModelWorker-based drafts, which carry no spec-worker hooks.
if isinstance(self.draft_worker, BaseSpecWorker):
@@ -1455,6 +1458,30 @@ class SchedulerBatchResultProcessor:
self._maybe_collect_customized_info(i, req, logits_output)
def _mamba_can_cache_finished_req(
self, req: Req, result: GenerationBatchResult, i: int
) -> bool:
if (
result.num_correct_drafts_per_req_cpu is None
or req.kv.mamba_ping_pong_track_buffer is not None
or not self.tree_cache.supports_mamba()
):
return True
# no_buffer donates the live state, which cannot be shortened with the key.
if req.owned_kv_len() != req.kv.kv_committed_len:
return False
retained = result.grammar_retained_tokens
if retained is None or retained[i] is None:
return True
num_verified = (
result.num_correct_drafts_per_req_cpu[i]
+ result.num_non_draft_tokens_per_req
)
return len(retained[i]) == num_verified
def _maybe_update_reasoning_tokens(
self,
req: Req,