[Fix] mamba: Keep prefill rows' checkpoint tracking through mixed chunked prefill (#45435)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: YavorGIvanov <YavorGIvanov@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-20 21:56:50 -07:00
committed by GitHub
co-authored by metamergebot YavorGIvanov
parent 573b5716f5
commit 32c26788f3
4 changed files with 381 additions and 2 deletions
@@ -1003,6 +1003,9 @@ class ReqKvInfo:
# Seq len of the other ping-pong slot's state. None means what is in
# that slot cannot be named (never written, donated, freed).
mamba_prev_track_seqlen: Optional[int] = None
# Decode batch whose boundary step rode a mixed forward, which wrote no
# track state; the undo there leaves next == last without an in-place write.
mamba_untracked_decode_batch_idx: Optional[int] = None
# Deferred COW: source mamba pool index from radix cache node (copy on forward stream)
mamba_cow_src_index: Optional[torch.Tensor] = None
# Deferred clear: newly allocated mamba slot needs zeroing on forward stream
@@ -2092,6 +2095,7 @@ class Req(ReqDllmMixin):
self.kv.mamba_last_track_idx = None
self.kv.mamba_last_track_seqlen = None
self.kv.mamba_prev_track_seqlen = None
self.kv.mamba_untracked_decode_batch_idx = None
self.mamba_branching_seqlen = None
self.kv.mamba_cow_src_index = None
self.kv.mamba_needs_clear = False
@@ -3312,8 +3316,33 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
merged_seq_lens_cpu = None
out_cache_loc = torch.cat([self.out_cache_loc, running_out_cache_loc])
# merge_batch drops the Mamba track fields, but checkpoint_kv_cache still
# donates each prefill row's track slot after this forward, so keep the
# prefill rows' tracking. Decode tails are not tracked in a mixed step.
prefill_track = None
if self.mamba_track_indices is not None:
if get_exec().mamba.enable_mamba_extra_buffer_lazy:
running_batch.mamba_lazy_undo_prealloc_at_boundary()
prefill_track = (
self.mamba_track_indices,
self.mamba_track_mask,
self.mamba_track_seqlens,
self.mamba_prefill_track_mask_cpu,
self.mamba_track_seqlens_cpu,
)
self.merge_batch(running_batch)
self.out_cache_loc = out_cache_loc
if prefill_track is not None:
indices, mask, seqlens, mask_cpu, seqlens_cpu = prefill_track
untracked = torch.full(
(running_bs,), -1, dtype=torch.int64, device=indices.device
)
self.mamba_track_indices = torch.cat([indices, untracked])
self.mamba_track_mask = torch.cat([mask, untracked >= 0])
self.mamba_track_seqlens = torch.cat([seqlens, untracked])
self.mamba_prefill_track_mask_cpu = mask_cpu + [False] * running_bs
self.mamba_track_seqlens_cpu = seqlens_cpu + [-1] * running_bs
if merged_seq_lens_cpu is not None:
self.seq_lens_cpu = merged_seq_lens_cpu
if tail_base is not None:
@@ -3618,6 +3647,26 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# The in-flight write invalidates the keep slot's old checkpoint.
req.kv.mamba_last_track_seqlen = None
def mamba_lazy_undo_prealloc_at_boundary(self):
"""Give back the slots mamba_lazy_prealloc_at_boundary handed this decode
batch before it rides a mixed forward, which writes no decode track state
and whose result path never frees them."""
if self.mamba_track_mask_cpu is None:
return
pool = self.req_to_token_pool
for req, at_boundary in zip(self.reqs, self.mamba_track_mask_cpu):
if not at_boundary:
continue
buf = req.kv.mamba_ping_pong_track_buffer
next_idx = req.kv.mamba_next_track_idx
req.kv.mamba_untracked_decode_batch_idx = req.decode_batch_idx
if buf[1 - next_idx].item() == -1:
# The prealloc failed, so tracking would have been in place.
continue
pool.mamba_allocator.free(buf[next_idx].unsqueeze(0))
pool.set_mamba_ping_pong_slot(req, next_idx, -1)
req.kv.mamba_next_track_idx = 1 - next_idx
def mamba_lazy_spec_prepare(self, mamba_track_interval: int, max_draft_tokens: int):
"""Lazy-mode spec counterpart of mamba_lazy_prealloc_at_boundary.
@@ -1400,6 +1400,7 @@ class SchedulerBatchResultProcessor:
and lookahead == 1
and known_mamba_boundary
and req.kv.mamba_next_track_idx == req.kv.mamba_last_track_idx
and req.kv.mamba_untracked_decode_batch_idx != req.decode_batch_idx
):
req.mamba_lazy_checkpoint = False