[Fix] mamba: Keep prefill rows' checkpoint tracking through mixed chunked prefill (#45435)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: YavorGIvanov <YavorGIvanov@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
YavorGIvanov
parent
573b5716f5
commit
32c26788f3
@@ -1003,6 +1003,9 @@ class ReqKvInfo:
|
||||
# Seq len of the other ping-pong slot's state. None means what is in
|
||||
# that slot cannot be named (never written, donated, freed).
|
||||
mamba_prev_track_seqlen: Optional[int] = None
|
||||
# Decode batch whose boundary step rode a mixed forward, which wrote no
|
||||
# track state; the undo there leaves next == last without an in-place write.
|
||||
mamba_untracked_decode_batch_idx: Optional[int] = None
|
||||
# Deferred COW: source mamba pool index from radix cache node (copy on forward stream)
|
||||
mamba_cow_src_index: Optional[torch.Tensor] = None
|
||||
# Deferred clear: newly allocated mamba slot needs zeroing on forward stream
|
||||
@@ -2092,6 +2095,7 @@ class Req(ReqDllmMixin):
|
||||
self.kv.mamba_last_track_idx = None
|
||||
self.kv.mamba_last_track_seqlen = None
|
||||
self.kv.mamba_prev_track_seqlen = None
|
||||
self.kv.mamba_untracked_decode_batch_idx = None
|
||||
self.mamba_branching_seqlen = None
|
||||
self.kv.mamba_cow_src_index = None
|
||||
self.kv.mamba_needs_clear = False
|
||||
@@ -3312,8 +3316,33 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
merged_seq_lens_cpu = None
|
||||
out_cache_loc = torch.cat([self.out_cache_loc, running_out_cache_loc])
|
||||
|
||||
# merge_batch drops the Mamba track fields, but checkpoint_kv_cache still
|
||||
# donates each prefill row's track slot after this forward, so keep the
|
||||
# prefill rows' tracking. Decode tails are not tracked in a mixed step.
|
||||
prefill_track = None
|
||||
if self.mamba_track_indices is not None:
|
||||
if get_exec().mamba.enable_mamba_extra_buffer_lazy:
|
||||
running_batch.mamba_lazy_undo_prealloc_at_boundary()
|
||||
prefill_track = (
|
||||
self.mamba_track_indices,
|
||||
self.mamba_track_mask,
|
||||
self.mamba_track_seqlens,
|
||||
self.mamba_prefill_track_mask_cpu,
|
||||
self.mamba_track_seqlens_cpu,
|
||||
)
|
||||
|
||||
self.merge_batch(running_batch)
|
||||
self.out_cache_loc = out_cache_loc
|
||||
if prefill_track is not None:
|
||||
indices, mask, seqlens, mask_cpu, seqlens_cpu = prefill_track
|
||||
untracked = torch.full(
|
||||
(running_bs,), -1, dtype=torch.int64, device=indices.device
|
||||
)
|
||||
self.mamba_track_indices = torch.cat([indices, untracked])
|
||||
self.mamba_track_mask = torch.cat([mask, untracked >= 0])
|
||||
self.mamba_track_seqlens = torch.cat([seqlens, untracked])
|
||||
self.mamba_prefill_track_mask_cpu = mask_cpu + [False] * running_bs
|
||||
self.mamba_track_seqlens_cpu = seqlens_cpu + [-1] * running_bs
|
||||
if merged_seq_lens_cpu is not None:
|
||||
self.seq_lens_cpu = merged_seq_lens_cpu
|
||||
if tail_base is not None:
|
||||
@@ -3618,6 +3647,26 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# The in-flight write invalidates the keep slot's old checkpoint.
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
|
||||
def mamba_lazy_undo_prealloc_at_boundary(self):
|
||||
"""Give back the slots mamba_lazy_prealloc_at_boundary handed this decode
|
||||
batch before it rides a mixed forward, which writes no decode track state
|
||||
and whose result path never frees them."""
|
||||
if self.mamba_track_mask_cpu is None:
|
||||
return
|
||||
pool = self.req_to_token_pool
|
||||
for req, at_boundary in zip(self.reqs, self.mamba_track_mask_cpu):
|
||||
if not at_boundary:
|
||||
continue
|
||||
buf = req.kv.mamba_ping_pong_track_buffer
|
||||
next_idx = req.kv.mamba_next_track_idx
|
||||
req.kv.mamba_untracked_decode_batch_idx = req.decode_batch_idx
|
||||
if buf[1 - next_idx].item() == -1:
|
||||
# The prealloc failed, so tracking would have been in place.
|
||||
continue
|
||||
pool.mamba_allocator.free(buf[next_idx].unsqueeze(0))
|
||||
pool.set_mamba_ping_pong_slot(req, next_idx, -1)
|
||||
req.kv.mamba_next_track_idx = 1 - next_idx
|
||||
|
||||
def mamba_lazy_spec_prepare(self, mamba_track_interval: int, max_draft_tokens: int):
|
||||
"""Lazy-mode spec counterpart of mamba_lazy_prealloc_at_boundary.
|
||||
|
||||
|
||||
@@ -1400,6 +1400,7 @@ class SchedulerBatchResultProcessor:
|
||||
and lookahead == 1
|
||||
and known_mamba_boundary
|
||||
and req.kv.mamba_next_track_idx == req.kv.mamba_last_track_idx
|
||||
and req.kv.mamba_untracked_decode_batch_idx != req.decode_batch_idx
|
||||
):
|
||||
req.mamba_lazy_checkpoint = False
|
||||
|
||||
|
||||
Reference in New Issue
Block a user