[Fix] mamba: Keep prefill rows' checkpoint tracking through mixed chunked prefill (#45435)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: YavorGIvanov <YavorGIvanov@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
YavorGIvanov
parent
573b5716f5
commit
32c26788f3
@@ -1003,6 +1003,9 @@ class ReqKvInfo:
|
||||
# Seq len of the other ping-pong slot's state. None means what is in
|
||||
# that slot cannot be named (never written, donated, freed).
|
||||
mamba_prev_track_seqlen: Optional[int] = None
|
||||
# Decode batch whose boundary step rode a mixed forward, which wrote no
|
||||
# track state; the undo there leaves next == last without an in-place write.
|
||||
mamba_untracked_decode_batch_idx: Optional[int] = None
|
||||
# Deferred COW: source mamba pool index from radix cache node (copy on forward stream)
|
||||
mamba_cow_src_index: Optional[torch.Tensor] = None
|
||||
# Deferred clear: newly allocated mamba slot needs zeroing on forward stream
|
||||
@@ -2092,6 +2095,7 @@ class Req(ReqDllmMixin):
|
||||
self.kv.mamba_last_track_idx = None
|
||||
self.kv.mamba_last_track_seqlen = None
|
||||
self.kv.mamba_prev_track_seqlen = None
|
||||
self.kv.mamba_untracked_decode_batch_idx = None
|
||||
self.mamba_branching_seqlen = None
|
||||
self.kv.mamba_cow_src_index = None
|
||||
self.kv.mamba_needs_clear = False
|
||||
@@ -3312,8 +3316,33 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
merged_seq_lens_cpu = None
|
||||
out_cache_loc = torch.cat([self.out_cache_loc, running_out_cache_loc])
|
||||
|
||||
# merge_batch drops the Mamba track fields, but checkpoint_kv_cache still
|
||||
# donates each prefill row's track slot after this forward, so keep the
|
||||
# prefill rows' tracking. Decode tails are not tracked in a mixed step.
|
||||
prefill_track = None
|
||||
if self.mamba_track_indices is not None:
|
||||
if get_exec().mamba.enable_mamba_extra_buffer_lazy:
|
||||
running_batch.mamba_lazy_undo_prealloc_at_boundary()
|
||||
prefill_track = (
|
||||
self.mamba_track_indices,
|
||||
self.mamba_track_mask,
|
||||
self.mamba_track_seqlens,
|
||||
self.mamba_prefill_track_mask_cpu,
|
||||
self.mamba_track_seqlens_cpu,
|
||||
)
|
||||
|
||||
self.merge_batch(running_batch)
|
||||
self.out_cache_loc = out_cache_loc
|
||||
if prefill_track is not None:
|
||||
indices, mask, seqlens, mask_cpu, seqlens_cpu = prefill_track
|
||||
untracked = torch.full(
|
||||
(running_bs,), -1, dtype=torch.int64, device=indices.device
|
||||
)
|
||||
self.mamba_track_indices = torch.cat([indices, untracked])
|
||||
self.mamba_track_mask = torch.cat([mask, untracked >= 0])
|
||||
self.mamba_track_seqlens = torch.cat([seqlens, untracked])
|
||||
self.mamba_prefill_track_mask_cpu = mask_cpu + [False] * running_bs
|
||||
self.mamba_track_seqlens_cpu = seqlens_cpu + [-1] * running_bs
|
||||
if merged_seq_lens_cpu is not None:
|
||||
self.seq_lens_cpu = merged_seq_lens_cpu
|
||||
if tail_base is not None:
|
||||
@@ -3618,6 +3647,26 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# The in-flight write invalidates the keep slot's old checkpoint.
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
|
||||
def mamba_lazy_undo_prealloc_at_boundary(self):
|
||||
"""Give back the slots mamba_lazy_prealloc_at_boundary handed this decode
|
||||
batch before it rides a mixed forward, which writes no decode track state
|
||||
and whose result path never frees them."""
|
||||
if self.mamba_track_mask_cpu is None:
|
||||
return
|
||||
pool = self.req_to_token_pool
|
||||
for req, at_boundary in zip(self.reqs, self.mamba_track_mask_cpu):
|
||||
if not at_boundary:
|
||||
continue
|
||||
buf = req.kv.mamba_ping_pong_track_buffer
|
||||
next_idx = req.kv.mamba_next_track_idx
|
||||
req.kv.mamba_untracked_decode_batch_idx = req.decode_batch_idx
|
||||
if buf[1 - next_idx].item() == -1:
|
||||
# The prealloc failed, so tracking would have been in place.
|
||||
continue
|
||||
pool.mamba_allocator.free(buf[next_idx].unsqueeze(0))
|
||||
pool.set_mamba_ping_pong_slot(req, next_idx, -1)
|
||||
req.kv.mamba_next_track_idx = 1 - next_idx
|
||||
|
||||
def mamba_lazy_spec_prepare(self, mamba_track_interval: int, max_draft_tokens: int):
|
||||
"""Lazy-mode spec counterpart of mamba_lazy_prealloc_at_boundary.
|
||||
|
||||
|
||||
@@ -1400,6 +1400,7 @@ class SchedulerBatchResultProcessor:
|
||||
and lookahead == 1
|
||||
and known_mamba_boundary
|
||||
and req.kv.mamba_next_track_idx == req.kv.mamba_last_track_idx
|
||||
and req.kv.mamba_untracked_decode_batch_idx != req.decode_batch_idx
|
||||
):
|
||||
req.mamba_lazy_checkpoint = False
|
||||
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
"""Mixed chunked prefill must keep the prefill rows' Mamba tracking.
|
||||
|
||||
The prefill rows of a mixed batch donate ``mamba_ping_pong_track_buffer[
|
||||
mamba_last_track_idx]`` to the radix tree after the forward (checkpoint_kv_cache).
|
||||
prepare_for_extend points that slot at the forward's track destination, so the
|
||||
mixed forward must still carry the track fields, or the donated checkpoint is
|
||||
whatever the slot held before.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
from sglang.test.test_utils import maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.layers.attention.hybrid_linear_attn_backend import ( # noqa: E402
|
||||
MambaAttnBackendBase,
|
||||
)
|
||||
from sglang.srt.managers import schedule_batch as sb # noqa: E402
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch # noqa: E402
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode # noqa: E402
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm # noqa: E402
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
GRID = 128
|
||||
STATE_CHUNK = 64
|
||||
|
||||
|
||||
def _settings(lazy):
|
||||
return SimpleNamespace(
|
||||
mamba=SimpleNamespace(
|
||||
enable_mamba_extra_buffer=True, enable_mamba_extra_buffer_lazy=lazy
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class _Pool:
|
||||
"""Ping-pong slot bookkeeping of HybridReqToTokenPool, CPU only."""
|
||||
|
||||
def __init__(self):
|
||||
self.freed = []
|
||||
self.mamba_allocator = SimpleNamespace(
|
||||
free=lambda slots: self.freed.extend(slots.tolist())
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def set_mamba_ping_pong_slot(req, idx, value):
|
||||
req.kv.mamba_ping_pong_track_buffer[idx] = value
|
||||
|
||||
@staticmethod
|
||||
def get_mamba_ping_pong_other_idx(idx):
|
||||
return 1 - idx
|
||||
|
||||
|
||||
class _Req:
|
||||
def __init__(self, rid, origin_len, output_len, buf, next_idx, last_idx=None):
|
||||
self.rid = rid
|
||||
self.origin_input_ids = list(range(origin_len))
|
||||
self.output_ids = list(range(output_len))
|
||||
self.prefix_indices = []
|
||||
self.prefix_len = 0
|
||||
self.extend_end = origin_len
|
||||
self.mamba_branching_seqlen = None
|
||||
self.decode_batch_idx = 3
|
||||
self.kv = SimpleNamespace(
|
||||
mamba_ping_pong_track_buffer=torch.tensor(buf, dtype=torch.int64),
|
||||
mamba_next_track_idx=next_idx,
|
||||
mamba_last_track_idx=last_idx,
|
||||
mamba_last_track_seqlen=None,
|
||||
mamba_untracked_decode_batch_idx=None,
|
||||
)
|
||||
|
||||
def refresh_fill_ids(self):
|
||||
pass
|
||||
|
||||
def set_extend_range(self, start, end):
|
||||
self.prefix_len = start
|
||||
self.extend_end = end
|
||||
|
||||
@property
|
||||
def extend_len(self):
|
||||
return self.extend_end - self.prefix_len
|
||||
|
||||
|
||||
def _batch(reqs, pool, **fields):
|
||||
bs = len(reqs)
|
||||
batch = ScheduleBatch(
|
||||
reqs=reqs,
|
||||
req_to_token_pool=pool,
|
||||
model_config=SimpleNamespace(
|
||||
is_encoder_decoder=False,
|
||||
hf_text_config=SimpleNamespace(mamba_chunk_size=STATE_CHUNK),
|
||||
),
|
||||
tree_cache=SimpleNamespace(page_size=GRID),
|
||||
sampling_info=MagicMock(),
|
||||
spec_algorithm=SpeculativeAlgorithm.NONE,
|
||||
req_pool_indices=torch.arange(bs),
|
||||
req_pool_indices_cpu=torch.arange(bs),
|
||||
orig_seq_lens=torch.zeros(bs, dtype=torch.int32),
|
||||
)
|
||||
for name, value in fields.items():
|
||||
setattr(batch, name, value)
|
||||
return batch
|
||||
|
||||
|
||||
class TestMixWithRunningMambaTrack(unittest.TestCase):
|
||||
def setUp(self):
|
||||
for name, value in (
|
||||
("get_parallel", lambda: SimpleNamespace(dcp_enabled=False)),
|
||||
("mamba_cache_chunk_size", lambda: STATE_CHUNK),
|
||||
("mamba_checkpoint_grid", lambda page_size: GRID),
|
||||
):
|
||||
p = patch.object(sb, name, value)
|
||||
p.start()
|
||||
self.addCleanup(p.stop)
|
||||
|
||||
def _mix(self, lazy):
|
||||
pool = _Pool()
|
||||
# Prefill rows: a 300-token extend after a 128-token prefix tracks at 384;
|
||||
# a 40-token extend tracks nothing.
|
||||
tracked = _Req("p0", 428, 0, [7, -1 if lazy else 8], next_idx=0)
|
||||
tracked.prefix_indices = list(range(128))
|
||||
tracked.set_extend_range(128, 428)
|
||||
short = _Req("p1", 40, 0, [9, -1 if lazy else 10], next_idx=0)
|
||||
short.set_extend_range(0, 40)
|
||||
extend = _batch(
|
||||
[tracked, short],
|
||||
pool,
|
||||
forward_mode=ForwardMode.EXTEND,
|
||||
out_cache_loc=torch.arange(340),
|
||||
seq_lens=torch.tensor([428, 40]),
|
||||
seq_lens_cpu=torch.tensor([428, 40]),
|
||||
prefix_lens=[128, 0],
|
||||
extend_lens=[300, 40],
|
||||
extend_num_tokens=340,
|
||||
extend_logprob_start_lens=[0, 0],
|
||||
return_logprob=False,
|
||||
is_prefill_only=True,
|
||||
)
|
||||
with patch.object(sb, "get_exec", return_value=_settings(lazy)):
|
||||
entries = [
|
||||
extend._mamba_radix_cache_v2_req_prepare_for_extend(r)
|
||||
for r in extend.reqs
|
||||
]
|
||||
# The assembly at the end of prepare_for_extend.
|
||||
extend.mamba_prefill_track_mask_cpu = [e.track_mask for e in entries]
|
||||
extend.mamba_track_seqlens_cpu = [e.track_seqlen for e in entries]
|
||||
extend.mamba_track_indices = torch.tensor([e.track_index for e in entries])
|
||||
extend.mamba_track_mask = torch.tensor([e.track_mask for e in entries])
|
||||
extend.mamba_track_seqlens = torch.tensor([e.track_seqlen for e in entries])
|
||||
|
||||
# Decode tails after prepare_for_decode. d0 sits on a track boundary
|
||||
# (seq_len 640); in lazy mode the prealloc handed it slot 11 and moved
|
||||
# its next index there.
|
||||
if lazy:
|
||||
d0 = _Req("d0", 600, 40, [5, 11], next_idx=1, last_idx=0)
|
||||
else:
|
||||
d0 = _Req("d0", 600, 40, [5, 12], next_idx=1, last_idx=0)
|
||||
d1 = _Req("d1", 200, 3, [6, -1 if lazy else 13], next_idx=0)
|
||||
running = _batch(
|
||||
[d0, d1],
|
||||
pool,
|
||||
forward_mode=ForwardMode.DECODE,
|
||||
out_cache_loc=torch.tensor([1000, 1001]),
|
||||
seq_lens=torch.tensor([640, 203]),
|
||||
seq_lens_cpu=torch.tensor([640, 203]),
|
||||
return_logprob=False,
|
||||
mamba_track_indices=torch.tensor([11 if lazy else 12, 6]),
|
||||
mamba_track_buffer_indices=[1, 0],
|
||||
mamba_track_mask=torch.tensor([True, False]),
|
||||
mamba_track_mask_cpu=[True, False],
|
||||
)
|
||||
with patch.object(sb, "get_exec", return_value=_settings(lazy)):
|
||||
extend.mix_with_running(running)
|
||||
return extend, pool
|
||||
|
||||
def test_mixed_batch_tracks_prefill_rows_into_the_donated_slot(self):
|
||||
for lazy in (False, True):
|
||||
with self.subTest(lazy=lazy):
|
||||
batch, _ = self._mix(lazy)
|
||||
self.assertEqual(batch.forward_mode, ForwardMode.MIXED)
|
||||
self.assertEqual(
|
||||
batch.mamba_prefill_track_mask_cpu, [True, False, False, False]
|
||||
)
|
||||
self.assertEqual(batch.mamba_track_seqlens_cpu, [428, -1, -1, -1])
|
||||
self.assertEqual(
|
||||
batch.mamba_track_mask.tolist(), [True, False, False, False]
|
||||
)
|
||||
self.assertEqual(batch.mamba_track_seqlens.tolist(), [428, -1, -1, -1])
|
||||
self.assertEqual(batch.mamba_track_indices.tolist()[:2], [7, 9])
|
||||
self.assertEqual(batch.mamba_track_indices.tolist()[2:], [-1, -1])
|
||||
|
||||
# The slot checkpoint_kv_cache donates (donate_mamba_ping_pong_slot)
|
||||
# for the tracked prefill row ...
|
||||
tracked = batch.reqs[0]
|
||||
self.assertEqual(tracked.kv.mamba_last_track_seqlen, 384)
|
||||
donated = tracked.kv.mamba_ping_pong_track_buffer[
|
||||
tracked.kv.mamba_last_track_idx
|
||||
].item()
|
||||
# ... is the one the mixed forward writes (eager track plan).
|
||||
backend = object.__new__(MambaAttnBackendBase)
|
||||
backend.device = "cpu"
|
||||
backend._mamba_chunk_size = STATE_CHUNK
|
||||
view = SimpleNamespace(
|
||||
batch_size=4,
|
||||
forward_mode=batch.forward_mode,
|
||||
mamba_prefill_track_mask_cpu=batch.mamba_prefill_track_mask_cpu,
|
||||
mamba_track_seqlens_cpu=batch.mamba_track_seqlens_cpu,
|
||||
extend_seq_lens_cpu=batch.extend_lens,
|
||||
extend_prefix_lens_cpu=batch.prefix_lens,
|
||||
mamba_track_indices=batch.mamba_track_indices,
|
||||
)
|
||||
self.assertTrue(backend._has_cpu_prefill_track_metadata(view))
|
||||
plan = backend._init_track_ssm_indices_from_cpu(
|
||||
torch.arange(100, 104), view
|
||||
)
|
||||
h_dst, final_dst = plan[2], plan[5]
|
||||
self.assertEqual(sorted(h_dst.tolist() + final_dst.tolist()), [donated])
|
||||
|
||||
def test_lazy_decode_tails_give_back_the_boundary_slot(self):
|
||||
batch, pool = self._mix(lazy=True)
|
||||
d0, d1 = batch.reqs[2], batch.reqs[3]
|
||||
self.assertEqual(pool.freed, [11])
|
||||
self.assertEqual(d0.kv.mamba_ping_pong_track_buffer.tolist(), [5, -1])
|
||||
self.assertEqual(d0.kv.mamba_next_track_idx, 0)
|
||||
self.assertEqual(d0.kv.mamba_last_track_idx, 0)
|
||||
self.assertEqual(d0.kv.mamba_untracked_decode_batch_idx, d0.decode_batch_idx)
|
||||
self.assertEqual(d1.kv.mamba_ping_pong_track_buffer.tolist(), [6, -1])
|
||||
self.assertEqual(d1.kv.mamba_next_track_idx, 0)
|
||||
self.assertIsNone(d1.kv.mamba_untracked_decode_batch_idx)
|
||||
|
||||
def test_lazy_undo_keeps_the_slot_when_the_prealloc_failed(self):
|
||||
pool = _Pool()
|
||||
req = _Req("d0", 600, 40, [5, -1], next_idx=0, last_idx=0)
|
||||
running = _batch([req], pool, mamba_track_mask_cpu=[True])
|
||||
running.mamba_lazy_undo_prealloc_at_boundary()
|
||||
self.assertEqual(pool.freed, [])
|
||||
self.assertEqual(req.kv.mamba_ping_pong_track_buffer.tolist(), [5, -1])
|
||||
self.assertEqual(req.kv.mamba_next_track_idx, 0)
|
||||
self.assertEqual(req.kv.mamba_untracked_decode_batch_idx, req.decode_batch_idx)
|
||||
|
||||
def test_non_lazy_decode_tails_keep_their_slots(self):
|
||||
batch, pool = self._mix(lazy=False)
|
||||
self.assertEqual(pool.freed, [])
|
||||
self.assertEqual(batch.reqs[2].kv.mamba_next_track_idx, 1)
|
||||
|
||||
|
||||
class TestLazyFinishBeforeMixedBoundary(unittest.TestCase):
|
||||
"""A request finishes at grid - 1 in step N while overlapped step N + 1, its
|
||||
boundary step, is mixed. The keep slot is untouched, so the radix insert must
|
||||
still happen; only an in-place boundary write may skip it."""
|
||||
|
||||
def _finish(self, untracked):
|
||||
from sglang.srt.managers.scheduler_components import (
|
||||
batch_result_processor as brp,
|
||||
)
|
||||
|
||||
req = SimpleNamespace(
|
||||
rid="r",
|
||||
decode_batch_idx=5,
|
||||
mamba_lazy_checkpoint=True,
|
||||
multimodal_inputs=None,
|
||||
finished=lambda: True,
|
||||
time_stats=MagicMock(),
|
||||
kv=SimpleNamespace(
|
||||
mamba_next_track_idx=0,
|
||||
mamba_last_track_idx=0,
|
||||
mamba_untracked_decode_batch_idx=5 if untracked else None,
|
||||
),
|
||||
)
|
||||
batch = SimpleNamespace(
|
||||
mamba_track_mask_cpu=[False],
|
||||
mamba_track_mask_next_cpu=[True],
|
||||
mamba_decode_batch_idx_cpu=[4],
|
||||
)
|
||||
processor = SimpleNamespace(
|
||||
draft_worker=None,
|
||||
model_worker=None,
|
||||
tree_cache=object(),
|
||||
_mamba_can_cache_finished_req=lambda req, result, i: True,
|
||||
_prepare_kv_cache_release=lambda req: None,
|
||||
_maybe_collect_routed_experts=lambda req: None,
|
||||
_maybe_collect_indexer_topk=lambda req: None,
|
||||
_maybe_collect_customized_info=lambda i, req, out: None,
|
||||
)
|
||||
released = []
|
||||
with (
|
||||
patch.object(brp, "get_exec", return_value=_settings(lazy=True)),
|
||||
patch.object(
|
||||
brp,
|
||||
"get_disagg",
|
||||
return_value=SimpleNamespace(
|
||||
disaggregation_decode_enable_offload_kvcache=False
|
||||
),
|
||||
),
|
||||
patch.object(
|
||||
brp, "get_memory", return_value=SimpleNamespace(enable_hisparse=False)
|
||||
),
|
||||
patch.object(
|
||||
brp,
|
||||
"release_kv_cache",
|
||||
lambda req, cache, checkpoint: released.append(checkpoint),
|
||||
),
|
||||
):
|
||||
brp.SchedulerBatchResultProcessor._handle_finish_state_updated_req(
|
||||
processor, req, batch, None, 0, None
|
||||
)
|
||||
return released
|
||||
|
||||
def test_mixed_boundary_step_keeps_the_radix_insert(self):
|
||||
self.assertEqual(self._finish(untracked=True), [True])
|
||||
|
||||
def test_in_place_boundary_write_skips_the_radix_insert(self):
|
||||
self.assertEqual(self._finish(untracked=False), [False])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -2,7 +2,7 @@ import dataclasses
|
||||
import types
|
||||
import unittest
|
||||
from array import array
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
@@ -11,6 +11,7 @@ from sglang.test.test_utils import maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
from sglang.srt.managers import schedule_batch # noqa: E402
|
||||
from sglang.srt.managers.schedule_batch import ( # noqa: E402
|
||||
ScheduleBatch,
|
||||
split_cached_prefix_by_tier,
|
||||
@@ -212,7 +213,11 @@ class TestMixWithRunningOutOfPlace(unittest.TestCase):
|
||||
extend_snapshot = _snapshot_mutable_fields(extend_batch)
|
||||
running_snapshot = _snapshot_mutable_fields(running_batch)
|
||||
|
||||
extend_batch.mix_with_running(running_batch)
|
||||
settings = types.SimpleNamespace(
|
||||
mamba=types.SimpleNamespace(enable_mamba_extra_buffer_lazy=False)
|
||||
)
|
||||
with patch.object(schedule_batch, "get_exec", return_value=settings):
|
||||
extend_batch.mix_with_running(running_batch)
|
||||
|
||||
self.assertEqual(extend_batch.forward_mode, ForwardMode.MIXED)
|
||||
self.assertIs(extend_batch.mix_running_indices, running_batch.req_pool_indices)
|
||||
|
||||
Reference in New Issue
Block a user