[Perf] scheduler: Plan hybrid Mamba extend batches without host-device syncs (#45472)

Co-authored-by: Lu Fang <116399278+luccafong@users.noreply.github.com>
Co-authored-by: Shiyan Deng <20773414+842974287@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-21 11:56:50 -07:00
committed by GitHub
co-authored by Lu Fang Shiyan Deng
parent 5e79333c95
commit 740d5f9f9e
14 changed files with 217 additions and 141 deletions
@@ -286,6 +286,7 @@ class MlxAuxiliaryStateReqToTokenPool(ReqToTokenPool):
if mamba_ping_pong_track_buffer_to_keep is None:
self.auxiliary_state_pool.free(track_buffer)
req.kv.mamba_ping_pong_track_buffer = None
req.kv.mamba_ping_pong_empty = None
req.kv.mamba_next_track_idx = None
req.kv.mamba_last_track_idx = None
@@ -387,6 +388,7 @@ class MlxAuxiliaryStateComponent(MambaComponent):
if track_buffer is not None:
self.cache.req_to_token_pool.auxiliary_state_pool.free(track_buffer)
req.kv.mamba_ping_pong_track_buffer = None
req.kv.mamba_ping_pong_empty = None
req.kv.mamba_next_track_idx = None
req.kv.mamba_last_track_idx = None
req.kv.mamba_last_track_seqlen = None
@@ -412,6 +414,7 @@ class MlxAuxiliaryStateComponent(MambaComponent):
if track_buffer is not None:
self.cache.req_to_token_pool.auxiliary_state_pool.free(track_buffer)
req.kv.mamba_ping_pong_track_buffer = None
req.kv.mamba_ping_pong_empty = None
req.kv.mamba_next_track_idx = None
req.kv.mamba_last_track_idx = None
req.kv.mamba_pool_idx = None
@@ -417,6 +417,7 @@ class MlxModelRunner:
)
return
req.kv.mamba_ping_pong_track_buffer = track_buffer
req.kv.mamba_ping_pong_empty = [False]
req.kv.mamba_next_track_idx = 0
req.kv.mamba_last_track_idx = 0
+9 -1
View File
@@ -655,7 +655,15 @@ def _get_chunked_prefill_embedding(
def _get_multimodal_mask(
input_ids: torch.Tensor, placeholder_tensor: torch.Tensor
) -> torch.Tensor:
return torch.isin(input_ids, placeholder_tensor).unsqueeze(-1)
# placeholder_tensor arrives sorted from embed_mm_inputs. torch.isin
# brute-forces only while P < 10 * N**0.145 (P placeholders, N ids); past
# that it sorts with unique(), whose count syncs the host. The sorted
# lookup never syncs.
assert placeholder_tensor.numel() > 0
placeholders = placeholder_tensor.to(input_ids.dtype)
idx = torch.searchsorted(placeholders, input_ids)
idx.clamp_(max=placeholders.numel() - 1)
return (placeholders[idx] == input_ids).unsqueeze(-1)
def _count_mm_tokens_in_extend(
+4 -1
View File
@@ -495,7 +495,10 @@ def embed_mm_inputs(
embedder = getattr(multimodal_model, f"get_{modality_id}_feature", None)
if len(items) != 0:
assert embedder is not None, f"no embedding method found for {modality}"
pad_values = [value for item in items for value in item.padding_values()]
# Sorted for the searchsorted lookup in _get_multimodal_mask.
pad_values = sorted(
value for item in items for value in item.padding_values()
)
if input_ids.device.type == "cuda":
# Pinned staging keeps the placeholder copy asynchronous on CUDA.
placeholder_cpu = torch.tensor(
+36 -36
View File
@@ -15,6 +15,7 @@ from sglang.srt.runtime_context import (
mamba_track_grid,
)
from sglang.srt.utils.common import (
async_h2d,
ceil_align,
flatten_arrays_to_pinned_cpu,
is_hip,
@@ -996,6 +997,8 @@ class ReqKvInfo:
# Mamba state: an independent resource; whether it is held is `holds_mamba`.
mamba_pool_idx: Optional[torch.Tensor] = None # shape (1)
mamba_ping_pong_track_buffer: Optional[torch.Tensor] = None # shape (2)
# Per entry of the ping-pong buffer: True where it holds no slot (-1).
mamba_ping_pong_empty: Optional[list[bool]] = None
mamba_next_track_idx: Optional[int] = None # 0 or 1
mamba_last_track_idx: Optional[int] = None # 0 or 1
# Seq len of the last cached mamba state
@@ -2091,6 +2094,7 @@ class Req(ReqDllmMixin):
self.inflight_middle_chunks = 0
self.kv.mamba_pool_idx = None
self.kv.mamba_ping_pong_track_buffer = None
self.kv.mamba_ping_pong_empty = None
self.kv.mamba_next_track_idx = None
self.kv.mamba_last_track_idx = None
self.kv.mamba_last_track_seqlen = None
@@ -2286,10 +2290,19 @@ class Req(ReqDllmMixin):
class _MambaRadixCacheV2TrackEntry(NamedTuple):
track_mask: bool
track_index: int
track_position: int
track_seqlen: int
def _gather_mamba_track_indices(
ping_pong_mapping: torch.Tensor,
req_pool_indices: torch.Tensor,
track_positions: torch.Tensor,
) -> torch.Tensor:
"""Each request's ping-pong slot id at its track position, read on the device."""
return ping_pong_mapping[req_pool_indices, track_positions]
def mamba_lazy_spec_in_window(
req, mamba_track_interval: int, max_draft_tokens: int
) -> bool:
@@ -2311,10 +2324,7 @@ def set_mamba_track_indices_from_reqs(
track_positions: optional per-req ping-pong position override (the lazy
spec track plan, see mamba_lazy_spec_prepare).
"""
req_to_token_pool = batch.req_to_token_pool
all_buffers = req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping[
batch.req_pool_indices
] # (bs, ping_pong_size), int64, on device
mapping = batch.req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping
if track_positions is None:
# Guard: mamba_next_track_idx may be None for requests that haven't
# gone through _alloc_ping_pong_buffer yet (e.g., spec v2 verify path).
@@ -2332,17 +2342,9 @@ def set_mamba_track_indices_from_reqs(
i for i, req in enumerate(batch.reqs) if req.kv.mamba_next_track_idx is None
]
batch.mamba_track_buffer_indices = list(track_positions)
idx = (
torch.tensor(
track_positions,
dtype=torch.int64,
pin_memory=True,
)
.unsqueeze(1)
.to(device=all_buffers.device, non_blocking=True)
)
batch.mamba_track_indices = (
torch.gather(all_buffers, 1, idx).squeeze(1).to(torch.int64)
positions = async_h2d(track_positions, dtype=torch.int64, device=mapping.device)
batch.mamba_track_indices = _gather_mamba_track_indices(
mapping, batch.req_pool_indices, positions
)
if freed_rows:
# Overlap can leave a request in TARGET_VERIFY after its Mamba state has
@@ -2918,7 +2920,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
extend_input_logprob_token_ids = []
multimodal_inputs = []
mamba_track_mask_cpu = []
mamba_track_indices_cpu = []
mamba_track_positions_cpu = []
mamba_track_seqlens_cpu = []
for i, (req, seq_len, pre_len) in enumerate(zip(reqs, seq_lens, prefix_lens)):
@@ -2991,7 +2993,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
if get_exec().mamba.enable_mamba_extra_buffer:
track_entry = self._mamba_radix_cache_v2_req_prepare_for_extend(req)
mamba_track_mask_cpu.append(track_entry.track_mask)
mamba_track_indices_cpu.append(track_entry.track_index)
mamba_track_positions_cpu.append(track_entry.track_position)
mamba_track_seqlens_cpu.append(track_entry.track_seqlen)
if self.return_logprob:
@@ -3098,21 +3100,21 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
if get_exec().mamba.enable_mamba_extra_buffer:
self.mamba_prefill_track_mask_cpu = mamba_track_mask_cpu
self.mamba_track_seqlens_cpu = mamba_track_seqlens_cpu
self.mamba_track_indices = torch.tensor(
mamba_track_indices_cpu,
dtype=torch.int64,
device=self.device,
# Ping-pong slot ids live on the device; gather them there.
track_positions = torch.tensor(
mamba_track_positions_cpu, dtype=torch.int64, pin_memory=_pin
).to(self.device, non_blocking=True)
self.mamba_track_indices = _gather_mamba_track_indices(
self.req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping,
self.req_pool_indices,
track_positions,
)
self.mamba_track_mask = torch.tensor(
mamba_track_mask_cpu,
dtype=torch.bool,
device=self.device,
)
mamba_track_mask_cpu, dtype=torch.bool, pin_memory=_pin
).to(self.device, non_blocking=True)
self.mamba_track_seqlens = torch.tensor(
mamba_track_seqlens_cpu,
dtype=torch.int64,
device=self.device,
)
mamba_track_seqlens_cpu, dtype=torch.int64, pin_memory=_pin
).to(self.device, non_blocking=True)
# Collect mamba init info for deferred ops on forward stream
if any(req.kv.holds_mamba for req in reqs):
@@ -3174,9 +3176,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
mamba_track_seqlen_aligned = (
prefix_len + (req.extend_len // checkpoint_grid) * checkpoint_grid
)
track_index = req.kv.mamba_ping_pong_track_buffer[
req.kv.mamba_next_track_idx
].item()
track_position = req.kv.mamba_next_track_idx
mamba_track_seqlen = -1
if mask:
# mamba_track_seqlen is used to calculate the indices to track in
@@ -3228,7 +3228,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
return _MambaRadixCacheV2TrackEntry(
track_mask=mask,
track_index=track_index,
track_position=track_position,
track_seqlen=mamba_track_seqlen,
)
@@ -3627,7 +3627,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
if self.seq_lens_cpu[i].item() % mamba_track_interval != 0:
continue
other_idx = 1 - req.kv.mamba_next_track_idx
if buf[other_idx].item() != -1:
if not req.kv.mamba_ping_pong_empty[other_idx]:
# With overlap the previous forward's post-processing
# (which frees this slot) hasn't run yet. Skip.
continue
@@ -3688,7 +3688,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
track_positions.append(req.kv.mamba_next_track_idx)
continue
other_idx = 1 - req.kv.mamba_next_track_idx
has_pending = buf[other_idx].item() != -1
has_pending = not req.kv.mamba_ping_pong_empty[other_idx]
if not has_pending:
if envs.SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL.get():
new_slot = None
@@ -1590,9 +1590,9 @@ class SchedulerBatchResultProcessor:
other_idx = 1 - req.kv.mamba_next_track_idx
# Recompute the in-flight verify's plan (kv_committed_len is
# frozen since its prepare, so the recompute is exact).
keep_may_be_written_in_flight = req.kv.mamba_ping_pong_track_buffer[
keep_may_be_written_in_flight = req.kv.mamba_ping_pong_empty[
other_idx
].item() == -1 and mamba_lazy_spec_in_window(
] and mamba_lazy_spec_in_window(
req,
mamba_track_grid(self.tree_cache.page_size),
max_speculative_num_draft_tokens(),
@@ -1677,9 +1677,8 @@ class SchedulerBatchResultProcessor:
req.kv.mamba_last_track_idx = track_idx
req.kv.mamba_next_track_idx = track_idx
other_idx = 1 - track_idx
other_val = req.kv.mamba_ping_pong_track_buffer[other_idx].item()
if other_val != -1:
pool = batch.req_to_token_pool
pool = batch.req_to_token_pool
if not req.kv.mamba_ping_pong_empty[other_idx]:
pool.mamba_allocator.free(
req.kv.mamba_ping_pong_track_buffer[other_idx].unsqueeze(0)
)
@@ -45,7 +45,7 @@ def kv_private_swa_tokens(kv: ReqKvInfo, page_size: int) -> int:
def kv_mamba_slots(kv: ReqKvInfo) -> int:
total = kv.mamba_pool_idx.numel() if kv.holds_mamba else 0
if kv.mamba_ping_pong_track_buffer is not None:
total += int((kv.mamba_ping_pong_track_buffer != -1).sum().item())
total += kv.mamba_ping_pong_empty.count(False)
return total
+45 -70
View File
@@ -95,6 +95,7 @@ from sglang.srt.utils import (
from sglang.srt.utils.async_probe import (
maybe_detect_oob,
)
from sglang.srt.utils.common import async_h2d
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
if TYPE_CHECKING:
@@ -1146,33 +1147,17 @@ class MambaPool:
"""Zero out mamba state at the given pool indices. Must run on forward stream."""
for sibling in self._slot_siblings:
sibling.reset_slots(indices)
idx64 = indices.to(torch.int64)
if self._should_fuse_slot_ops():
from sglang.srt.mem_cache.mamba_slot_fused import fused_clear_conv_slots
fused_clear_conv_slots(self._conv_slot_desc, indices)
temporal = self.mamba_cache.temporal
if temporal.numel() > 0:
temporal[:, indices] = 0
return
if not _is_npu:
need_size = len(indices)
for i in range(len(self.mamba_cache.conv)):
t = self.mamba_cache.conv[i]
z = torch.zeros(1, dtype=t.dtype, device=t.device).expand(
t.shape[0], need_size, *t.shape[2:]
)
t[:, indices] = z
t = self.mamba_cache.temporal
z = torch.zeros(1, dtype=t.dtype, device=t.device).expand(
t.shape[0], need_size, *t.shape[2:]
)
t[:, indices] = z
fused_clear_conv_slots(self._conv_slot_desc, idx64)
else:
for i in range(len(self.mamba_cache.conv)):
t = self.mamba_cache.conv[i]
t[:, indices] = 0
t = self.mamba_cache.temporal
t[:, indices] = 0
for conv in self.mamba_cache.conv:
conv.index_fill_(1, idx64, 0)
temporal = self.mamba_cache.temporal
if temporal.numel() > 0:
temporal.index_fill_(1, idx64, 0)
def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor):
"""Clone mamba state (conv + temporal) from src slots into dst slots.
@@ -1589,15 +1574,18 @@ class HybridReqToTokenPool(ReqToTokenPool):
if select_index is None:
return None
spec_write_pos = getattr(self.mamba_pool, "replayssm_spec_write_pos", None)
spec_write_pos = self.mamba_pool.replayssm_spec_write_pos
if spec_write_pos is not None:
fresh_indices = [
idx for idx, fresh in zip(select_index, fresh_req_rows) if fresh
]
if fresh_indices:
spec_write_pos[fresh_indices] = 0
self.mamba_pool.replayssm_cache_base[fresh_indices] = 0
self.mamba_pool.replayssm_is_flush[fresh_indices] = 0
fresh_rows = async_h2d(
fresh_indices, dtype=torch.int64, device=self.device
)
spec_write_pos.index_fill_(0, fresh_rows, 0)
self.mamba_pool.replayssm_cache_base.index_fill_(0, fresh_rows, 0)
self.mamba_pool.replayssm_is_flush.index_fill_(0, fresh_rows, 0)
mamba_indices: list[torch.Tensor] = []
mamba_ping_pong_track_buffers: list[torch.Tensor] = []
@@ -1631,11 +1619,13 @@ class HybridReqToTokenPool(ReqToTokenPool):
assert len(select_index) == len(mamba_ping_pong_track_buffers), (
"Not enough space for mamba ping pong idx, try to increase --mamba-full-memory-ratio."
)
# Device rows: indexing a device tensor with a Python list syncs the host.
rows = async_h2d(select_index, dtype=torch.int64, device=self.device)
mamba_index_tensor = torch.stack(mamba_indices).to(dtype=torch.int32)
self.req_index_to_mamba_index_mapping[select_index] = mamba_index_tensor
self.req_index_to_mamba_index_mapping[rows] = mamba_index_tensor
if self.enable_mamba_extra_buffer:
ping_pong_tensor = torch.stack(mamba_ping_pong_track_buffers)
self.req_index_to_mamba_ping_pong_track_buffer_mapping[select_index] = (
self.req_index_to_mamba_ping_pong_track_buffer_mapping[rows] = (
ping_pong_tensor
)
return select_index
@@ -1779,6 +1769,9 @@ class HybridReqToTokenPool(ReqToTokenPool):
)
buf[:n] = slots
req.kv.mamba_ping_pong_track_buffer = buf
req.kv.mamba_ping_pong_empty = [False] * n + [True] * (
self.mamba_ping_pong_track_buffer_size - n
)
req.kv.mamba_prev_track_seqlen = None
req.kv.mamba_next_track_idx = 0
req.kv.mamba_last_track_idx = (
@@ -1794,10 +1787,17 @@ class HybridReqToTokenPool(ReqToTokenPool):
req_index_to_mamba_ping_pong_track_buffer_mapping in sync so that
set_mamba_track_indices_from_reqs reads correct slot indices.
"""
req.kv.mamba_ping_pong_track_buffer[idx] = value
if torch.is_tensor(value):
req.kv.mamba_ping_pong_track_buffer[idx] = value
else:
assert value == -1, f"expected a slot tensor or -1, got {value!r}"
# fill_ stays on the device; assigning the int is a blocking host copy.
req.kv.mamba_ping_pong_track_buffer[idx].fill_(-1)
self.req_index_to_mamba_ping_pong_track_buffer_mapping[req.kv.req_pool_idx] = (
req.kv.mamba_ping_pong_track_buffer
)
# value is polymorphic: a slot tensor fills the entry, the int -1 empties it.
req.kv.mamba_ping_pong_empty[idx] = not torch.is_tensor(value)
def donate_mamba_ping_pong_slot(
self, req: Req, new_slot: torch.Tensor
@@ -1831,52 +1831,27 @@ class HybridReqToTokenPool(ReqToTokenPool):
req.kv.mamba_pool_idx = None
if self.enable_mamba_extra_buffer:
mamba_ping_pong_track_buffer_to_free = (
self.req_index_to_mamba_ping_pong_track_buffer_mapping[
req.kv.req_pool_idx
]
size = self.mamba_ping_pong_track_buffer_size
keep = mamba_ping_pong_track_buffer_to_keep
assert keep in (None, 0, 1) and (keep != 1 or size == 2), (
f"cannot keep ping-pong entry {keep} of {size}"
)
if mamba_ping_pong_track_buffer_to_keep is not None:
assert mamba_ping_pong_track_buffer_to_keep in [
0,
1,
], (
f"mamba_ping_pong_track_buffer_to_keep must be 0 or 1, {mamba_ping_pong_track_buffer_to_keep=}"
)
# Avoid Python-list advanced indexing on a device tensor.
# The ping-pong buffer size is either 2 (normal) or 1 (spec decode).
if self.mamba_ping_pong_track_buffer_size == 2:
idx_to_free = 1 - mamba_ping_pong_track_buffer_to_keep
mamba_ping_pong_track_buffer_to_free = (
mamba_ping_pong_track_buffer_to_free[
idx_to_free : idx_to_free + 1
]
)
else:
assert self.mamba_ping_pong_track_buffer_size == 1, (
f"Unexpected mamba_ping_pong_track_buffer_size="
f"{self.mamba_ping_pong_track_buffer_size}"
)
assert mamba_ping_pong_track_buffer_to_keep == 0, (
"mamba_ping_pong_track_buffer_to_keep must be 0 when "
"mamba_ping_pong_track_buffer_size is 1"
)
# Keep the only slot, so free nothing.
mamba_ping_pong_track_buffer_to_free = (
mamba_ping_pong_track_buffer_to_free[0:0]
)
if self.enable_mamba_extra_buffer_lazy:
mamba_ping_pong_track_buffer_to_free = (
mamba_ping_pong_track_buffer_to_free[
mamba_ping_pong_track_buffer_to_free != -1
]
)
self.mamba_allocator.free(mamba_ping_pong_track_buffer_to_free)
empty = req.kv.mamba_ping_pong_empty
positions = [p for p in range(size) if p != keep and not empty[p]]
row = self.req_index_to_mamba_ping_pong_track_buffer_mapping[
req.kv.req_pool_idx
]
if len(positions) == size:
self.mamba_allocator.free(row)
else:
for p in positions:
self.mamba_allocator.free(row[p : p + 1])
# Match the req.kv.mamba_pool_idx=None clear above so the next
# alloc() doesn't see a stale ping-pong reference on the req
# and skip allocation (which would silently reuse a freed
# tensor on the req side while the new pool slot leaks).
req.kv.mamba_ping_pong_track_buffer = None
req.kv.mamba_ping_pong_empty = None
req.kv.mamba_next_track_idx = None
req.kv.mamba_last_track_idx = None
req.kv.mamba_last_track_seqlen = None
@@ -1115,6 +1115,7 @@ class UnifiedMambaPool(MambaPool):
self.replayssm_is_kda = False
self.enable_linear_replayssm_spec = False
self.replayssm_spec_fold = False
self.replayssm_spec_write_pos = None
self.replayssm_cache_base = None
self.replayssm_is_flush = None
self.debug_memory_pool = False
@@ -255,9 +255,12 @@ class StreamingSession:
mamba_allocator.free(slot.kv.mamba_pool_idx.unsqueeze(0))
slot.kv.mamba_pool_idx = None
if slot.kv.mamba_ping_pong_track_buffer is not None:
indices = slot.kv.mamba_ping_pong_track_buffer
mamba_allocator.free(indices[indices != -1])
buffer = slot.kv.mamba_ping_pong_track_buffer
for position, empty in enumerate(slot.kv.mamba_ping_pong_empty):
if not empty:
mamba_allocator.free(buffer[position : position + 1])
slot.kv.mamba_ping_pong_track_buffer = None
slot.kv.mamba_ping_pong_empty = None
# -- Internal helpers (streaming body bits) --
@@ -482,6 +482,34 @@ def test_precomputed_audio_slices_concatenate_once(monkeypatch, prefix, length,
concat.assert_called_once()
@pytest.mark.parametrize(
"n_ids,n_placeholders,ids_dtype",
[
(1, 1, torch.int64),
(64, 3, torch.int64),
(4096, 800, torch.int64),
(16384, 800, torch.int32),
],
)
def test_multimodal_mask_matches_broadcast_compare(n_ids, n_placeholders, ids_dtype):
gen = torch.Generator().manual_seed(n_ids)
pad_values = torch.randperm(4 * n_placeholders + 8, generator=gen)[
:n_placeholders
].tolist()
# Host-sorted placeholders with a repeat, as embed_mm_inputs builds them; ids
# below, between and above them.
placeholders = torch.tensor(sorted(pad_values + pad_values[:1]))
input_ids = torch.randint(-2, 4 * n_placeholders + 10, (n_ids,), generator=gen).to(
ids_dtype
)
mask = mm_schedule._get_multimodal_mask(input_ids, placeholders)
broadcast = (input_ids.unsqueeze(-1) == placeholders).any(-1).unsqueeze(-1)
assert torch.equal(mask, broadcast)
assert torch.equal(mask, torch.isin(input_ids, placeholders).unsqueeze(-1))
if __name__ == "__main__":
import sys
@@ -27,20 +27,6 @@ register_cpu_ci(est_time=12, suite="base-a-test-cpu")
TRACK_INTERVAL = 4
def _set_mamba_track_indices_on_cpu(batch, track_positions=None):
real_tensor = torch.tensor
def tensor_without_pinning(*args, **kwargs):
kwargs["pin_memory"] = False
return real_tensor(*args, **kwargs)
with patch(
"sglang.srt.managers.schedule_batch.torch.tensor",
side_effect=tensor_without_pinning,
):
set_mamba_track_indices_from_reqs(batch, track_positions)
def _make_batch() -> tuple[Req, ScheduleBatch]:
sampling_params = SamplingParams(max_new_tokens=32)
sampling_params.normalize(None)
@@ -202,6 +188,7 @@ class TestMambaPrevTrackSeqlen(unittest.TestCase):
req, batch = _make_batch()
processor = _make_processor()
req.kv.mamba_ping_pong_track_buffer = torch.tensor([10, 11])
req.kv.mamba_ping_pong_empty = [False, False]
req.kv.mamba_next_track_idx = 0
req.kv.mamba_last_track_idx = 1
batch.mamba_track_buffer_indices = None
@@ -242,7 +229,7 @@ class TestMambaFreedTrackIndicesCPU(unittest.TestCase):
req_index_to_mamba_ping_pong_track_buffer_mapping=mapping
),
)
_set_mamba_track_indices_on_cpu(batch, positions)
set_mamba_track_indices_from_reqs(batch, positions)
self.assertEqual(batch.mamba_track_indices.tolist(), expected)
@@ -14,9 +14,11 @@ from array import array
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import torch
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
from sglang.srt.managers.schedule_batch import (
Req,
ScheduleBatch,
_MambaRadixCacheV2TrackEntry,
)
from sglang.srt.runtime_context import get_context, mamba_track_grid
from sglang.srt.sampling.sampling_params import SamplingParams
from sglang.srt.server_args import (
@@ -30,15 +32,15 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu")
CHUNK = 64
def _track_seqlen(
def _prepare_extend(
*,
tree_page: int,
prefix_len: int,
extend_len: int,
dcp_enabled: bool = False,
) -> int | None:
"""Run one extend through the tracker and report the donated depth, or
None when the extend donates no checkpoint."""
next_idx: int = 0,
) -> tuple[_MambaRadixCacheV2TrackEntry, Req]:
"""Run one extend through the tracker; return its track entry and the request."""
server_args = ServerArgs(model_path="dummy", page_size=CHUNK)
# The property would otherwise load the HF config for the dummy model.
server_args._mamba_cache_chunk_size = CHUNK
@@ -55,8 +57,7 @@ def _track_seqlen(
)
req.prefix_len = prefix_len
req.extend_end = prefix_len + extend_len
req.kv.mamba_ping_pong_track_buffer = torch.tensor([0, 1], dtype=torch.int64)
req.kv.mamba_next_track_idx = 0
req.kv.mamba_next_track_idx = next_idx
req.mamba_branching_seqlen = None
batch = ScheduleBatch(reqs=[req])
@@ -65,13 +66,19 @@ def _track_seqlen(
)
batch.tree_cache = SimpleNamespace(page_size=tree_page)
batch.req_to_token_pool = MagicMock()
batch.req_to_token_pool.get_mamba_ping_pong_other_idx.return_value = 1
batch.req_to_token_pool.get_mamba_ping_pong_other_idx.return_value = 1 - next_idx
with patch(
"sglang.srt.managers.schedule_batch.get_parallel",
return_value=SimpleNamespace(dcp_enabled=dcp_enabled),
):
entry = batch._mamba_radix_cache_v2_req_prepare_for_extend(req)
return entry, req
def _track_seqlen(**kwargs) -> int | None:
"""The donated depth of one extend, or None when it donates no checkpoint."""
entry, req = _prepare_extend(**kwargs)
if not entry.track_mask:
assert req.kv.mamba_last_track_seqlen is None
return None
@@ -124,6 +131,20 @@ class TestMambaCheckpointDepth(unittest.TestCase):
)
class TestMambaTrackPosition(unittest.TestCase):
def test_track_position_is_taken_before_the_ping_pong_swap(self):
# prepare_for_extend gathers the slot id at this position; the read it
# replaced took buffer[mamba_next_track_idx] before the boundary swap.
for next_idx in (0, 1):
with self.subTest(next_idx=next_idx):
entry, req = _prepare_extend(
tree_page=CHUNK, prefix_len=0, extend_len=96, next_idx=next_idx
)
self.assertTrue(entry.track_mask)
self.assertEqual(entry.track_position, next_idx)
self.assertEqual(req.kv.mamba_next_track_idx, 1 - next_idx)
class TestMambaTrackGrid(unittest.TestCase):
def _grid(self, *, interval: int, tree_page: int, chunk: int = CHUNK) -> int:
with get_context().override_server_args(
@@ -1027,6 +1027,7 @@ class TestTriFactorySizing(unittest.TestCase):
req_pool_idx=1,
mamba_pool_idx=live[0],
mamba_ping_pong_track_buffer=track,
mamba_ping_pong_empty=[index == -1 for index in ids],
)
)
session = StreamingSession(SimpleNamespace(req_to_token_pool=pool))
@@ -1189,6 +1190,52 @@ class TestTriFactorySizing(unittest.TestCase):
allocator.free(retained)
self.assertEqual(allocator.available_size(), initial_available)
def test_ping_pong_mapping_mirrors_the_request_buffers(self):
from types import SimpleNamespace
from sglang.srt.managers.schedule_batch import (
ReqKvInfo,
_gather_mamba_track_indices,
)
from sglang.srt.runtime_context import get_parallel
for lazy in (False, True):
with self.subTest(lazy=lazy):
kw = self._factory_kwargs(
enable_mamba_extra_buffer=True,
enable_mamba_extra_buffer_lazy=lazy,
disable_overlap_schedule=False,
max_mamba_cache_size=16,
)
with get_parallel().override(attn_dcp_size=1):
pool = init_unified_mamba_swa_pools(**kw).req_to_token_pool
mapping = pool.req_index_to_mamba_ping_pong_track_buffer_mapping
allocator = pool.mamba_allocator
reqs = [SimpleNamespace(kv=ReqKvInfo()) for _ in range(3)]
rows = pool.alloc(reqs)
def buffers():
return [
req.kv.mamba_ping_pong_track_buffer.tolist() for req in reqs
]
self.assertEqual(mapping[rows].tolist(), buffers())
# Every writer of a buffer entry rewrites the request's row.
pool.donate_mamba_ping_pong_slot(reqs[0], allocator.alloc(1))
pool.set_mamba_ping_pong_slot(reqs[1], 0, -1)
pool.set_mamba_ping_pong_slot(reqs[1], 1, allocator.alloc(1)[0])
self.assertEqual(mapping[rows].tolist(), buffers())
# The device gather returns what the per-request .item() read did.
positions = [0, 0, 1]
gathered = _gather_mamba_track_indices(
mapping, torch.tensor(rows), torch.tensor(positions)
)
item_reads = [
req.kv.mamba_ping_pong_track_buffer[pos].item()
for req, pos in zip(reqs, positions)
]
self.assertEqual(gathered.tolist(), item_reads)
def test_budget_sizing_and_boot_signature(self):
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
UnifiedSWAAllocatorBase,