[Perf] scheduler: Plan hybrid Mamba extend batches without host-device syncs (#45472)
Co-authored-by: Lu Fang <116399278+luccafong@users.noreply.github.com> Co-authored-by: Shiyan Deng <20773414+842974287@users.noreply.github.com>
This commit is contained in:
co-authored by
Lu Fang
Shiyan Deng
parent
5e79333c95
commit
740d5f9f9e
@@ -286,6 +286,7 @@ class MlxAuxiliaryStateReqToTokenPool(ReqToTokenPool):
|
||||
if mamba_ping_pong_track_buffer_to_keep is None:
|
||||
self.auxiliary_state_pool.free(track_buffer)
|
||||
req.kv.mamba_ping_pong_track_buffer = None
|
||||
req.kv.mamba_ping_pong_empty = None
|
||||
req.kv.mamba_next_track_idx = None
|
||||
req.kv.mamba_last_track_idx = None
|
||||
|
||||
@@ -387,6 +388,7 @@ class MlxAuxiliaryStateComponent(MambaComponent):
|
||||
if track_buffer is not None:
|
||||
self.cache.req_to_token_pool.auxiliary_state_pool.free(track_buffer)
|
||||
req.kv.mamba_ping_pong_track_buffer = None
|
||||
req.kv.mamba_ping_pong_empty = None
|
||||
req.kv.mamba_next_track_idx = None
|
||||
req.kv.mamba_last_track_idx = None
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
@@ -412,6 +414,7 @@ class MlxAuxiliaryStateComponent(MambaComponent):
|
||||
if track_buffer is not None:
|
||||
self.cache.req_to_token_pool.auxiliary_state_pool.free(track_buffer)
|
||||
req.kv.mamba_ping_pong_track_buffer = None
|
||||
req.kv.mamba_ping_pong_empty = None
|
||||
req.kv.mamba_next_track_idx = None
|
||||
req.kv.mamba_last_track_idx = None
|
||||
req.kv.mamba_pool_idx = None
|
||||
|
||||
@@ -417,6 +417,7 @@ class MlxModelRunner:
|
||||
)
|
||||
return
|
||||
req.kv.mamba_ping_pong_track_buffer = track_buffer
|
||||
req.kv.mamba_ping_pong_empty = [False]
|
||||
req.kv.mamba_next_track_idx = 0
|
||||
req.kv.mamba_last_track_idx = 0
|
||||
|
||||
|
||||
@@ -655,7 +655,15 @@ def _get_chunked_prefill_embedding(
|
||||
def _get_multimodal_mask(
|
||||
input_ids: torch.Tensor, placeholder_tensor: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return torch.isin(input_ids, placeholder_tensor).unsqueeze(-1)
|
||||
# placeholder_tensor arrives sorted from embed_mm_inputs. torch.isin
|
||||
# brute-forces only while P < 10 * N**0.145 (P placeholders, N ids); past
|
||||
# that it sorts with unique(), whose count syncs the host. The sorted
|
||||
# lookup never syncs.
|
||||
assert placeholder_tensor.numel() > 0
|
||||
placeholders = placeholder_tensor.to(input_ids.dtype)
|
||||
idx = torch.searchsorted(placeholders, input_ids)
|
||||
idx.clamp_(max=placeholders.numel() - 1)
|
||||
return (placeholders[idx] == input_ids).unsqueeze(-1)
|
||||
|
||||
|
||||
def _count_mm_tokens_in_extend(
|
||||
|
||||
@@ -495,7 +495,10 @@ def embed_mm_inputs(
|
||||
embedder = getattr(multimodal_model, f"get_{modality_id}_feature", None)
|
||||
if len(items) != 0:
|
||||
assert embedder is not None, f"no embedding method found for {modality}"
|
||||
pad_values = [value for item in items for value in item.padding_values()]
|
||||
# Sorted for the searchsorted lookup in _get_multimodal_mask.
|
||||
pad_values = sorted(
|
||||
value for item in items for value in item.padding_values()
|
||||
)
|
||||
if input_ids.device.type == "cuda":
|
||||
# Pinned staging keeps the placeholder copy asynchronous on CUDA.
|
||||
placeholder_cpu = torch.tensor(
|
||||
|
||||
@@ -15,6 +15,7 @@ from sglang.srt.runtime_context import (
|
||||
mamba_track_grid,
|
||||
)
|
||||
from sglang.srt.utils.common import (
|
||||
async_h2d,
|
||||
ceil_align,
|
||||
flatten_arrays_to_pinned_cpu,
|
||||
is_hip,
|
||||
@@ -996,6 +997,8 @@ class ReqKvInfo:
|
||||
# Mamba state: an independent resource; whether it is held is `holds_mamba`.
|
||||
mamba_pool_idx: Optional[torch.Tensor] = None # shape (1)
|
||||
mamba_ping_pong_track_buffer: Optional[torch.Tensor] = None # shape (2)
|
||||
# Per entry of the ping-pong buffer: True where it holds no slot (-1).
|
||||
mamba_ping_pong_empty: Optional[list[bool]] = None
|
||||
mamba_next_track_idx: Optional[int] = None # 0 or 1
|
||||
mamba_last_track_idx: Optional[int] = None # 0 or 1
|
||||
# Seq len of the last cached mamba state
|
||||
@@ -2091,6 +2094,7 @@ class Req(ReqDllmMixin):
|
||||
self.inflight_middle_chunks = 0
|
||||
self.kv.mamba_pool_idx = None
|
||||
self.kv.mamba_ping_pong_track_buffer = None
|
||||
self.kv.mamba_ping_pong_empty = None
|
||||
self.kv.mamba_next_track_idx = None
|
||||
self.kv.mamba_last_track_idx = None
|
||||
self.kv.mamba_last_track_seqlen = None
|
||||
@@ -2286,10 +2290,19 @@ class Req(ReqDllmMixin):
|
||||
|
||||
class _MambaRadixCacheV2TrackEntry(NamedTuple):
|
||||
track_mask: bool
|
||||
track_index: int
|
||||
track_position: int
|
||||
track_seqlen: int
|
||||
|
||||
|
||||
def _gather_mamba_track_indices(
|
||||
ping_pong_mapping: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
track_positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Each request's ping-pong slot id at its track position, read on the device."""
|
||||
return ping_pong_mapping[req_pool_indices, track_positions]
|
||||
|
||||
|
||||
def mamba_lazy_spec_in_window(
|
||||
req, mamba_track_interval: int, max_draft_tokens: int
|
||||
) -> bool:
|
||||
@@ -2311,10 +2324,7 @@ def set_mamba_track_indices_from_reqs(
|
||||
track_positions: optional per-req ping-pong position override (the lazy
|
||||
spec track plan, see mamba_lazy_spec_prepare).
|
||||
"""
|
||||
req_to_token_pool = batch.req_to_token_pool
|
||||
all_buffers = req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping[
|
||||
batch.req_pool_indices
|
||||
] # (bs, ping_pong_size), int64, on device
|
||||
mapping = batch.req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping
|
||||
if track_positions is None:
|
||||
# Guard: mamba_next_track_idx may be None for requests that haven't
|
||||
# gone through _alloc_ping_pong_buffer yet (e.g., spec v2 verify path).
|
||||
@@ -2332,17 +2342,9 @@ def set_mamba_track_indices_from_reqs(
|
||||
i for i, req in enumerate(batch.reqs) if req.kv.mamba_next_track_idx is None
|
||||
]
|
||||
batch.mamba_track_buffer_indices = list(track_positions)
|
||||
idx = (
|
||||
torch.tensor(
|
||||
track_positions,
|
||||
dtype=torch.int64,
|
||||
pin_memory=True,
|
||||
)
|
||||
.unsqueeze(1)
|
||||
.to(device=all_buffers.device, non_blocking=True)
|
||||
)
|
||||
batch.mamba_track_indices = (
|
||||
torch.gather(all_buffers, 1, idx).squeeze(1).to(torch.int64)
|
||||
positions = async_h2d(track_positions, dtype=torch.int64, device=mapping.device)
|
||||
batch.mamba_track_indices = _gather_mamba_track_indices(
|
||||
mapping, batch.req_pool_indices, positions
|
||||
)
|
||||
if freed_rows:
|
||||
# Overlap can leave a request in TARGET_VERIFY after its Mamba state has
|
||||
@@ -2918,7 +2920,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
extend_input_logprob_token_ids = []
|
||||
multimodal_inputs = []
|
||||
mamba_track_mask_cpu = []
|
||||
mamba_track_indices_cpu = []
|
||||
mamba_track_positions_cpu = []
|
||||
mamba_track_seqlens_cpu = []
|
||||
|
||||
for i, (req, seq_len, pre_len) in enumerate(zip(reqs, seq_lens, prefix_lens)):
|
||||
@@ -2991,7 +2993,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
if get_exec().mamba.enable_mamba_extra_buffer:
|
||||
track_entry = self._mamba_radix_cache_v2_req_prepare_for_extend(req)
|
||||
mamba_track_mask_cpu.append(track_entry.track_mask)
|
||||
mamba_track_indices_cpu.append(track_entry.track_index)
|
||||
mamba_track_positions_cpu.append(track_entry.track_position)
|
||||
mamba_track_seqlens_cpu.append(track_entry.track_seqlen)
|
||||
|
||||
if self.return_logprob:
|
||||
@@ -3098,21 +3100,21 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
if get_exec().mamba.enable_mamba_extra_buffer:
|
||||
self.mamba_prefill_track_mask_cpu = mamba_track_mask_cpu
|
||||
self.mamba_track_seqlens_cpu = mamba_track_seqlens_cpu
|
||||
self.mamba_track_indices = torch.tensor(
|
||||
mamba_track_indices_cpu,
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
# Ping-pong slot ids live on the device; gather them there.
|
||||
track_positions = torch.tensor(
|
||||
mamba_track_positions_cpu, dtype=torch.int64, pin_memory=_pin
|
||||
).to(self.device, non_blocking=True)
|
||||
self.mamba_track_indices = _gather_mamba_track_indices(
|
||||
self.req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping,
|
||||
self.req_pool_indices,
|
||||
track_positions,
|
||||
)
|
||||
self.mamba_track_mask = torch.tensor(
|
||||
mamba_track_mask_cpu,
|
||||
dtype=torch.bool,
|
||||
device=self.device,
|
||||
)
|
||||
mamba_track_mask_cpu, dtype=torch.bool, pin_memory=_pin
|
||||
).to(self.device, non_blocking=True)
|
||||
self.mamba_track_seqlens = torch.tensor(
|
||||
mamba_track_seqlens_cpu,
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
mamba_track_seqlens_cpu, dtype=torch.int64, pin_memory=_pin
|
||||
).to(self.device, non_blocking=True)
|
||||
|
||||
# Collect mamba init info for deferred ops on forward stream
|
||||
if any(req.kv.holds_mamba for req in reqs):
|
||||
@@ -3174,9 +3176,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
mamba_track_seqlen_aligned = (
|
||||
prefix_len + (req.extend_len // checkpoint_grid) * checkpoint_grid
|
||||
)
|
||||
track_index = req.kv.mamba_ping_pong_track_buffer[
|
||||
req.kv.mamba_next_track_idx
|
||||
].item()
|
||||
track_position = req.kv.mamba_next_track_idx
|
||||
mamba_track_seqlen = -1
|
||||
if mask:
|
||||
# mamba_track_seqlen is used to calculate the indices to track in
|
||||
@@ -3228,7 +3228,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
|
||||
return _MambaRadixCacheV2TrackEntry(
|
||||
track_mask=mask,
|
||||
track_index=track_index,
|
||||
track_position=track_position,
|
||||
track_seqlen=mamba_track_seqlen,
|
||||
)
|
||||
|
||||
@@ -3627,7 +3627,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
if self.seq_lens_cpu[i].item() % mamba_track_interval != 0:
|
||||
continue
|
||||
other_idx = 1 - req.kv.mamba_next_track_idx
|
||||
if buf[other_idx].item() != -1:
|
||||
if not req.kv.mamba_ping_pong_empty[other_idx]:
|
||||
# With overlap the previous forward's post-processing
|
||||
# (which frees this slot) hasn't run yet. Skip.
|
||||
continue
|
||||
@@ -3688,7 +3688,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
track_positions.append(req.kv.mamba_next_track_idx)
|
||||
continue
|
||||
other_idx = 1 - req.kv.mamba_next_track_idx
|
||||
has_pending = buf[other_idx].item() != -1
|
||||
has_pending = not req.kv.mamba_ping_pong_empty[other_idx]
|
||||
if not has_pending:
|
||||
if envs.SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL.get():
|
||||
new_slot = None
|
||||
|
||||
@@ -1590,9 +1590,9 @@ class SchedulerBatchResultProcessor:
|
||||
other_idx = 1 - req.kv.mamba_next_track_idx
|
||||
# Recompute the in-flight verify's plan (kv_committed_len is
|
||||
# frozen since its prepare, so the recompute is exact).
|
||||
keep_may_be_written_in_flight = req.kv.mamba_ping_pong_track_buffer[
|
||||
keep_may_be_written_in_flight = req.kv.mamba_ping_pong_empty[
|
||||
other_idx
|
||||
].item() == -1 and mamba_lazy_spec_in_window(
|
||||
] and mamba_lazy_spec_in_window(
|
||||
req,
|
||||
mamba_track_grid(self.tree_cache.page_size),
|
||||
max_speculative_num_draft_tokens(),
|
||||
@@ -1677,9 +1677,8 @@ class SchedulerBatchResultProcessor:
|
||||
req.kv.mamba_last_track_idx = track_idx
|
||||
req.kv.mamba_next_track_idx = track_idx
|
||||
other_idx = 1 - track_idx
|
||||
other_val = req.kv.mamba_ping_pong_track_buffer[other_idx].item()
|
||||
if other_val != -1:
|
||||
pool = batch.req_to_token_pool
|
||||
pool = batch.req_to_token_pool
|
||||
if not req.kv.mamba_ping_pong_empty[other_idx]:
|
||||
pool.mamba_allocator.free(
|
||||
req.kv.mamba_ping_pong_track_buffer[other_idx].unsqueeze(0)
|
||||
)
|
||||
|
||||
@@ -45,7 +45,7 @@ def kv_private_swa_tokens(kv: ReqKvInfo, page_size: int) -> int:
|
||||
def kv_mamba_slots(kv: ReqKvInfo) -> int:
|
||||
total = kv.mamba_pool_idx.numel() if kv.holds_mamba else 0
|
||||
if kv.mamba_ping_pong_track_buffer is not None:
|
||||
total += int((kv.mamba_ping_pong_track_buffer != -1).sum().item())
|
||||
total += kv.mamba_ping_pong_empty.count(False)
|
||||
return total
|
||||
|
||||
|
||||
|
||||
@@ -95,6 +95,7 @@ from sglang.srt.utils import (
|
||||
from sglang.srt.utils.async_probe import (
|
||||
maybe_detect_oob,
|
||||
)
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -1146,33 +1147,17 @@ class MambaPool:
|
||||
"""Zero out mamba state at the given pool indices. Must run on forward stream."""
|
||||
for sibling in self._slot_siblings:
|
||||
sibling.reset_slots(indices)
|
||||
idx64 = indices.to(torch.int64)
|
||||
if self._should_fuse_slot_ops():
|
||||
from sglang.srt.mem_cache.mamba_slot_fused import fused_clear_conv_slots
|
||||
|
||||
fused_clear_conv_slots(self._conv_slot_desc, indices)
|
||||
temporal = self.mamba_cache.temporal
|
||||
if temporal.numel() > 0:
|
||||
temporal[:, indices] = 0
|
||||
return
|
||||
if not _is_npu:
|
||||
need_size = len(indices)
|
||||
for i in range(len(self.mamba_cache.conv)):
|
||||
t = self.mamba_cache.conv[i]
|
||||
z = torch.zeros(1, dtype=t.dtype, device=t.device).expand(
|
||||
t.shape[0], need_size, *t.shape[2:]
|
||||
)
|
||||
t[:, indices] = z
|
||||
t = self.mamba_cache.temporal
|
||||
z = torch.zeros(1, dtype=t.dtype, device=t.device).expand(
|
||||
t.shape[0], need_size, *t.shape[2:]
|
||||
)
|
||||
t[:, indices] = z
|
||||
fused_clear_conv_slots(self._conv_slot_desc, idx64)
|
||||
else:
|
||||
for i in range(len(self.mamba_cache.conv)):
|
||||
t = self.mamba_cache.conv[i]
|
||||
t[:, indices] = 0
|
||||
t = self.mamba_cache.temporal
|
||||
t[:, indices] = 0
|
||||
for conv in self.mamba_cache.conv:
|
||||
conv.index_fill_(1, idx64, 0)
|
||||
temporal = self.mamba_cache.temporal
|
||||
if temporal.numel() > 0:
|
||||
temporal.index_fill_(1, idx64, 0)
|
||||
|
||||
def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor):
|
||||
"""Clone mamba state (conv + temporal) from src slots into dst slots.
|
||||
@@ -1589,15 +1574,18 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
if select_index is None:
|
||||
return None
|
||||
|
||||
spec_write_pos = getattr(self.mamba_pool, "replayssm_spec_write_pos", None)
|
||||
spec_write_pos = self.mamba_pool.replayssm_spec_write_pos
|
||||
if spec_write_pos is not None:
|
||||
fresh_indices = [
|
||||
idx for idx, fresh in zip(select_index, fresh_req_rows) if fresh
|
||||
]
|
||||
if fresh_indices:
|
||||
spec_write_pos[fresh_indices] = 0
|
||||
self.mamba_pool.replayssm_cache_base[fresh_indices] = 0
|
||||
self.mamba_pool.replayssm_is_flush[fresh_indices] = 0
|
||||
fresh_rows = async_h2d(
|
||||
fresh_indices, dtype=torch.int64, device=self.device
|
||||
)
|
||||
spec_write_pos.index_fill_(0, fresh_rows, 0)
|
||||
self.mamba_pool.replayssm_cache_base.index_fill_(0, fresh_rows, 0)
|
||||
self.mamba_pool.replayssm_is_flush.index_fill_(0, fresh_rows, 0)
|
||||
|
||||
mamba_indices: list[torch.Tensor] = []
|
||||
mamba_ping_pong_track_buffers: list[torch.Tensor] = []
|
||||
@@ -1631,11 +1619,13 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
assert len(select_index) == len(mamba_ping_pong_track_buffers), (
|
||||
"Not enough space for mamba ping pong idx, try to increase --mamba-full-memory-ratio."
|
||||
)
|
||||
# Device rows: indexing a device tensor with a Python list syncs the host.
|
||||
rows = async_h2d(select_index, dtype=torch.int64, device=self.device)
|
||||
mamba_index_tensor = torch.stack(mamba_indices).to(dtype=torch.int32)
|
||||
self.req_index_to_mamba_index_mapping[select_index] = mamba_index_tensor
|
||||
self.req_index_to_mamba_index_mapping[rows] = mamba_index_tensor
|
||||
if self.enable_mamba_extra_buffer:
|
||||
ping_pong_tensor = torch.stack(mamba_ping_pong_track_buffers)
|
||||
self.req_index_to_mamba_ping_pong_track_buffer_mapping[select_index] = (
|
||||
self.req_index_to_mamba_ping_pong_track_buffer_mapping[rows] = (
|
||||
ping_pong_tensor
|
||||
)
|
||||
return select_index
|
||||
@@ -1779,6 +1769,9 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
)
|
||||
buf[:n] = slots
|
||||
req.kv.mamba_ping_pong_track_buffer = buf
|
||||
req.kv.mamba_ping_pong_empty = [False] * n + [True] * (
|
||||
self.mamba_ping_pong_track_buffer_size - n
|
||||
)
|
||||
req.kv.mamba_prev_track_seqlen = None
|
||||
req.kv.mamba_next_track_idx = 0
|
||||
req.kv.mamba_last_track_idx = (
|
||||
@@ -1794,10 +1787,17 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
req_index_to_mamba_ping_pong_track_buffer_mapping in sync so that
|
||||
set_mamba_track_indices_from_reqs reads correct slot indices.
|
||||
"""
|
||||
req.kv.mamba_ping_pong_track_buffer[idx] = value
|
||||
if torch.is_tensor(value):
|
||||
req.kv.mamba_ping_pong_track_buffer[idx] = value
|
||||
else:
|
||||
assert value == -1, f"expected a slot tensor or -1, got {value!r}"
|
||||
# fill_ stays on the device; assigning the int is a blocking host copy.
|
||||
req.kv.mamba_ping_pong_track_buffer[idx].fill_(-1)
|
||||
self.req_index_to_mamba_ping_pong_track_buffer_mapping[req.kv.req_pool_idx] = (
|
||||
req.kv.mamba_ping_pong_track_buffer
|
||||
)
|
||||
# value is polymorphic: a slot tensor fills the entry, the int -1 empties it.
|
||||
req.kv.mamba_ping_pong_empty[idx] = not torch.is_tensor(value)
|
||||
|
||||
def donate_mamba_ping_pong_slot(
|
||||
self, req: Req, new_slot: torch.Tensor
|
||||
@@ -1831,52 +1831,27 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
req.kv.mamba_pool_idx = None
|
||||
|
||||
if self.enable_mamba_extra_buffer:
|
||||
mamba_ping_pong_track_buffer_to_free = (
|
||||
self.req_index_to_mamba_ping_pong_track_buffer_mapping[
|
||||
req.kv.req_pool_idx
|
||||
]
|
||||
size = self.mamba_ping_pong_track_buffer_size
|
||||
keep = mamba_ping_pong_track_buffer_to_keep
|
||||
assert keep in (None, 0, 1) and (keep != 1 or size == 2), (
|
||||
f"cannot keep ping-pong entry {keep} of {size}"
|
||||
)
|
||||
if mamba_ping_pong_track_buffer_to_keep is not None:
|
||||
assert mamba_ping_pong_track_buffer_to_keep in [
|
||||
0,
|
||||
1,
|
||||
], (
|
||||
f"mamba_ping_pong_track_buffer_to_keep must be 0 or 1, {mamba_ping_pong_track_buffer_to_keep=}"
|
||||
)
|
||||
# Avoid Python-list advanced indexing on a device tensor.
|
||||
# The ping-pong buffer size is either 2 (normal) or 1 (spec decode).
|
||||
if self.mamba_ping_pong_track_buffer_size == 2:
|
||||
idx_to_free = 1 - mamba_ping_pong_track_buffer_to_keep
|
||||
mamba_ping_pong_track_buffer_to_free = (
|
||||
mamba_ping_pong_track_buffer_to_free[
|
||||
idx_to_free : idx_to_free + 1
|
||||
]
|
||||
)
|
||||
else:
|
||||
assert self.mamba_ping_pong_track_buffer_size == 1, (
|
||||
f"Unexpected mamba_ping_pong_track_buffer_size="
|
||||
f"{self.mamba_ping_pong_track_buffer_size}"
|
||||
)
|
||||
assert mamba_ping_pong_track_buffer_to_keep == 0, (
|
||||
"mamba_ping_pong_track_buffer_to_keep must be 0 when "
|
||||
"mamba_ping_pong_track_buffer_size is 1"
|
||||
)
|
||||
# Keep the only slot, so free nothing.
|
||||
mamba_ping_pong_track_buffer_to_free = (
|
||||
mamba_ping_pong_track_buffer_to_free[0:0]
|
||||
)
|
||||
if self.enable_mamba_extra_buffer_lazy:
|
||||
mamba_ping_pong_track_buffer_to_free = (
|
||||
mamba_ping_pong_track_buffer_to_free[
|
||||
mamba_ping_pong_track_buffer_to_free != -1
|
||||
]
|
||||
)
|
||||
self.mamba_allocator.free(mamba_ping_pong_track_buffer_to_free)
|
||||
empty = req.kv.mamba_ping_pong_empty
|
||||
positions = [p for p in range(size) if p != keep and not empty[p]]
|
||||
row = self.req_index_to_mamba_ping_pong_track_buffer_mapping[
|
||||
req.kv.req_pool_idx
|
||||
]
|
||||
if len(positions) == size:
|
||||
self.mamba_allocator.free(row)
|
||||
else:
|
||||
for p in positions:
|
||||
self.mamba_allocator.free(row[p : p + 1])
|
||||
# Match the req.kv.mamba_pool_idx=None clear above so the next
|
||||
# alloc() doesn't see a stale ping-pong reference on the req
|
||||
# and skip allocation (which would silently reuse a freed
|
||||
# tensor on the req side while the new pool slot leaks).
|
||||
req.kv.mamba_ping_pong_track_buffer = None
|
||||
req.kv.mamba_ping_pong_empty = None
|
||||
req.kv.mamba_next_track_idx = None
|
||||
req.kv.mamba_last_track_idx = None
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
|
||||
@@ -1115,6 +1115,7 @@ class UnifiedMambaPool(MambaPool):
|
||||
self.replayssm_is_kda = False
|
||||
self.enable_linear_replayssm_spec = False
|
||||
self.replayssm_spec_fold = False
|
||||
self.replayssm_spec_write_pos = None
|
||||
self.replayssm_cache_base = None
|
||||
self.replayssm_is_flush = None
|
||||
self.debug_memory_pool = False
|
||||
|
||||
@@ -255,9 +255,12 @@ class StreamingSession:
|
||||
mamba_allocator.free(slot.kv.mamba_pool_idx.unsqueeze(0))
|
||||
slot.kv.mamba_pool_idx = None
|
||||
if slot.kv.mamba_ping_pong_track_buffer is not None:
|
||||
indices = slot.kv.mamba_ping_pong_track_buffer
|
||||
mamba_allocator.free(indices[indices != -1])
|
||||
buffer = slot.kv.mamba_ping_pong_track_buffer
|
||||
for position, empty in enumerate(slot.kv.mamba_ping_pong_empty):
|
||||
if not empty:
|
||||
mamba_allocator.free(buffer[position : position + 1])
|
||||
slot.kv.mamba_ping_pong_track_buffer = None
|
||||
slot.kv.mamba_ping_pong_empty = None
|
||||
|
||||
# -- Internal helpers (streaming body bits) --
|
||||
|
||||
|
||||
@@ -482,6 +482,34 @@ def test_precomputed_audio_slices_concatenate_once(monkeypatch, prefix, length,
|
||||
concat.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"n_ids,n_placeholders,ids_dtype",
|
||||
[
|
||||
(1, 1, torch.int64),
|
||||
(64, 3, torch.int64),
|
||||
(4096, 800, torch.int64),
|
||||
(16384, 800, torch.int32),
|
||||
],
|
||||
)
|
||||
def test_multimodal_mask_matches_broadcast_compare(n_ids, n_placeholders, ids_dtype):
|
||||
gen = torch.Generator().manual_seed(n_ids)
|
||||
pad_values = torch.randperm(4 * n_placeholders + 8, generator=gen)[
|
||||
:n_placeholders
|
||||
].tolist()
|
||||
# Host-sorted placeholders with a repeat, as embed_mm_inputs builds them; ids
|
||||
# below, between and above them.
|
||||
placeholders = torch.tensor(sorted(pad_values + pad_values[:1]))
|
||||
input_ids = torch.randint(-2, 4 * n_placeholders + 10, (n_ids,), generator=gen).to(
|
||||
ids_dtype
|
||||
)
|
||||
|
||||
mask = mm_schedule._get_multimodal_mask(input_ids, placeholders)
|
||||
|
||||
broadcast = (input_ids.unsqueeze(-1) == placeholders).any(-1).unsqueeze(-1)
|
||||
assert torch.equal(mask, broadcast)
|
||||
assert torch.equal(mask, torch.isin(input_ids, placeholders).unsqueeze(-1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
|
||||
@@ -27,20 +27,6 @@ register_cpu_ci(est_time=12, suite="base-a-test-cpu")
|
||||
TRACK_INTERVAL = 4
|
||||
|
||||
|
||||
def _set_mamba_track_indices_on_cpu(batch, track_positions=None):
|
||||
real_tensor = torch.tensor
|
||||
|
||||
def tensor_without_pinning(*args, **kwargs):
|
||||
kwargs["pin_memory"] = False
|
||||
return real_tensor(*args, **kwargs)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.managers.schedule_batch.torch.tensor",
|
||||
side_effect=tensor_without_pinning,
|
||||
):
|
||||
set_mamba_track_indices_from_reqs(batch, track_positions)
|
||||
|
||||
|
||||
def _make_batch() -> tuple[Req, ScheduleBatch]:
|
||||
sampling_params = SamplingParams(max_new_tokens=32)
|
||||
sampling_params.normalize(None)
|
||||
@@ -202,6 +188,7 @@ class TestMambaPrevTrackSeqlen(unittest.TestCase):
|
||||
req, batch = _make_batch()
|
||||
processor = _make_processor()
|
||||
req.kv.mamba_ping_pong_track_buffer = torch.tensor([10, 11])
|
||||
req.kv.mamba_ping_pong_empty = [False, False]
|
||||
req.kv.mamba_next_track_idx = 0
|
||||
req.kv.mamba_last_track_idx = 1
|
||||
batch.mamba_track_buffer_indices = None
|
||||
@@ -242,7 +229,7 @@ class TestMambaFreedTrackIndicesCPU(unittest.TestCase):
|
||||
req_index_to_mamba_ping_pong_track_buffer_mapping=mapping
|
||||
),
|
||||
)
|
||||
_set_mamba_track_indices_on_cpu(batch, positions)
|
||||
set_mamba_track_indices_from_reqs(batch, positions)
|
||||
self.assertEqual(batch.mamba_track_indices.tolist(), expected)
|
||||
|
||||
|
||||
|
||||
@@ -14,9 +14,11 @@ from array import array
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.managers.schedule_batch import Req, ScheduleBatch
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
Req,
|
||||
ScheduleBatch,
|
||||
_MambaRadixCacheV2TrackEntry,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_context, mamba_track_grid
|
||||
from sglang.srt.sampling.sampling_params import SamplingParams
|
||||
from sglang.srt.server_args import (
|
||||
@@ -30,15 +32,15 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
CHUNK = 64
|
||||
|
||||
|
||||
def _track_seqlen(
|
||||
def _prepare_extend(
|
||||
*,
|
||||
tree_page: int,
|
||||
prefix_len: int,
|
||||
extend_len: int,
|
||||
dcp_enabled: bool = False,
|
||||
) -> int | None:
|
||||
"""Run one extend through the tracker and report the donated depth, or
|
||||
None when the extend donates no checkpoint."""
|
||||
next_idx: int = 0,
|
||||
) -> tuple[_MambaRadixCacheV2TrackEntry, Req]:
|
||||
"""Run one extend through the tracker; return its track entry and the request."""
|
||||
server_args = ServerArgs(model_path="dummy", page_size=CHUNK)
|
||||
# The property would otherwise load the HF config for the dummy model.
|
||||
server_args._mamba_cache_chunk_size = CHUNK
|
||||
@@ -55,8 +57,7 @@ def _track_seqlen(
|
||||
)
|
||||
req.prefix_len = prefix_len
|
||||
req.extend_end = prefix_len + extend_len
|
||||
req.kv.mamba_ping_pong_track_buffer = torch.tensor([0, 1], dtype=torch.int64)
|
||||
req.kv.mamba_next_track_idx = 0
|
||||
req.kv.mamba_next_track_idx = next_idx
|
||||
req.mamba_branching_seqlen = None
|
||||
|
||||
batch = ScheduleBatch(reqs=[req])
|
||||
@@ -65,13 +66,19 @@ def _track_seqlen(
|
||||
)
|
||||
batch.tree_cache = SimpleNamespace(page_size=tree_page)
|
||||
batch.req_to_token_pool = MagicMock()
|
||||
batch.req_to_token_pool.get_mamba_ping_pong_other_idx.return_value = 1
|
||||
batch.req_to_token_pool.get_mamba_ping_pong_other_idx.return_value = 1 - next_idx
|
||||
|
||||
with patch(
|
||||
"sglang.srt.managers.schedule_batch.get_parallel",
|
||||
return_value=SimpleNamespace(dcp_enabled=dcp_enabled),
|
||||
):
|
||||
entry = batch._mamba_radix_cache_v2_req_prepare_for_extend(req)
|
||||
return entry, req
|
||||
|
||||
|
||||
def _track_seqlen(**kwargs) -> int | None:
|
||||
"""The donated depth of one extend, or None when it donates no checkpoint."""
|
||||
entry, req = _prepare_extend(**kwargs)
|
||||
if not entry.track_mask:
|
||||
assert req.kv.mamba_last_track_seqlen is None
|
||||
return None
|
||||
@@ -124,6 +131,20 @@ class TestMambaCheckpointDepth(unittest.TestCase):
|
||||
)
|
||||
|
||||
|
||||
class TestMambaTrackPosition(unittest.TestCase):
|
||||
def test_track_position_is_taken_before_the_ping_pong_swap(self):
|
||||
# prepare_for_extend gathers the slot id at this position; the read it
|
||||
# replaced took buffer[mamba_next_track_idx] before the boundary swap.
|
||||
for next_idx in (0, 1):
|
||||
with self.subTest(next_idx=next_idx):
|
||||
entry, req = _prepare_extend(
|
||||
tree_page=CHUNK, prefix_len=0, extend_len=96, next_idx=next_idx
|
||||
)
|
||||
self.assertTrue(entry.track_mask)
|
||||
self.assertEqual(entry.track_position, next_idx)
|
||||
self.assertEqual(req.kv.mamba_next_track_idx, 1 - next_idx)
|
||||
|
||||
|
||||
class TestMambaTrackGrid(unittest.TestCase):
|
||||
def _grid(self, *, interval: int, tree_page: int, chunk: int = CHUNK) -> int:
|
||||
with get_context().override_server_args(
|
||||
|
||||
@@ -1027,6 +1027,7 @@ class TestTriFactorySizing(unittest.TestCase):
|
||||
req_pool_idx=1,
|
||||
mamba_pool_idx=live[0],
|
||||
mamba_ping_pong_track_buffer=track,
|
||||
mamba_ping_pong_empty=[index == -1 for index in ids],
|
||||
)
|
||||
)
|
||||
session = StreamingSession(SimpleNamespace(req_to_token_pool=pool))
|
||||
@@ -1189,6 +1190,52 @@ class TestTriFactorySizing(unittest.TestCase):
|
||||
allocator.free(retained)
|
||||
self.assertEqual(allocator.available_size(), initial_available)
|
||||
|
||||
def test_ping_pong_mapping_mirrors_the_request_buffers(self):
|
||||
from types import SimpleNamespace
|
||||
|
||||
from sglang.srt.managers.schedule_batch import (
|
||||
ReqKvInfo,
|
||||
_gather_mamba_track_indices,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
for lazy in (False, True):
|
||||
with self.subTest(lazy=lazy):
|
||||
kw = self._factory_kwargs(
|
||||
enable_mamba_extra_buffer=True,
|
||||
enable_mamba_extra_buffer_lazy=lazy,
|
||||
disable_overlap_schedule=False,
|
||||
max_mamba_cache_size=16,
|
||||
)
|
||||
with get_parallel().override(attn_dcp_size=1):
|
||||
pool = init_unified_mamba_swa_pools(**kw).req_to_token_pool
|
||||
mapping = pool.req_index_to_mamba_ping_pong_track_buffer_mapping
|
||||
allocator = pool.mamba_allocator
|
||||
reqs = [SimpleNamespace(kv=ReqKvInfo()) for _ in range(3)]
|
||||
rows = pool.alloc(reqs)
|
||||
|
||||
def buffers():
|
||||
return [
|
||||
req.kv.mamba_ping_pong_track_buffer.tolist() for req in reqs
|
||||
]
|
||||
|
||||
self.assertEqual(mapping[rows].tolist(), buffers())
|
||||
# Every writer of a buffer entry rewrites the request's row.
|
||||
pool.donate_mamba_ping_pong_slot(reqs[0], allocator.alloc(1))
|
||||
pool.set_mamba_ping_pong_slot(reqs[1], 0, -1)
|
||||
pool.set_mamba_ping_pong_slot(reqs[1], 1, allocator.alloc(1)[0])
|
||||
self.assertEqual(mapping[rows].tolist(), buffers())
|
||||
# The device gather returns what the per-request .item() read did.
|
||||
positions = [0, 0, 1]
|
||||
gathered = _gather_mamba_track_indices(
|
||||
mapping, torch.tensor(rows), torch.tensor(positions)
|
||||
)
|
||||
item_reads = [
|
||||
req.kv.mamba_ping_pong_track_buffer[pos].item()
|
||||
for req, pos in zip(reqs, positions)
|
||||
]
|
||||
self.assertEqual(gathered.tolist(), item_reads)
|
||||
|
||||
def test_budget_sizing_and_boot_signature(self):
|
||||
from sglang.srt.mem_cache.allocator.unified_hybrid_swa import (
|
||||
UnifiedSWAAllocatorBase,
|
||||
|
||||
Reference in New Issue
Block a user