[Perf] scheduler: Plan hybrid Mamba extend batches without host-device syncs (#45472)
Co-authored-by: Lu Fang <116399278+luccafong@users.noreply.github.com> Co-authored-by: Shiyan Deng <20773414+842974287@users.noreply.github.com>
This commit is contained in:
co-authored by
Lu Fang
Shiyan Deng
parent
5e79333c95
commit
740d5f9f9e
@@ -286,6 +286,7 @@ class MlxAuxiliaryStateReqToTokenPool(ReqToTokenPool):
|
||||
if mamba_ping_pong_track_buffer_to_keep is None:
|
||||
self.auxiliary_state_pool.free(track_buffer)
|
||||
req.kv.mamba_ping_pong_track_buffer = None
|
||||
req.kv.mamba_ping_pong_empty = None
|
||||
req.kv.mamba_next_track_idx = None
|
||||
req.kv.mamba_last_track_idx = None
|
||||
|
||||
@@ -387,6 +388,7 @@ class MlxAuxiliaryStateComponent(MambaComponent):
|
||||
if track_buffer is not None:
|
||||
self.cache.req_to_token_pool.auxiliary_state_pool.free(track_buffer)
|
||||
req.kv.mamba_ping_pong_track_buffer = None
|
||||
req.kv.mamba_ping_pong_empty = None
|
||||
req.kv.mamba_next_track_idx = None
|
||||
req.kv.mamba_last_track_idx = None
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
@@ -412,6 +414,7 @@ class MlxAuxiliaryStateComponent(MambaComponent):
|
||||
if track_buffer is not None:
|
||||
self.cache.req_to_token_pool.auxiliary_state_pool.free(track_buffer)
|
||||
req.kv.mamba_ping_pong_track_buffer = None
|
||||
req.kv.mamba_ping_pong_empty = None
|
||||
req.kv.mamba_next_track_idx = None
|
||||
req.kv.mamba_last_track_idx = None
|
||||
req.kv.mamba_pool_idx = None
|
||||
|
||||
@@ -417,6 +417,7 @@ class MlxModelRunner:
|
||||
)
|
||||
return
|
||||
req.kv.mamba_ping_pong_track_buffer = track_buffer
|
||||
req.kv.mamba_ping_pong_empty = [False]
|
||||
req.kv.mamba_next_track_idx = 0
|
||||
req.kv.mamba_last_track_idx = 0
|
||||
|
||||
|
||||
@@ -655,7 +655,15 @@ def _get_chunked_prefill_embedding(
|
||||
def _get_multimodal_mask(
|
||||
input_ids: torch.Tensor, placeholder_tensor: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return torch.isin(input_ids, placeholder_tensor).unsqueeze(-1)
|
||||
# placeholder_tensor arrives sorted from embed_mm_inputs. torch.isin
|
||||
# brute-forces only while P < 10 * N**0.145 (P placeholders, N ids); past
|
||||
# that it sorts with unique(), whose count syncs the host. The sorted
|
||||
# lookup never syncs.
|
||||
assert placeholder_tensor.numel() > 0
|
||||
placeholders = placeholder_tensor.to(input_ids.dtype)
|
||||
idx = torch.searchsorted(placeholders, input_ids)
|
||||
idx.clamp_(max=placeholders.numel() - 1)
|
||||
return (placeholders[idx] == input_ids).unsqueeze(-1)
|
||||
|
||||
|
||||
def _count_mm_tokens_in_extend(
|
||||
|
||||
@@ -495,7 +495,10 @@ def embed_mm_inputs(
|
||||
embedder = getattr(multimodal_model, f"get_{modality_id}_feature", None)
|
||||
if len(items) != 0:
|
||||
assert embedder is not None, f"no embedding method found for {modality}"
|
||||
pad_values = [value for item in items for value in item.padding_values()]
|
||||
# Sorted for the searchsorted lookup in _get_multimodal_mask.
|
||||
pad_values = sorted(
|
||||
value for item in items for value in item.padding_values()
|
||||
)
|
||||
if input_ids.device.type == "cuda":
|
||||
# Pinned staging keeps the placeholder copy asynchronous on CUDA.
|
||||
placeholder_cpu = torch.tensor(
|
||||
|
||||
@@ -15,6 +15,7 @@ from sglang.srt.runtime_context import (
|
||||
mamba_track_grid,
|
||||
)
|
||||
from sglang.srt.utils.common import (
|
||||
async_h2d,
|
||||
ceil_align,
|
||||
flatten_arrays_to_pinned_cpu,
|
||||
is_hip,
|
||||
@@ -996,6 +997,8 @@ class ReqKvInfo:
|
||||
# Mamba state: an independent resource; whether it is held is `holds_mamba`.
|
||||
mamba_pool_idx: Optional[torch.Tensor] = None # shape (1)
|
||||
mamba_ping_pong_track_buffer: Optional[torch.Tensor] = None # shape (2)
|
||||
# Per entry of the ping-pong buffer: True where it holds no slot (-1).
|
||||
mamba_ping_pong_empty: Optional[list[bool]] = None
|
||||
mamba_next_track_idx: Optional[int] = None # 0 or 1
|
||||
mamba_last_track_idx: Optional[int] = None # 0 or 1
|
||||
# Seq len of the last cached mamba state
|
||||
@@ -2091,6 +2094,7 @@ class Req(ReqDllmMixin):
|
||||
self.inflight_middle_chunks = 0
|
||||
self.kv.mamba_pool_idx = None
|
||||
self.kv.mamba_ping_pong_track_buffer = None
|
||||
self.kv.mamba_ping_pong_empty = None
|
||||
self.kv.mamba_next_track_idx = None
|
||||
self.kv.mamba_last_track_idx = None
|
||||
self.kv.mamba_last_track_seqlen = None
|
||||
@@ -2286,10 +2290,19 @@ class Req(ReqDllmMixin):
|
||||
|
||||
class _MambaRadixCacheV2TrackEntry(NamedTuple):
|
||||
track_mask: bool
|
||||
track_index: int
|
||||
track_position: int
|
||||
track_seqlen: int
|
||||
|
||||
|
||||
def _gather_mamba_track_indices(
|
||||
ping_pong_mapping: torch.Tensor,
|
||||
req_pool_indices: torch.Tensor,
|
||||
track_positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Each request's ping-pong slot id at its track position, read on the device."""
|
||||
return ping_pong_mapping[req_pool_indices, track_positions]
|
||||
|
||||
|
||||
def mamba_lazy_spec_in_window(
|
||||
req, mamba_track_interval: int, max_draft_tokens: int
|
||||
) -> bool:
|
||||
@@ -2311,10 +2324,7 @@ def set_mamba_track_indices_from_reqs(
|
||||
track_positions: optional per-req ping-pong position override (the lazy
|
||||
spec track plan, see mamba_lazy_spec_prepare).
|
||||
"""
|
||||
req_to_token_pool = batch.req_to_token_pool
|
||||
all_buffers = req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping[
|
||||
batch.req_pool_indices
|
||||
] # (bs, ping_pong_size), int64, on device
|
||||
mapping = batch.req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping
|
||||
if track_positions is None:
|
||||
# Guard: mamba_next_track_idx may be None for requests that haven't
|
||||
# gone through _alloc_ping_pong_buffer yet (e.g., spec v2 verify path).
|
||||
@@ -2332,17 +2342,9 @@ def set_mamba_track_indices_from_reqs(
|
||||
i for i, req in enumerate(batch.reqs) if req.kv.mamba_next_track_idx is None
|
||||
]
|
||||
batch.mamba_track_buffer_indices = list(track_positions)
|
||||
idx = (
|
||||
torch.tensor(
|
||||
track_positions,
|
||||
dtype=torch.int64,
|
||||
pin_memory=True,
|
||||
)
|
||||
.unsqueeze(1)
|
||||
.to(device=all_buffers.device, non_blocking=True)
|
||||
)
|
||||
batch.mamba_track_indices = (
|
||||
torch.gather(all_buffers, 1, idx).squeeze(1).to(torch.int64)
|
||||
positions = async_h2d(track_positions, dtype=torch.int64, device=mapping.device)
|
||||
batch.mamba_track_indices = _gather_mamba_track_indices(
|
||||
mapping, batch.req_pool_indices, positions
|
||||
)
|
||||
if freed_rows:
|
||||
# Overlap can leave a request in TARGET_VERIFY after its Mamba state has
|
||||
@@ -2918,7 +2920,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
extend_input_logprob_token_ids = []
|
||||
multimodal_inputs = []
|
||||
mamba_track_mask_cpu = []
|
||||
mamba_track_indices_cpu = []
|
||||
mamba_track_positions_cpu = []
|
||||
mamba_track_seqlens_cpu = []
|
||||
|
||||
for i, (req, seq_len, pre_len) in enumerate(zip(reqs, seq_lens, prefix_lens)):
|
||||
@@ -2991,7 +2993,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
if get_exec().mamba.enable_mamba_extra_buffer:
|
||||
track_entry = self._mamba_radix_cache_v2_req_prepare_for_extend(req)
|
||||
mamba_track_mask_cpu.append(track_entry.track_mask)
|
||||
mamba_track_indices_cpu.append(track_entry.track_index)
|
||||
mamba_track_positions_cpu.append(track_entry.track_position)
|
||||
mamba_track_seqlens_cpu.append(track_entry.track_seqlen)
|
||||
|
||||
if self.return_logprob:
|
||||
@@ -3098,21 +3100,21 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
if get_exec().mamba.enable_mamba_extra_buffer:
|
||||
self.mamba_prefill_track_mask_cpu = mamba_track_mask_cpu
|
||||
self.mamba_track_seqlens_cpu = mamba_track_seqlens_cpu
|
||||
self.mamba_track_indices = torch.tensor(
|
||||
mamba_track_indices_cpu,
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
# Ping-pong slot ids live on the device; gather them there.
|
||||
track_positions = torch.tensor(
|
||||
mamba_track_positions_cpu, dtype=torch.int64, pin_memory=_pin
|
||||
).to(self.device, non_blocking=True)
|
||||
self.mamba_track_indices = _gather_mamba_track_indices(
|
||||
self.req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping,
|
||||
self.req_pool_indices,
|
||||
track_positions,
|
||||
)
|
||||
self.mamba_track_mask = torch.tensor(
|
||||
mamba_track_mask_cpu,
|
||||
dtype=torch.bool,
|
||||
device=self.device,
|
||||
)
|
||||
mamba_track_mask_cpu, dtype=torch.bool, pin_memory=_pin
|
||||
).to(self.device, non_blocking=True)
|
||||
self.mamba_track_seqlens = torch.tensor(
|
||||
mamba_track_seqlens_cpu,
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
mamba_track_seqlens_cpu, dtype=torch.int64, pin_memory=_pin
|
||||
).to(self.device, non_blocking=True)
|
||||
|
||||
# Collect mamba init info for deferred ops on forward stream
|
||||
if any(req.kv.holds_mamba for req in reqs):
|
||||
@@ -3174,9 +3176,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
mamba_track_seqlen_aligned = (
|
||||
prefix_len + (req.extend_len // checkpoint_grid) * checkpoint_grid
|
||||
)
|
||||
track_index = req.kv.mamba_ping_pong_track_buffer[
|
||||
req.kv.mamba_next_track_idx
|
||||
].item()
|
||||
track_position = req.kv.mamba_next_track_idx
|
||||
mamba_track_seqlen = -1
|
||||
if mask:
|
||||
# mamba_track_seqlen is used to calculate the indices to track in
|
||||
@@ -3228,7 +3228,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
|
||||
return _MambaRadixCacheV2TrackEntry(
|
||||
track_mask=mask,
|
||||
track_index=track_index,
|
||||
track_position=track_position,
|
||||
track_seqlen=mamba_track_seqlen,
|
||||
)
|
||||
|
||||
@@ -3627,7 +3627,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
if self.seq_lens_cpu[i].item() % mamba_track_interval != 0:
|
||||
continue
|
||||
other_idx = 1 - req.kv.mamba_next_track_idx
|
||||
if buf[other_idx].item() != -1:
|
||||
if not req.kv.mamba_ping_pong_empty[other_idx]:
|
||||
# With overlap the previous forward's post-processing
|
||||
# (which frees this slot) hasn't run yet. Skip.
|
||||
continue
|
||||
@@ -3688,7 +3688,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
track_positions.append(req.kv.mamba_next_track_idx)
|
||||
continue
|
||||
other_idx = 1 - req.kv.mamba_next_track_idx
|
||||
has_pending = buf[other_idx].item() != -1
|
||||
has_pending = not req.kv.mamba_ping_pong_empty[other_idx]
|
||||
if not has_pending:
|
||||
if envs.SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL.get():
|
||||
new_slot = None
|
||||
|
||||
@@ -1590,9 +1590,9 @@ class SchedulerBatchResultProcessor:
|
||||
other_idx = 1 - req.kv.mamba_next_track_idx
|
||||
# Recompute the in-flight verify's plan (kv_committed_len is
|
||||
# frozen since its prepare, so the recompute is exact).
|
||||
keep_may_be_written_in_flight = req.kv.mamba_ping_pong_track_buffer[
|
||||
keep_may_be_written_in_flight = req.kv.mamba_ping_pong_empty[
|
||||
other_idx
|
||||
].item() == -1 and mamba_lazy_spec_in_window(
|
||||
] and mamba_lazy_spec_in_window(
|
||||
req,
|
||||
mamba_track_grid(self.tree_cache.page_size),
|
||||
max_speculative_num_draft_tokens(),
|
||||
@@ -1677,9 +1677,8 @@ class SchedulerBatchResultProcessor:
|
||||
req.kv.mamba_last_track_idx = track_idx
|
||||
req.kv.mamba_next_track_idx = track_idx
|
||||
other_idx = 1 - track_idx
|
||||
other_val = req.kv.mamba_ping_pong_track_buffer[other_idx].item()
|
||||
if other_val != -1:
|
||||
pool = batch.req_to_token_pool
|
||||
pool = batch.req_to_token_pool
|
||||
if not req.kv.mamba_ping_pong_empty[other_idx]:
|
||||
pool.mamba_allocator.free(
|
||||
req.kv.mamba_ping_pong_track_buffer[other_idx].unsqueeze(0)
|
||||
)
|
||||
|
||||
@@ -45,7 +45,7 @@ def kv_private_swa_tokens(kv: ReqKvInfo, page_size: int) -> int:
|
||||
def kv_mamba_slots(kv: ReqKvInfo) -> int:
|
||||
total = kv.mamba_pool_idx.numel() if kv.holds_mamba else 0
|
||||
if kv.mamba_ping_pong_track_buffer is not None:
|
||||
total += int((kv.mamba_ping_pong_track_buffer != -1).sum().item())
|
||||
total += kv.mamba_ping_pong_empty.count(False)
|
||||
return total
|
||||
|
||||
|
||||
|
||||
@@ -95,6 +95,7 @@ from sglang.srt.utils import (
|
||||
from sglang.srt.utils.async_probe import (
|
||||
maybe_detect_oob,
|
||||
)
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -1146,33 +1147,17 @@ class MambaPool:
|
||||
"""Zero out mamba state at the given pool indices. Must run on forward stream."""
|
||||
for sibling in self._slot_siblings:
|
||||
sibling.reset_slots(indices)
|
||||
idx64 = indices.to(torch.int64)
|
||||
if self._should_fuse_slot_ops():
|
||||
from sglang.srt.mem_cache.mamba_slot_fused import fused_clear_conv_slots
|
||||
|
||||
fused_clear_conv_slots(self._conv_slot_desc, indices)
|
||||
temporal = self.mamba_cache.temporal
|
||||
if temporal.numel() > 0:
|
||||
temporal[:, indices] = 0
|
||||
return
|
||||
if not _is_npu:
|
||||
need_size = len(indices)
|
||||
for i in range(len(self.mamba_cache.conv)):
|
||||
t = self.mamba_cache.conv[i]
|
||||
z = torch.zeros(1, dtype=t.dtype, device=t.device).expand(
|
||||
t.shape[0], need_size, *t.shape[2:]
|
||||
)
|
||||
t[:, indices] = z
|
||||
t = self.mamba_cache.temporal
|
||||
z = torch.zeros(1, dtype=t.dtype, device=t.device).expand(
|
||||
t.shape[0], need_size, *t.shape[2:]
|
||||
)
|
||||
t[:, indices] = z
|
||||
fused_clear_conv_slots(self._conv_slot_desc, idx64)
|
||||
else:
|
||||
for i in range(len(self.mamba_cache.conv)):
|
||||
t = self.mamba_cache.conv[i]
|
||||
t[:, indices] = 0
|
||||
t = self.mamba_cache.temporal
|
||||
t[:, indices] = 0
|
||||
for conv in self.mamba_cache.conv:
|
||||
conv.index_fill_(1, idx64, 0)
|
||||
temporal = self.mamba_cache.temporal
|
||||
if temporal.numel() > 0:
|
||||
temporal.index_fill_(1, idx64, 0)
|
||||
|
||||
def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor):
|
||||
"""Clone mamba state (conv + temporal) from src slots into dst slots.
|
||||
@@ -1589,15 +1574,18 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
if select_index is None:
|
||||
return None
|
||||
|
||||
spec_write_pos = getattr(self.mamba_pool, "replayssm_spec_write_pos", None)
|
||||
spec_write_pos = self.mamba_pool.replayssm_spec_write_pos
|
||||
if spec_write_pos is not None:
|
||||
fresh_indices = [
|
||||
idx for idx, fresh in zip(select_index, fresh_req_rows) if fresh
|
||||
]
|
||||
if fresh_indices:
|
||||
spec_write_pos[fresh_indices] = 0
|
||||
self.mamba_pool.replayssm_cache_base[fresh_indices] = 0
|
||||
self.mamba_pool.replayssm_is_flush[fresh_indices] = 0
|
||||
fresh_rows = async_h2d(
|
||||
fresh_indices, dtype=torch.int64, device=self.device
|
||||
)
|
||||
spec_write_pos.index_fill_(0, fresh_rows, 0)
|
||||
self.mamba_pool.replayssm_cache_base.index_fill_(0, fresh_rows, 0)
|
||||
self.mamba_pool.replayssm_is_flush.index_fill_(0, fresh_rows, 0)
|
||||
|
||||
mamba_indices: list[torch.Tensor] = []
|
||||
mamba_ping_pong_track_buffers: list[torch.Tensor] = []
|
||||
@@ -1631,11 +1619,13 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
assert len(select_index) == len(mamba_ping_pong_track_buffers), (
|
||||
"Not enough space for mamba ping pong idx, try to increase --mamba-full-memory-ratio."
|
||||
)
|
||||
# Device rows: indexing a device tensor with a Python list syncs the host.
|
||||
rows = async_h2d(select_index, dtype=torch.int64, device=self.device)
|
||||
mamba_index_tensor = torch.stack(mamba_indices).to(dtype=torch.int32)
|
||||
self.req_index_to_mamba_index_mapping[select_index] = mamba_index_tensor
|
||||
self.req_index_to_mamba_index_mapping[rows] = mamba_index_tensor
|
||||
if self.enable_mamba_extra_buffer:
|
||||
ping_pong_tensor = torch.stack(mamba_ping_pong_track_buffers)
|
||||
self.req_index_to_mamba_ping_pong_track_buffer_mapping[select_index] = (
|
||||
self.req_index_to_mamba_ping_pong_track_buffer_mapping[rows] = (
|
||||
ping_pong_tensor
|
||||
)
|
||||
return select_index
|
||||
@@ -1779,6 +1769,9 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
)
|
||||
buf[:n] = slots
|
||||
req.kv.mamba_ping_pong_track_buffer = buf
|
||||
req.kv.mamba_ping_pong_empty = [False] * n + [True] * (
|
||||
self.mamba_ping_pong_track_buffer_size - n
|
||||
)
|
||||
req.kv.mamba_prev_track_seqlen = None
|
||||
req.kv.mamba_next_track_idx = 0
|
||||
req.kv.mamba_last_track_idx = (
|
||||
@@ -1794,10 +1787,17 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
req_index_to_mamba_ping_pong_track_buffer_mapping in sync so that
|
||||
set_mamba_track_indices_from_reqs reads correct slot indices.
|
||||
"""
|
||||
req.kv.mamba_ping_pong_track_buffer[idx] = value
|
||||
if torch.is_tensor(value):
|
||||
req.kv.mamba_ping_pong_track_buffer[idx] = value
|
||||
else:
|
||||
assert value == -1, f"expected a slot tensor or -1, got {value!r}"
|
||||
# fill_ stays on the device; assigning the int is a blocking host copy.
|
||||
req.kv.mamba_ping_pong_track_buffer[idx].fill_(-1)
|
||||
self.req_index_to_mamba_ping_pong_track_buffer_mapping[req.kv.req_pool_idx] = (
|
||||
req.kv.mamba_ping_pong_track_buffer
|
||||
)
|
||||
# value is polymorphic: a slot tensor fills the entry, the int -1 empties it.
|
||||
req.kv.mamba_ping_pong_empty[idx] = not torch.is_tensor(value)
|
||||
|
||||
def donate_mamba_ping_pong_slot(
|
||||
self, req: Req, new_slot: torch.Tensor
|
||||
@@ -1831,52 +1831,27 @@ class HybridReqToTokenPool(ReqToTokenPool):
|
||||
req.kv.mamba_pool_idx = None
|
||||
|
||||
if self.enable_mamba_extra_buffer:
|
||||
mamba_ping_pong_track_buffer_to_free = (
|
||||
self.req_index_to_mamba_ping_pong_track_buffer_mapping[
|
||||
req.kv.req_pool_idx
|
||||
]
|
||||
size = self.mamba_ping_pong_track_buffer_size
|
||||
keep = mamba_ping_pong_track_buffer_to_keep
|
||||
assert keep in (None, 0, 1) and (keep != 1 or size == 2), (
|
||||
f"cannot keep ping-pong entry {keep} of {size}"
|
||||
)
|
||||
if mamba_ping_pong_track_buffer_to_keep is not None:
|
||||
assert mamba_ping_pong_track_buffer_to_keep in [
|
||||
0,
|
||||
1,
|
||||
], (
|
||||
f"mamba_ping_pong_track_buffer_to_keep must be 0 or 1, {mamba_ping_pong_track_buffer_to_keep=}"
|
||||
)
|
||||
# Avoid Python-list advanced indexing on a device tensor.
|
||||
# The ping-pong buffer size is either 2 (normal) or 1 (spec decode).
|
||||
if self.mamba_ping_pong_track_buffer_size == 2:
|
||||
idx_to_free = 1 - mamba_ping_pong_track_buffer_to_keep
|
||||
mamba_ping_pong_track_buffer_to_free = (
|
||||
mamba_ping_pong_track_buffer_to_free[
|
||||
idx_to_free : idx_to_free + 1
|
||||
]
|
||||
)
|
||||
else:
|
||||
assert self.mamba_ping_pong_track_buffer_size == 1, (
|
||||
f"Unexpected mamba_ping_pong_track_buffer_size="
|
||||
f"{self.mamba_ping_pong_track_buffer_size}"
|
||||
)
|
||||
assert mamba_ping_pong_track_buffer_to_keep == 0, (
|
||||
"mamba_ping_pong_track_buffer_to_keep must be 0 when "
|
||||
"mamba_ping_pong_track_buffer_size is 1"
|
||||
)
|
||||
# Keep the only slot, so free nothing.
|
||||
mamba_ping_pong_track_buffer_to_free = (
|
||||
mamba_ping_pong_track_buffer_to_free[0:0]
|
||||
)
|
||||
if self.enable_mamba_extra_buffer_lazy:
|
||||
mamba_ping_pong_track_buffer_to_free = (
|
||||
mamba_ping_pong_track_buffer_to_free[
|
||||
mamba_ping_pong_track_buffer_to_free != -1
|
||||
]
|
||||
)
|
||||
self.mamba_allocator.free(mamba_ping_pong_track_buffer_to_free)
|
||||
empty = req.kv.mamba_ping_pong_empty
|
||||
positions = [p for p in range(size) if p != keep and not empty[p]]
|
||||
row = self.req_index_to_mamba_ping_pong_track_buffer_mapping[
|
||||
req.kv.req_pool_idx
|
||||
]
|
||||
if len(positions) == size:
|
||||
self.mamba_allocator.free(row)
|
||||
else:
|
||||
for p in positions:
|
||||
self.mamba_allocator.free(row[p : p + 1])
|
||||
# Match the req.kv.mamba_pool_idx=None clear above so the next
|
||||
# alloc() doesn't see a stale ping-pong reference on the req
|
||||
# and skip allocation (which would silently reuse a freed
|
||||
# tensor on the req side while the new pool slot leaks).
|
||||
req.kv.mamba_ping_pong_track_buffer = None
|
||||
req.kv.mamba_ping_pong_empty = None
|
||||
req.kv.mamba_next_track_idx = None
|
||||
req.kv.mamba_last_track_idx = None
|
||||
req.kv.mamba_last_track_seqlen = None
|
||||
|
||||
@@ -1115,6 +1115,7 @@ class UnifiedMambaPool(MambaPool):
|
||||
self.replayssm_is_kda = False
|
||||
self.enable_linear_replayssm_spec = False
|
||||
self.replayssm_spec_fold = False
|
||||
self.replayssm_spec_write_pos = None
|
||||
self.replayssm_cache_base = None
|
||||
self.replayssm_is_flush = None
|
||||
self.debug_memory_pool = False
|
||||
|
||||
@@ -255,9 +255,12 @@ class StreamingSession:
|
||||
mamba_allocator.free(slot.kv.mamba_pool_idx.unsqueeze(0))
|
||||
slot.kv.mamba_pool_idx = None
|
||||
if slot.kv.mamba_ping_pong_track_buffer is not None:
|
||||
indices = slot.kv.mamba_ping_pong_track_buffer
|
||||
mamba_allocator.free(indices[indices != -1])
|
||||
buffer = slot.kv.mamba_ping_pong_track_buffer
|
||||
for position, empty in enumerate(slot.kv.mamba_ping_pong_empty):
|
||||
if not empty:
|
||||
mamba_allocator.free(buffer[position : position + 1])
|
||||
slot.kv.mamba_ping_pong_track_buffer = None
|
||||
slot.kv.mamba_ping_pong_empty = None
|
||||
|
||||
# -- Internal helpers (streaming body bits) --
|
||||
|
||||
|
||||
Reference in New Issue
Block a user