[Perf] scheduler: Plan hybrid Mamba extend batches without host-device syncs (#45472)

Co-authored-by: Lu Fang <116399278+luccafong@users.noreply.github.com>
Co-authored-by: Shiyan Deng <20773414+842974287@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-21 11:56:50 -07:00
committed by GitHub
co-authored by Lu Fang Shiyan Deng
parent 5e79333c95
commit 740d5f9f9e
14 changed files with 217 additions and 141 deletions
@@ -286,6 +286,7 @@ class MlxAuxiliaryStateReqToTokenPool(ReqToTokenPool):
if mamba_ping_pong_track_buffer_to_keep is None:
self.auxiliary_state_pool.free(track_buffer)
req.kv.mamba_ping_pong_track_buffer = None
req.kv.mamba_ping_pong_empty = None
req.kv.mamba_next_track_idx = None
req.kv.mamba_last_track_idx = None
@@ -387,6 +388,7 @@ class MlxAuxiliaryStateComponent(MambaComponent):
if track_buffer is not None:
self.cache.req_to_token_pool.auxiliary_state_pool.free(track_buffer)
req.kv.mamba_ping_pong_track_buffer = None
req.kv.mamba_ping_pong_empty = None
req.kv.mamba_next_track_idx = None
req.kv.mamba_last_track_idx = None
req.kv.mamba_last_track_seqlen = None
@@ -412,6 +414,7 @@ class MlxAuxiliaryStateComponent(MambaComponent):
if track_buffer is not None:
self.cache.req_to_token_pool.auxiliary_state_pool.free(track_buffer)
req.kv.mamba_ping_pong_track_buffer = None
req.kv.mamba_ping_pong_empty = None
req.kv.mamba_next_track_idx = None
req.kv.mamba_last_track_idx = None
req.kv.mamba_pool_idx = None
@@ -417,6 +417,7 @@ class MlxModelRunner:
)
return
req.kv.mamba_ping_pong_track_buffer = track_buffer
req.kv.mamba_ping_pong_empty = [False]
req.kv.mamba_next_track_idx = 0
req.kv.mamba_last_track_idx = 0
+9 -1
View File
@@ -655,7 +655,15 @@ def _get_chunked_prefill_embedding(
def _get_multimodal_mask(
input_ids: torch.Tensor, placeholder_tensor: torch.Tensor
) -> torch.Tensor:
return torch.isin(input_ids, placeholder_tensor).unsqueeze(-1)
# placeholder_tensor arrives sorted from embed_mm_inputs. torch.isin
# brute-forces only while P < 10 * N**0.145 (P placeholders, N ids); past
# that it sorts with unique(), whose count syncs the host. The sorted
# lookup never syncs.
assert placeholder_tensor.numel() > 0
placeholders = placeholder_tensor.to(input_ids.dtype)
idx = torch.searchsorted(placeholders, input_ids)
idx.clamp_(max=placeholders.numel() - 1)
return (placeholders[idx] == input_ids).unsqueeze(-1)
def _count_mm_tokens_in_extend(
+4 -1
View File
@@ -495,7 +495,10 @@ def embed_mm_inputs(
embedder = getattr(multimodal_model, f"get_{modality_id}_feature", None)
if len(items) != 0:
assert embedder is not None, f"no embedding method found for {modality}"
pad_values = [value for item in items for value in item.padding_values()]
# Sorted for the searchsorted lookup in _get_multimodal_mask.
pad_values = sorted(
value for item in items for value in item.padding_values()
)
if input_ids.device.type == "cuda":
# Pinned staging keeps the placeholder copy asynchronous on CUDA.
placeholder_cpu = torch.tensor(
+36 -36
View File
@@ -15,6 +15,7 @@ from sglang.srt.runtime_context import (
mamba_track_grid,
)
from sglang.srt.utils.common import (
async_h2d,
ceil_align,
flatten_arrays_to_pinned_cpu,
is_hip,
@@ -996,6 +997,8 @@ class ReqKvInfo:
# Mamba state: an independent resource; whether it is held is `holds_mamba`.
mamba_pool_idx: Optional[torch.Tensor] = None # shape (1)
mamba_ping_pong_track_buffer: Optional[torch.Tensor] = None # shape (2)
# Per entry of the ping-pong buffer: True where it holds no slot (-1).
mamba_ping_pong_empty: Optional[list[bool]] = None
mamba_next_track_idx: Optional[int] = None # 0 or 1
mamba_last_track_idx: Optional[int] = None # 0 or 1
# Seq len of the last cached mamba state
@@ -2091,6 +2094,7 @@ class Req(ReqDllmMixin):
self.inflight_middle_chunks = 0
self.kv.mamba_pool_idx = None
self.kv.mamba_ping_pong_track_buffer = None
self.kv.mamba_ping_pong_empty = None
self.kv.mamba_next_track_idx = None
self.kv.mamba_last_track_idx = None
self.kv.mamba_last_track_seqlen = None
@@ -2286,10 +2290,19 @@ class Req(ReqDllmMixin):
class _MambaRadixCacheV2TrackEntry(NamedTuple):
track_mask: bool
track_index: int
track_position: int
track_seqlen: int
def _gather_mamba_track_indices(
ping_pong_mapping: torch.Tensor,
req_pool_indices: torch.Tensor,
track_positions: torch.Tensor,
) -> torch.Tensor:
"""Each request's ping-pong slot id at its track position, read on the device."""
return ping_pong_mapping[req_pool_indices, track_positions]
def mamba_lazy_spec_in_window(
req, mamba_track_interval: int, max_draft_tokens: int
) -> bool:
@@ -2311,10 +2324,7 @@ def set_mamba_track_indices_from_reqs(
track_positions: optional per-req ping-pong position override (the lazy
spec track plan, see mamba_lazy_spec_prepare).
"""
req_to_token_pool = batch.req_to_token_pool
all_buffers = req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping[
batch.req_pool_indices
] # (bs, ping_pong_size), int64, on device
mapping = batch.req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping
if track_positions is None:
# Guard: mamba_next_track_idx may be None for requests that haven't
# gone through _alloc_ping_pong_buffer yet (e.g., spec v2 verify path).
@@ -2332,17 +2342,9 @@ def set_mamba_track_indices_from_reqs(
i for i, req in enumerate(batch.reqs) if req.kv.mamba_next_track_idx is None
]
batch.mamba_track_buffer_indices = list(track_positions)
idx = (
torch.tensor(
track_positions,
dtype=torch.int64,
pin_memory=True,
)
.unsqueeze(1)
.to(device=all_buffers.device, non_blocking=True)
)
batch.mamba_track_indices = (
torch.gather(all_buffers, 1, idx).squeeze(1).to(torch.int64)
positions = async_h2d(track_positions, dtype=torch.int64, device=mapping.device)
batch.mamba_track_indices = _gather_mamba_track_indices(
mapping, batch.req_pool_indices, positions
)
if freed_rows:
# Overlap can leave a request in TARGET_VERIFY after its Mamba state has
@@ -2918,7 +2920,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
extend_input_logprob_token_ids = []
multimodal_inputs = []
mamba_track_mask_cpu = []
mamba_track_indices_cpu = []
mamba_track_positions_cpu = []
mamba_track_seqlens_cpu = []
for i, (req, seq_len, pre_len) in enumerate(zip(reqs, seq_lens, prefix_lens)):
@@ -2991,7 +2993,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
if get_exec().mamba.enable_mamba_extra_buffer:
track_entry = self._mamba_radix_cache_v2_req_prepare_for_extend(req)
mamba_track_mask_cpu.append(track_entry.track_mask)
mamba_track_indices_cpu.append(track_entry.track_index)
mamba_track_positions_cpu.append(track_entry.track_position)
mamba_track_seqlens_cpu.append(track_entry.track_seqlen)
if self.return_logprob:
@@ -3098,21 +3100,21 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
if get_exec().mamba.enable_mamba_extra_buffer:
self.mamba_prefill_track_mask_cpu = mamba_track_mask_cpu
self.mamba_track_seqlens_cpu = mamba_track_seqlens_cpu
self.mamba_track_indices = torch.tensor(
mamba_track_indices_cpu,
dtype=torch.int64,
device=self.device,
# Ping-pong slot ids live on the device; gather them there.
track_positions = torch.tensor(
mamba_track_positions_cpu, dtype=torch.int64, pin_memory=_pin
).to(self.device, non_blocking=True)
self.mamba_track_indices = _gather_mamba_track_indices(
self.req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping,
self.req_pool_indices,
track_positions,
)
self.mamba_track_mask = torch.tensor(
mamba_track_mask_cpu,
dtype=torch.bool,
device=self.device,
)
mamba_track_mask_cpu, dtype=torch.bool, pin_memory=_pin
).to(self.device, non_blocking=True)
self.mamba_track_seqlens = torch.tensor(
mamba_track_seqlens_cpu,
dtype=torch.int64,
device=self.device,
)
mamba_track_seqlens_cpu, dtype=torch.int64, pin_memory=_pin
).to(self.device, non_blocking=True)
# Collect mamba init info for deferred ops on forward stream
if any(req.kv.holds_mamba for req in reqs):
@@ -3174,9 +3176,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
mamba_track_seqlen_aligned = (
prefix_len + (req.extend_len // checkpoint_grid) * checkpoint_grid
)
track_index = req.kv.mamba_ping_pong_track_buffer[
req.kv.mamba_next_track_idx
].item()
track_position = req.kv.mamba_next_track_idx
mamba_track_seqlen = -1
if mask:
# mamba_track_seqlen is used to calculate the indices to track in
@@ -3228,7 +3228,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
return _MambaRadixCacheV2TrackEntry(
track_mask=mask,
track_index=track_index,
track_position=track_position,
track_seqlen=mamba_track_seqlen,
)
@@ -3627,7 +3627,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
if self.seq_lens_cpu[i].item() % mamba_track_interval != 0:
continue
other_idx = 1 - req.kv.mamba_next_track_idx
if buf[other_idx].item() != -1:
if not req.kv.mamba_ping_pong_empty[other_idx]:
# With overlap the previous forward's post-processing
# (which frees this slot) hasn't run yet. Skip.
continue
@@ -3688,7 +3688,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
track_positions.append(req.kv.mamba_next_track_idx)
continue
other_idx = 1 - req.kv.mamba_next_track_idx
has_pending = buf[other_idx].item() != -1
has_pending = not req.kv.mamba_ping_pong_empty[other_idx]
if not has_pending:
if envs.SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL.get():
new_slot = None
@@ -1590,9 +1590,9 @@ class SchedulerBatchResultProcessor:
other_idx = 1 - req.kv.mamba_next_track_idx
# Recompute the in-flight verify's plan (kv_committed_len is
# frozen since its prepare, so the recompute is exact).
keep_may_be_written_in_flight = req.kv.mamba_ping_pong_track_buffer[
keep_may_be_written_in_flight = req.kv.mamba_ping_pong_empty[
other_idx
].item() == -1 and mamba_lazy_spec_in_window(
] and mamba_lazy_spec_in_window(
req,
mamba_track_grid(self.tree_cache.page_size),
max_speculative_num_draft_tokens(),
@@ -1677,9 +1677,8 @@ class SchedulerBatchResultProcessor:
req.kv.mamba_last_track_idx = track_idx
req.kv.mamba_next_track_idx = track_idx
other_idx = 1 - track_idx
other_val = req.kv.mamba_ping_pong_track_buffer[other_idx].item()
if other_val != -1:
pool = batch.req_to_token_pool
pool = batch.req_to_token_pool
if not req.kv.mamba_ping_pong_empty[other_idx]:
pool.mamba_allocator.free(
req.kv.mamba_ping_pong_track_buffer[other_idx].unsqueeze(0)
)
@@ -45,7 +45,7 @@ def kv_private_swa_tokens(kv: ReqKvInfo, page_size: int) -> int:
def kv_mamba_slots(kv: ReqKvInfo) -> int:
total = kv.mamba_pool_idx.numel() if kv.holds_mamba else 0
if kv.mamba_ping_pong_track_buffer is not None:
total += int((kv.mamba_ping_pong_track_buffer != -1).sum().item())
total += kv.mamba_ping_pong_empty.count(False)
return total
+45 -70
View File
@@ -95,6 +95,7 @@ from sglang.srt.utils import (
from sglang.srt.utils.async_probe import (
maybe_detect_oob,
)
from sglang.srt.utils.common import async_h2d
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
if TYPE_CHECKING:
@@ -1146,33 +1147,17 @@ class MambaPool:
"""Zero out mamba state at the given pool indices. Must run on forward stream."""
for sibling in self._slot_siblings:
sibling.reset_slots(indices)
idx64 = indices.to(torch.int64)
if self._should_fuse_slot_ops():
from sglang.srt.mem_cache.mamba_slot_fused import fused_clear_conv_slots
fused_clear_conv_slots(self._conv_slot_desc, indices)
temporal = self.mamba_cache.temporal
if temporal.numel() > 0:
temporal[:, indices] = 0
return
if not _is_npu:
need_size = len(indices)
for i in range(len(self.mamba_cache.conv)):
t = self.mamba_cache.conv[i]
z = torch.zeros(1, dtype=t.dtype, device=t.device).expand(
t.shape[0], need_size, *t.shape[2:]
)
t[:, indices] = z
t = self.mamba_cache.temporal
z = torch.zeros(1, dtype=t.dtype, device=t.device).expand(
t.shape[0], need_size, *t.shape[2:]
)
t[:, indices] = z
fused_clear_conv_slots(self._conv_slot_desc, idx64)
else:
for i in range(len(self.mamba_cache.conv)):
t = self.mamba_cache.conv[i]
t[:, indices] = 0
t = self.mamba_cache.temporal
t[:, indices] = 0
for conv in self.mamba_cache.conv:
conv.index_fill_(1, idx64, 0)
temporal = self.mamba_cache.temporal
if temporal.numel() > 0:
temporal.index_fill_(1, idx64, 0)
def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor):
"""Clone mamba state (conv + temporal) from src slots into dst slots.
@@ -1589,15 +1574,18 @@ class HybridReqToTokenPool(ReqToTokenPool):
if select_index is None:
return None
spec_write_pos = getattr(self.mamba_pool, "replayssm_spec_write_pos", None)
spec_write_pos = self.mamba_pool.replayssm_spec_write_pos
if spec_write_pos is not None:
fresh_indices = [
idx for idx, fresh in zip(select_index, fresh_req_rows) if fresh
]
if fresh_indices:
spec_write_pos[fresh_indices] = 0
self.mamba_pool.replayssm_cache_base[fresh_indices] = 0
self.mamba_pool.replayssm_is_flush[fresh_indices] = 0
fresh_rows = async_h2d(
fresh_indices, dtype=torch.int64, device=self.device
)
spec_write_pos.index_fill_(0, fresh_rows, 0)
self.mamba_pool.replayssm_cache_base.index_fill_(0, fresh_rows, 0)
self.mamba_pool.replayssm_is_flush.index_fill_(0, fresh_rows, 0)
mamba_indices: list[torch.Tensor] = []
mamba_ping_pong_track_buffers: list[torch.Tensor] = []
@@ -1631,11 +1619,13 @@ class HybridReqToTokenPool(ReqToTokenPool):
assert len(select_index) == len(mamba_ping_pong_track_buffers), (
"Not enough space for mamba ping pong idx, try to increase --mamba-full-memory-ratio."
)
# Device rows: indexing a device tensor with a Python list syncs the host.
rows = async_h2d(select_index, dtype=torch.int64, device=self.device)
mamba_index_tensor = torch.stack(mamba_indices).to(dtype=torch.int32)
self.req_index_to_mamba_index_mapping[select_index] = mamba_index_tensor
self.req_index_to_mamba_index_mapping[rows] = mamba_index_tensor
if self.enable_mamba_extra_buffer:
ping_pong_tensor = torch.stack(mamba_ping_pong_track_buffers)
self.req_index_to_mamba_ping_pong_track_buffer_mapping[select_index] = (
self.req_index_to_mamba_ping_pong_track_buffer_mapping[rows] = (
ping_pong_tensor
)
return select_index
@@ -1779,6 +1769,9 @@ class HybridReqToTokenPool(ReqToTokenPool):
)
buf[:n] = slots
req.kv.mamba_ping_pong_track_buffer = buf
req.kv.mamba_ping_pong_empty = [False] * n + [True] * (
self.mamba_ping_pong_track_buffer_size - n
)
req.kv.mamba_prev_track_seqlen = None
req.kv.mamba_next_track_idx = 0
req.kv.mamba_last_track_idx = (
@@ -1794,10 +1787,17 @@ class HybridReqToTokenPool(ReqToTokenPool):
req_index_to_mamba_ping_pong_track_buffer_mapping in sync so that
set_mamba_track_indices_from_reqs reads correct slot indices.
"""
req.kv.mamba_ping_pong_track_buffer[idx] = value
if torch.is_tensor(value):
req.kv.mamba_ping_pong_track_buffer[idx] = value
else:
assert value == -1, f"expected a slot tensor or -1, got {value!r}"
# fill_ stays on the device; assigning the int is a blocking host copy.
req.kv.mamba_ping_pong_track_buffer[idx].fill_(-1)
self.req_index_to_mamba_ping_pong_track_buffer_mapping[req.kv.req_pool_idx] = (
req.kv.mamba_ping_pong_track_buffer
)
# value is polymorphic: a slot tensor fills the entry, the int -1 empties it.
req.kv.mamba_ping_pong_empty[idx] = not torch.is_tensor(value)
def donate_mamba_ping_pong_slot(
self, req: Req, new_slot: torch.Tensor
@@ -1831,52 +1831,27 @@ class HybridReqToTokenPool(ReqToTokenPool):
req.kv.mamba_pool_idx = None
if self.enable_mamba_extra_buffer:
mamba_ping_pong_track_buffer_to_free = (
self.req_index_to_mamba_ping_pong_track_buffer_mapping[
req.kv.req_pool_idx
]
size = self.mamba_ping_pong_track_buffer_size
keep = mamba_ping_pong_track_buffer_to_keep
assert keep in (None, 0, 1) and (keep != 1 or size == 2), (
f"cannot keep ping-pong entry {keep} of {size}"
)
if mamba_ping_pong_track_buffer_to_keep is not None:
assert mamba_ping_pong_track_buffer_to_keep in [
0,
1,
], (
f"mamba_ping_pong_track_buffer_to_keep must be 0 or 1, {mamba_ping_pong_track_buffer_to_keep=}"
)
# Avoid Python-list advanced indexing on a device tensor.
# The ping-pong buffer size is either 2 (normal) or 1 (spec decode).
if self.mamba_ping_pong_track_buffer_size == 2:
idx_to_free = 1 - mamba_ping_pong_track_buffer_to_keep
mamba_ping_pong_track_buffer_to_free = (
mamba_ping_pong_track_buffer_to_free[
idx_to_free : idx_to_free + 1
]
)
else:
assert self.mamba_ping_pong_track_buffer_size == 1, (
f"Unexpected mamba_ping_pong_track_buffer_size="
f"{self.mamba_ping_pong_track_buffer_size}"
)
assert mamba_ping_pong_track_buffer_to_keep == 0, (
"mamba_ping_pong_track_buffer_to_keep must be 0 when "
"mamba_ping_pong_track_buffer_size is 1"
)
# Keep the only slot, so free nothing.
mamba_ping_pong_track_buffer_to_free = (
mamba_ping_pong_track_buffer_to_free[0:0]
)
if self.enable_mamba_extra_buffer_lazy:
mamba_ping_pong_track_buffer_to_free = (
mamba_ping_pong_track_buffer_to_free[
mamba_ping_pong_track_buffer_to_free != -1
]
)
self.mamba_allocator.free(mamba_ping_pong_track_buffer_to_free)
empty = req.kv.mamba_ping_pong_empty
positions = [p for p in range(size) if p != keep and not empty[p]]
row = self.req_index_to_mamba_ping_pong_track_buffer_mapping[
req.kv.req_pool_idx
]
if len(positions) == size:
self.mamba_allocator.free(row)
else:
for p in positions:
self.mamba_allocator.free(row[p : p + 1])
# Match the req.kv.mamba_pool_idx=None clear above so the next
# alloc() doesn't see a stale ping-pong reference on the req
# and skip allocation (which would silently reuse a freed
# tensor on the req side while the new pool slot leaks).
req.kv.mamba_ping_pong_track_buffer = None
req.kv.mamba_ping_pong_empty = None
req.kv.mamba_next_track_idx = None
req.kv.mamba_last_track_idx = None
req.kv.mamba_last_track_seqlen = None
@@ -1115,6 +1115,7 @@ class UnifiedMambaPool(MambaPool):
self.replayssm_is_kda = False
self.enable_linear_replayssm_spec = False
self.replayssm_spec_fold = False
self.replayssm_spec_write_pos = None
self.replayssm_cache_base = None
self.replayssm_is_flush = None
self.debug_memory_pool = False
@@ -255,9 +255,12 @@ class StreamingSession:
mamba_allocator.free(slot.kv.mamba_pool_idx.unsqueeze(0))
slot.kv.mamba_pool_idx = None
if slot.kv.mamba_ping_pong_track_buffer is not None:
indices = slot.kv.mamba_ping_pong_track_buffer
mamba_allocator.free(indices[indices != -1])
buffer = slot.kv.mamba_ping_pong_track_buffer
for position, empty in enumerate(slot.kv.mamba_ping_pong_empty):
if not empty:
mamba_allocator.free(buffer[position : position + 1])
slot.kv.mamba_ping_pong_track_buffer = None
slot.kv.mamba_ping_pong_empty = None
# -- Internal helpers (streaming body bits) --