From 740d5f9f9e33aeebc6b5bcb1ba73f25bf6db062b Mon Sep 17 00:00:00 2001 From: metamergebot Date: Wed, 21 Oct 2026 11:56:50 -0700 Subject: [PATCH] [Perf] scheduler: Plan hybrid Mamba extend batches without host-device syncs (#45472) Co-authored-by: Lu Fang <116399278+luccafong@users.noreply.github.com> Co-authored-by: Shiyan Deng <20773414+842974287@users.noreply.github.com> --- .../mlx/kv_cache/auxiliary_state.py | 3 + .../srt/hardware_backend/mlx/model_runner.py | 1 + python/sglang/srt/managers/mm_schedule.py | 10 +- python/sglang/srt/managers/mm_utils.py | 5 +- python/sglang/srt/managers/schedule_batch.py | 72 +++++------ .../batch_result_processor.py | 9 +- .../pool_stats_observer.py | 2 +- python/sglang/srt/mem_cache/memory_pool.py | 115 +++++++----------- .../srt/mem_cache/unified_memory_pool.py | 1 + .../sglang/srt/session/streaming_session.py | 7 +- .../test_mm_chunked_embedding_unit.py | 28 +++++ ...t_batch_result_processor_mamba_boundary.py | 17 +-- .../managers/test_mamba_checkpoint_depth.py | 41 +++++-- .../unit/mem_cache/test_unified_tri_pool.py | 47 +++++++ 14 files changed, 217 insertions(+), 141 deletions(-) diff --git a/python/sglang/srt/hardware_backend/mlx/kv_cache/auxiliary_state.py b/python/sglang/srt/hardware_backend/mlx/kv_cache/auxiliary_state.py index 85bd3a1996..0f4d27d94d 100644 --- a/python/sglang/srt/hardware_backend/mlx/kv_cache/auxiliary_state.py +++ b/python/sglang/srt/hardware_backend/mlx/kv_cache/auxiliary_state.py @@ -286,6 +286,7 @@ class MlxAuxiliaryStateReqToTokenPool(ReqToTokenPool): if mamba_ping_pong_track_buffer_to_keep is None: self.auxiliary_state_pool.free(track_buffer) req.kv.mamba_ping_pong_track_buffer = None + req.kv.mamba_ping_pong_empty = None req.kv.mamba_next_track_idx = None req.kv.mamba_last_track_idx = None @@ -387,6 +388,7 @@ class MlxAuxiliaryStateComponent(MambaComponent): if track_buffer is not None: self.cache.req_to_token_pool.auxiliary_state_pool.free(track_buffer) req.kv.mamba_ping_pong_track_buffer = None + req.kv.mamba_ping_pong_empty = None req.kv.mamba_next_track_idx = None req.kv.mamba_last_track_idx = None req.kv.mamba_last_track_seqlen = None @@ -412,6 +414,7 @@ class MlxAuxiliaryStateComponent(MambaComponent): if track_buffer is not None: self.cache.req_to_token_pool.auxiliary_state_pool.free(track_buffer) req.kv.mamba_ping_pong_track_buffer = None + req.kv.mamba_ping_pong_empty = None req.kv.mamba_next_track_idx = None req.kv.mamba_last_track_idx = None req.kv.mamba_pool_idx = None diff --git a/python/sglang/srt/hardware_backend/mlx/model_runner.py b/python/sglang/srt/hardware_backend/mlx/model_runner.py index 703e6b70be..90d74690fb 100644 --- a/python/sglang/srt/hardware_backend/mlx/model_runner.py +++ b/python/sglang/srt/hardware_backend/mlx/model_runner.py @@ -417,6 +417,7 @@ class MlxModelRunner: ) return req.kv.mamba_ping_pong_track_buffer = track_buffer + req.kv.mamba_ping_pong_empty = [False] req.kv.mamba_next_track_idx = 0 req.kv.mamba_last_track_idx = 0 diff --git a/python/sglang/srt/managers/mm_schedule.py b/python/sglang/srt/managers/mm_schedule.py index a003efb893..c5818e07b0 100644 --- a/python/sglang/srt/managers/mm_schedule.py +++ b/python/sglang/srt/managers/mm_schedule.py @@ -655,7 +655,15 @@ def _get_chunked_prefill_embedding( def _get_multimodal_mask( input_ids: torch.Tensor, placeholder_tensor: torch.Tensor ) -> torch.Tensor: - return torch.isin(input_ids, placeholder_tensor).unsqueeze(-1) + # placeholder_tensor arrives sorted from embed_mm_inputs. torch.isin + # brute-forces only while P < 10 * N**0.145 (P placeholders, N ids); past + # that it sorts with unique(), whose count syncs the host. The sorted + # lookup never syncs. + assert placeholder_tensor.numel() > 0 + placeholders = placeholder_tensor.to(input_ids.dtype) + idx = torch.searchsorted(placeholders, input_ids) + idx.clamp_(max=placeholders.numel() - 1) + return (placeholders[idx] == input_ids).unsqueeze(-1) def _count_mm_tokens_in_extend( diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index d4a8346afd..33f02ecdcc 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -495,7 +495,10 @@ def embed_mm_inputs( embedder = getattr(multimodal_model, f"get_{modality_id}_feature", None) if len(items) != 0: assert embedder is not None, f"no embedding method found for {modality}" - pad_values = [value for item in items for value in item.padding_values()] + # Sorted for the searchsorted lookup in _get_multimodal_mask. + pad_values = sorted( + value for item in items for value in item.padding_values() + ) if input_ids.device.type == "cuda": # Pinned staging keeps the placeholder copy asynchronous on CUDA. placeholder_cpu = torch.tensor( diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index de85d52138..0133caafcf 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -15,6 +15,7 @@ from sglang.srt.runtime_context import ( mamba_track_grid, ) from sglang.srt.utils.common import ( + async_h2d, ceil_align, flatten_arrays_to_pinned_cpu, is_hip, @@ -996,6 +997,8 @@ class ReqKvInfo: # Mamba state: an independent resource; whether it is held is `holds_mamba`. mamba_pool_idx: Optional[torch.Tensor] = None # shape (1) mamba_ping_pong_track_buffer: Optional[torch.Tensor] = None # shape (2) + # Per entry of the ping-pong buffer: True where it holds no slot (-1). + mamba_ping_pong_empty: Optional[list[bool]] = None mamba_next_track_idx: Optional[int] = None # 0 or 1 mamba_last_track_idx: Optional[int] = None # 0 or 1 # Seq len of the last cached mamba state @@ -2091,6 +2094,7 @@ class Req(ReqDllmMixin): self.inflight_middle_chunks = 0 self.kv.mamba_pool_idx = None self.kv.mamba_ping_pong_track_buffer = None + self.kv.mamba_ping_pong_empty = None self.kv.mamba_next_track_idx = None self.kv.mamba_last_track_idx = None self.kv.mamba_last_track_seqlen = None @@ -2286,10 +2290,19 @@ class Req(ReqDllmMixin): class _MambaRadixCacheV2TrackEntry(NamedTuple): track_mask: bool - track_index: int + track_position: int track_seqlen: int +def _gather_mamba_track_indices( + ping_pong_mapping: torch.Tensor, + req_pool_indices: torch.Tensor, + track_positions: torch.Tensor, +) -> torch.Tensor: + """Each request's ping-pong slot id at its track position, read on the device.""" + return ping_pong_mapping[req_pool_indices, track_positions] + + def mamba_lazy_spec_in_window( req, mamba_track_interval: int, max_draft_tokens: int ) -> bool: @@ -2311,10 +2324,7 @@ def set_mamba_track_indices_from_reqs( track_positions: optional per-req ping-pong position override (the lazy spec track plan, see mamba_lazy_spec_prepare). """ - req_to_token_pool = batch.req_to_token_pool - all_buffers = req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping[ - batch.req_pool_indices - ] # (bs, ping_pong_size), int64, on device + mapping = batch.req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping if track_positions is None: # Guard: mamba_next_track_idx may be None for requests that haven't # gone through _alloc_ping_pong_buffer yet (e.g., spec v2 verify path). @@ -2332,17 +2342,9 @@ def set_mamba_track_indices_from_reqs( i for i, req in enumerate(batch.reqs) if req.kv.mamba_next_track_idx is None ] batch.mamba_track_buffer_indices = list(track_positions) - idx = ( - torch.tensor( - track_positions, - dtype=torch.int64, - pin_memory=True, - ) - .unsqueeze(1) - .to(device=all_buffers.device, non_blocking=True) - ) - batch.mamba_track_indices = ( - torch.gather(all_buffers, 1, idx).squeeze(1).to(torch.int64) + positions = async_h2d(track_positions, dtype=torch.int64, device=mapping.device) + batch.mamba_track_indices = _gather_mamba_track_indices( + mapping, batch.req_pool_indices, positions ) if freed_rows: # Overlap can leave a request in TARGET_VERIFY after its Mamba state has @@ -2918,7 +2920,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): extend_input_logprob_token_ids = [] multimodal_inputs = [] mamba_track_mask_cpu = [] - mamba_track_indices_cpu = [] + mamba_track_positions_cpu = [] mamba_track_seqlens_cpu = [] for i, (req, seq_len, pre_len) in enumerate(zip(reqs, seq_lens, prefix_lens)): @@ -2991,7 +2993,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): if get_exec().mamba.enable_mamba_extra_buffer: track_entry = self._mamba_radix_cache_v2_req_prepare_for_extend(req) mamba_track_mask_cpu.append(track_entry.track_mask) - mamba_track_indices_cpu.append(track_entry.track_index) + mamba_track_positions_cpu.append(track_entry.track_position) mamba_track_seqlens_cpu.append(track_entry.track_seqlen) if self.return_logprob: @@ -3098,21 +3100,21 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): if get_exec().mamba.enable_mamba_extra_buffer: self.mamba_prefill_track_mask_cpu = mamba_track_mask_cpu self.mamba_track_seqlens_cpu = mamba_track_seqlens_cpu - self.mamba_track_indices = torch.tensor( - mamba_track_indices_cpu, - dtype=torch.int64, - device=self.device, + # Ping-pong slot ids live on the device; gather them there. + track_positions = torch.tensor( + mamba_track_positions_cpu, dtype=torch.int64, pin_memory=_pin + ).to(self.device, non_blocking=True) + self.mamba_track_indices = _gather_mamba_track_indices( + self.req_to_token_pool.req_index_to_mamba_ping_pong_track_buffer_mapping, + self.req_pool_indices, + track_positions, ) self.mamba_track_mask = torch.tensor( - mamba_track_mask_cpu, - dtype=torch.bool, - device=self.device, - ) + mamba_track_mask_cpu, dtype=torch.bool, pin_memory=_pin + ).to(self.device, non_blocking=True) self.mamba_track_seqlens = torch.tensor( - mamba_track_seqlens_cpu, - dtype=torch.int64, - device=self.device, - ) + mamba_track_seqlens_cpu, dtype=torch.int64, pin_memory=_pin + ).to(self.device, non_blocking=True) # Collect mamba init info for deferred ops on forward stream if any(req.kv.holds_mamba for req in reqs): @@ -3174,9 +3176,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): mamba_track_seqlen_aligned = ( prefix_len + (req.extend_len // checkpoint_grid) * checkpoint_grid ) - track_index = req.kv.mamba_ping_pong_track_buffer[ - req.kv.mamba_next_track_idx - ].item() + track_position = req.kv.mamba_next_track_idx mamba_track_seqlen = -1 if mask: # mamba_track_seqlen is used to calculate the indices to track in @@ -3228,7 +3228,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): return _MambaRadixCacheV2TrackEntry( track_mask=mask, - track_index=track_index, + track_position=track_position, track_seqlen=mamba_track_seqlen, ) @@ -3627,7 +3627,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): if self.seq_lens_cpu[i].item() % mamba_track_interval != 0: continue other_idx = 1 - req.kv.mamba_next_track_idx - if buf[other_idx].item() != -1: + if not req.kv.mamba_ping_pong_empty[other_idx]: # With overlap the previous forward's post-processing # (which frees this slot) hasn't run yet. Skip. continue @@ -3688,7 +3688,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): track_positions.append(req.kv.mamba_next_track_idx) continue other_idx = 1 - req.kv.mamba_next_track_idx - has_pending = buf[other_idx].item() != -1 + has_pending = not req.kv.mamba_ping_pong_empty[other_idx] if not has_pending: if envs.SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL.get(): new_slot = None diff --git a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py index dbdf4d6dcc..59aa51413d 100644 --- a/python/sglang/srt/managers/scheduler_components/batch_result_processor.py +++ b/python/sglang/srt/managers/scheduler_components/batch_result_processor.py @@ -1590,9 +1590,9 @@ class SchedulerBatchResultProcessor: other_idx = 1 - req.kv.mamba_next_track_idx # Recompute the in-flight verify's plan (kv_committed_len is # frozen since its prepare, so the recompute is exact). - keep_may_be_written_in_flight = req.kv.mamba_ping_pong_track_buffer[ + keep_may_be_written_in_flight = req.kv.mamba_ping_pong_empty[ other_idx - ].item() == -1 and mamba_lazy_spec_in_window( + ] and mamba_lazy_spec_in_window( req, mamba_track_grid(self.tree_cache.page_size), max_speculative_num_draft_tokens(), @@ -1677,9 +1677,8 @@ class SchedulerBatchResultProcessor: req.kv.mamba_last_track_idx = track_idx req.kv.mamba_next_track_idx = track_idx other_idx = 1 - track_idx - other_val = req.kv.mamba_ping_pong_track_buffer[other_idx].item() - if other_val != -1: - pool = batch.req_to_token_pool + pool = batch.req_to_token_pool + if not req.kv.mamba_ping_pong_empty[other_idx]: pool.mamba_allocator.free( req.kv.mamba_ping_pong_track_buffer[other_idx].unsqueeze(0) ) diff --git a/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py b/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py index 1c29d6e06b..8711e58120 100644 --- a/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py +++ b/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py @@ -45,7 +45,7 @@ def kv_private_swa_tokens(kv: ReqKvInfo, page_size: int) -> int: def kv_mamba_slots(kv: ReqKvInfo) -> int: total = kv.mamba_pool_idx.numel() if kv.holds_mamba else 0 if kv.mamba_ping_pong_track_buffer is not None: - total += int((kv.mamba_ping_pong_track_buffer != -1).sum().item()) + total += kv.mamba_ping_pong_empty.count(False) return total diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 23eeaacea8..53b271a1c8 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -95,6 +95,7 @@ from sglang.srt.utils import ( from sglang.srt.utils.async_probe import ( maybe_detect_oob, ) +from sglang.srt.utils.common import async_h2d from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter if TYPE_CHECKING: @@ -1146,33 +1147,17 @@ class MambaPool: """Zero out mamba state at the given pool indices. Must run on forward stream.""" for sibling in self._slot_siblings: sibling.reset_slots(indices) + idx64 = indices.to(torch.int64) if self._should_fuse_slot_ops(): from sglang.srt.mem_cache.mamba_slot_fused import fused_clear_conv_slots - fused_clear_conv_slots(self._conv_slot_desc, indices) - temporal = self.mamba_cache.temporal - if temporal.numel() > 0: - temporal[:, indices] = 0 - return - if not _is_npu: - need_size = len(indices) - for i in range(len(self.mamba_cache.conv)): - t = self.mamba_cache.conv[i] - z = torch.zeros(1, dtype=t.dtype, device=t.device).expand( - t.shape[0], need_size, *t.shape[2:] - ) - t[:, indices] = z - t = self.mamba_cache.temporal - z = torch.zeros(1, dtype=t.dtype, device=t.device).expand( - t.shape[0], need_size, *t.shape[2:] - ) - t[:, indices] = z + fused_clear_conv_slots(self._conv_slot_desc, idx64) else: - for i in range(len(self.mamba_cache.conv)): - t = self.mamba_cache.conv[i] - t[:, indices] = 0 - t = self.mamba_cache.temporal - t[:, indices] = 0 + for conv in self.mamba_cache.conv: + conv.index_fill_(1, idx64, 0) + temporal = self.mamba_cache.temporal + if temporal.numel() > 0: + temporal.index_fill_(1, idx64, 0) def copy_from(self, src_indices: torch.Tensor, dst_indices: torch.Tensor): """Clone mamba state (conv + temporal) from src slots into dst slots. @@ -1589,15 +1574,18 @@ class HybridReqToTokenPool(ReqToTokenPool): if select_index is None: return None - spec_write_pos = getattr(self.mamba_pool, "replayssm_spec_write_pos", None) + spec_write_pos = self.mamba_pool.replayssm_spec_write_pos if spec_write_pos is not None: fresh_indices = [ idx for idx, fresh in zip(select_index, fresh_req_rows) if fresh ] if fresh_indices: - spec_write_pos[fresh_indices] = 0 - self.mamba_pool.replayssm_cache_base[fresh_indices] = 0 - self.mamba_pool.replayssm_is_flush[fresh_indices] = 0 + fresh_rows = async_h2d( + fresh_indices, dtype=torch.int64, device=self.device + ) + spec_write_pos.index_fill_(0, fresh_rows, 0) + self.mamba_pool.replayssm_cache_base.index_fill_(0, fresh_rows, 0) + self.mamba_pool.replayssm_is_flush.index_fill_(0, fresh_rows, 0) mamba_indices: list[torch.Tensor] = [] mamba_ping_pong_track_buffers: list[torch.Tensor] = [] @@ -1631,11 +1619,13 @@ class HybridReqToTokenPool(ReqToTokenPool): assert len(select_index) == len(mamba_ping_pong_track_buffers), ( "Not enough space for mamba ping pong idx, try to increase --mamba-full-memory-ratio." ) + # Device rows: indexing a device tensor with a Python list syncs the host. + rows = async_h2d(select_index, dtype=torch.int64, device=self.device) mamba_index_tensor = torch.stack(mamba_indices).to(dtype=torch.int32) - self.req_index_to_mamba_index_mapping[select_index] = mamba_index_tensor + self.req_index_to_mamba_index_mapping[rows] = mamba_index_tensor if self.enable_mamba_extra_buffer: ping_pong_tensor = torch.stack(mamba_ping_pong_track_buffers) - self.req_index_to_mamba_ping_pong_track_buffer_mapping[select_index] = ( + self.req_index_to_mamba_ping_pong_track_buffer_mapping[rows] = ( ping_pong_tensor ) return select_index @@ -1779,6 +1769,9 @@ class HybridReqToTokenPool(ReqToTokenPool): ) buf[:n] = slots req.kv.mamba_ping_pong_track_buffer = buf + req.kv.mamba_ping_pong_empty = [False] * n + [True] * ( + self.mamba_ping_pong_track_buffer_size - n + ) req.kv.mamba_prev_track_seqlen = None req.kv.mamba_next_track_idx = 0 req.kv.mamba_last_track_idx = ( @@ -1794,10 +1787,17 @@ class HybridReqToTokenPool(ReqToTokenPool): req_index_to_mamba_ping_pong_track_buffer_mapping in sync so that set_mamba_track_indices_from_reqs reads correct slot indices. """ - req.kv.mamba_ping_pong_track_buffer[idx] = value + if torch.is_tensor(value): + req.kv.mamba_ping_pong_track_buffer[idx] = value + else: + assert value == -1, f"expected a slot tensor or -1, got {value!r}" + # fill_ stays on the device; assigning the int is a blocking host copy. + req.kv.mamba_ping_pong_track_buffer[idx].fill_(-1) self.req_index_to_mamba_ping_pong_track_buffer_mapping[req.kv.req_pool_idx] = ( req.kv.mamba_ping_pong_track_buffer ) + # value is polymorphic: a slot tensor fills the entry, the int -1 empties it. + req.kv.mamba_ping_pong_empty[idx] = not torch.is_tensor(value) def donate_mamba_ping_pong_slot( self, req: Req, new_slot: torch.Tensor @@ -1831,52 +1831,27 @@ class HybridReqToTokenPool(ReqToTokenPool): req.kv.mamba_pool_idx = None if self.enable_mamba_extra_buffer: - mamba_ping_pong_track_buffer_to_free = ( - self.req_index_to_mamba_ping_pong_track_buffer_mapping[ - req.kv.req_pool_idx - ] + size = self.mamba_ping_pong_track_buffer_size + keep = mamba_ping_pong_track_buffer_to_keep + assert keep in (None, 0, 1) and (keep != 1 or size == 2), ( + f"cannot keep ping-pong entry {keep} of {size}" ) - if mamba_ping_pong_track_buffer_to_keep is not None: - assert mamba_ping_pong_track_buffer_to_keep in [ - 0, - 1, - ], ( - f"mamba_ping_pong_track_buffer_to_keep must be 0 or 1, {mamba_ping_pong_track_buffer_to_keep=}" - ) - # Avoid Python-list advanced indexing on a device tensor. - # The ping-pong buffer size is either 2 (normal) or 1 (spec decode). - if self.mamba_ping_pong_track_buffer_size == 2: - idx_to_free = 1 - mamba_ping_pong_track_buffer_to_keep - mamba_ping_pong_track_buffer_to_free = ( - mamba_ping_pong_track_buffer_to_free[ - idx_to_free : idx_to_free + 1 - ] - ) - else: - assert self.mamba_ping_pong_track_buffer_size == 1, ( - f"Unexpected mamba_ping_pong_track_buffer_size=" - f"{self.mamba_ping_pong_track_buffer_size}" - ) - assert mamba_ping_pong_track_buffer_to_keep == 0, ( - "mamba_ping_pong_track_buffer_to_keep must be 0 when " - "mamba_ping_pong_track_buffer_size is 1" - ) - # Keep the only slot, so free nothing. - mamba_ping_pong_track_buffer_to_free = ( - mamba_ping_pong_track_buffer_to_free[0:0] - ) - if self.enable_mamba_extra_buffer_lazy: - mamba_ping_pong_track_buffer_to_free = ( - mamba_ping_pong_track_buffer_to_free[ - mamba_ping_pong_track_buffer_to_free != -1 - ] - ) - self.mamba_allocator.free(mamba_ping_pong_track_buffer_to_free) + empty = req.kv.mamba_ping_pong_empty + positions = [p for p in range(size) if p != keep and not empty[p]] + row = self.req_index_to_mamba_ping_pong_track_buffer_mapping[ + req.kv.req_pool_idx + ] + if len(positions) == size: + self.mamba_allocator.free(row) + else: + for p in positions: + self.mamba_allocator.free(row[p : p + 1]) # Match the req.kv.mamba_pool_idx=None clear above so the next # alloc() doesn't see a stale ping-pong reference on the req # and skip allocation (which would silently reuse a freed # tensor on the req side while the new pool slot leaks). req.kv.mamba_ping_pong_track_buffer = None + req.kv.mamba_ping_pong_empty = None req.kv.mamba_next_track_idx = None req.kv.mamba_last_track_idx = None req.kv.mamba_last_track_seqlen = None diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index f5820ff544..8abdf41695 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -1115,6 +1115,7 @@ class UnifiedMambaPool(MambaPool): self.replayssm_is_kda = False self.enable_linear_replayssm_spec = False self.replayssm_spec_fold = False + self.replayssm_spec_write_pos = None self.replayssm_cache_base = None self.replayssm_is_flush = None self.debug_memory_pool = False diff --git a/python/sglang/srt/session/streaming_session.py b/python/sglang/srt/session/streaming_session.py index d3ac6823db..280c36acf3 100644 --- a/python/sglang/srt/session/streaming_session.py +++ b/python/sglang/srt/session/streaming_session.py @@ -255,9 +255,12 @@ class StreamingSession: mamba_allocator.free(slot.kv.mamba_pool_idx.unsqueeze(0)) slot.kv.mamba_pool_idx = None if slot.kv.mamba_ping_pong_track_buffer is not None: - indices = slot.kv.mamba_ping_pong_track_buffer - mamba_allocator.free(indices[indices != -1]) + buffer = slot.kv.mamba_ping_pong_track_buffer + for position, empty in enumerate(slot.kv.mamba_ping_pong_empty): + if not empty: + mamba_allocator.free(buffer[position : position + 1]) slot.kv.mamba_ping_pong_track_buffer = None + slot.kv.mamba_ping_pong_empty = None # -- Internal helpers (streaming body bits) -- diff --git a/test/registered/chunked_prefill/test_mm_chunked_embedding_unit.py b/test/registered/chunked_prefill/test_mm_chunked_embedding_unit.py index d03811cc5e..b2079cdaa7 100644 --- a/test/registered/chunked_prefill/test_mm_chunked_embedding_unit.py +++ b/test/registered/chunked_prefill/test_mm_chunked_embedding_unit.py @@ -482,6 +482,34 @@ def test_precomputed_audio_slices_concatenate_once(monkeypatch, prefix, length, concat.assert_called_once() +@pytest.mark.parametrize( + "n_ids,n_placeholders,ids_dtype", + [ + (1, 1, torch.int64), + (64, 3, torch.int64), + (4096, 800, torch.int64), + (16384, 800, torch.int32), + ], +) +def test_multimodal_mask_matches_broadcast_compare(n_ids, n_placeholders, ids_dtype): + gen = torch.Generator().manual_seed(n_ids) + pad_values = torch.randperm(4 * n_placeholders + 8, generator=gen)[ + :n_placeholders + ].tolist() + # Host-sorted placeholders with a repeat, as embed_mm_inputs builds them; ids + # below, between and above them. + placeholders = torch.tensor(sorted(pad_values + pad_values[:1])) + input_ids = torch.randint(-2, 4 * n_placeholders + 10, (n_ids,), generator=gen).to( + ids_dtype + ) + + mask = mm_schedule._get_multimodal_mask(input_ids, placeholders) + + broadcast = (input_ids.unsqueeze(-1) == placeholders).any(-1).unsqueeze(-1) + assert torch.equal(mask, broadcast) + assert torch.equal(mask, torch.isin(input_ids, placeholders).unsqueeze(-1)) + + if __name__ == "__main__": import sys diff --git a/test/registered/unit/managers/test_batch_result_processor_mamba_boundary.py b/test/registered/unit/managers/test_batch_result_processor_mamba_boundary.py index cb3b0e2ce1..0549d12dd2 100644 --- a/test/registered/unit/managers/test_batch_result_processor_mamba_boundary.py +++ b/test/registered/unit/managers/test_batch_result_processor_mamba_boundary.py @@ -27,20 +27,6 @@ register_cpu_ci(est_time=12, suite="base-a-test-cpu") TRACK_INTERVAL = 4 -def _set_mamba_track_indices_on_cpu(batch, track_positions=None): - real_tensor = torch.tensor - - def tensor_without_pinning(*args, **kwargs): - kwargs["pin_memory"] = False - return real_tensor(*args, **kwargs) - - with patch( - "sglang.srt.managers.schedule_batch.torch.tensor", - side_effect=tensor_without_pinning, - ): - set_mamba_track_indices_from_reqs(batch, track_positions) - - def _make_batch() -> tuple[Req, ScheduleBatch]: sampling_params = SamplingParams(max_new_tokens=32) sampling_params.normalize(None) @@ -202,6 +188,7 @@ class TestMambaPrevTrackSeqlen(unittest.TestCase): req, batch = _make_batch() processor = _make_processor() req.kv.mamba_ping_pong_track_buffer = torch.tensor([10, 11]) + req.kv.mamba_ping_pong_empty = [False, False] req.kv.mamba_next_track_idx = 0 req.kv.mamba_last_track_idx = 1 batch.mamba_track_buffer_indices = None @@ -242,7 +229,7 @@ class TestMambaFreedTrackIndicesCPU(unittest.TestCase): req_index_to_mamba_ping_pong_track_buffer_mapping=mapping ), ) - _set_mamba_track_indices_on_cpu(batch, positions) + set_mamba_track_indices_from_reqs(batch, positions) self.assertEqual(batch.mamba_track_indices.tolist(), expected) diff --git a/test/registered/unit/managers/test_mamba_checkpoint_depth.py b/test/registered/unit/managers/test_mamba_checkpoint_depth.py index addb5d90ea..fcf30330bb 100644 --- a/test/registered/unit/managers/test_mamba_checkpoint_depth.py +++ b/test/registered/unit/managers/test_mamba_checkpoint_depth.py @@ -14,9 +14,11 @@ from array import array from types import SimpleNamespace from unittest.mock import MagicMock, patch -import torch - -from sglang.srt.managers.schedule_batch import Req, ScheduleBatch +from sglang.srt.managers.schedule_batch import ( + Req, + ScheduleBatch, + _MambaRadixCacheV2TrackEntry, +) from sglang.srt.runtime_context import get_context, mamba_track_grid from sglang.srt.sampling.sampling_params import SamplingParams from sglang.srt.server_args import ( @@ -30,15 +32,15 @@ register_cpu_ci(est_time=10, suite="base-a-test-cpu") CHUNK = 64 -def _track_seqlen( +def _prepare_extend( *, tree_page: int, prefix_len: int, extend_len: int, dcp_enabled: bool = False, -) -> int | None: - """Run one extend through the tracker and report the donated depth, or - None when the extend donates no checkpoint.""" + next_idx: int = 0, +) -> tuple[_MambaRadixCacheV2TrackEntry, Req]: + """Run one extend through the tracker; return its track entry and the request.""" server_args = ServerArgs(model_path="dummy", page_size=CHUNK) # The property would otherwise load the HF config for the dummy model. server_args._mamba_cache_chunk_size = CHUNK @@ -55,8 +57,7 @@ def _track_seqlen( ) req.prefix_len = prefix_len req.extend_end = prefix_len + extend_len - req.kv.mamba_ping_pong_track_buffer = torch.tensor([0, 1], dtype=torch.int64) - req.kv.mamba_next_track_idx = 0 + req.kv.mamba_next_track_idx = next_idx req.mamba_branching_seqlen = None batch = ScheduleBatch(reqs=[req]) @@ -65,13 +66,19 @@ def _track_seqlen( ) batch.tree_cache = SimpleNamespace(page_size=tree_page) batch.req_to_token_pool = MagicMock() - batch.req_to_token_pool.get_mamba_ping_pong_other_idx.return_value = 1 + batch.req_to_token_pool.get_mamba_ping_pong_other_idx.return_value = 1 - next_idx with patch( "sglang.srt.managers.schedule_batch.get_parallel", return_value=SimpleNamespace(dcp_enabled=dcp_enabled), ): entry = batch._mamba_radix_cache_v2_req_prepare_for_extend(req) + return entry, req + + +def _track_seqlen(**kwargs) -> int | None: + """The donated depth of one extend, or None when it donates no checkpoint.""" + entry, req = _prepare_extend(**kwargs) if not entry.track_mask: assert req.kv.mamba_last_track_seqlen is None return None @@ -124,6 +131,20 @@ class TestMambaCheckpointDepth(unittest.TestCase): ) +class TestMambaTrackPosition(unittest.TestCase): + def test_track_position_is_taken_before_the_ping_pong_swap(self): + # prepare_for_extend gathers the slot id at this position; the read it + # replaced took buffer[mamba_next_track_idx] before the boundary swap. + for next_idx in (0, 1): + with self.subTest(next_idx=next_idx): + entry, req = _prepare_extend( + tree_page=CHUNK, prefix_len=0, extend_len=96, next_idx=next_idx + ) + self.assertTrue(entry.track_mask) + self.assertEqual(entry.track_position, next_idx) + self.assertEqual(req.kv.mamba_next_track_idx, 1 - next_idx) + + class TestMambaTrackGrid(unittest.TestCase): def _grid(self, *, interval: int, tree_page: int, chunk: int = CHUNK) -> int: with get_context().override_server_args( diff --git a/test/registered/unit/mem_cache/test_unified_tri_pool.py b/test/registered/unit/mem_cache/test_unified_tri_pool.py index 22bc05cf7b..cb10817292 100644 --- a/test/registered/unit/mem_cache/test_unified_tri_pool.py +++ b/test/registered/unit/mem_cache/test_unified_tri_pool.py @@ -1027,6 +1027,7 @@ class TestTriFactorySizing(unittest.TestCase): req_pool_idx=1, mamba_pool_idx=live[0], mamba_ping_pong_track_buffer=track, + mamba_ping_pong_empty=[index == -1 for index in ids], ) ) session = StreamingSession(SimpleNamespace(req_to_token_pool=pool)) @@ -1189,6 +1190,52 @@ class TestTriFactorySizing(unittest.TestCase): allocator.free(retained) self.assertEqual(allocator.available_size(), initial_available) + def test_ping_pong_mapping_mirrors_the_request_buffers(self): + from types import SimpleNamespace + + from sglang.srt.managers.schedule_batch import ( + ReqKvInfo, + _gather_mamba_track_indices, + ) + from sglang.srt.runtime_context import get_parallel + + for lazy in (False, True): + with self.subTest(lazy=lazy): + kw = self._factory_kwargs( + enable_mamba_extra_buffer=True, + enable_mamba_extra_buffer_lazy=lazy, + disable_overlap_schedule=False, + max_mamba_cache_size=16, + ) + with get_parallel().override(attn_dcp_size=1): + pool = init_unified_mamba_swa_pools(**kw).req_to_token_pool + mapping = pool.req_index_to_mamba_ping_pong_track_buffer_mapping + allocator = pool.mamba_allocator + reqs = [SimpleNamespace(kv=ReqKvInfo()) for _ in range(3)] + rows = pool.alloc(reqs) + + def buffers(): + return [ + req.kv.mamba_ping_pong_track_buffer.tolist() for req in reqs + ] + + self.assertEqual(mapping[rows].tolist(), buffers()) + # Every writer of a buffer entry rewrites the request's row. + pool.donate_mamba_ping_pong_slot(reqs[0], allocator.alloc(1)) + pool.set_mamba_ping_pong_slot(reqs[1], 0, -1) + pool.set_mamba_ping_pong_slot(reqs[1], 1, allocator.alloc(1)[0]) + self.assertEqual(mapping[rows].tolist(), buffers()) + # The device gather returns what the per-request .item() read did. + positions = [0, 0, 1] + gathered = _gather_mamba_track_indices( + mapping, torch.tensor(rows), torch.tensor(positions) + ) + item_reads = [ + req.kv.mamba_ping_pong_track_buffer[pos].item() + for req, pos in zip(reqs, positions) + ] + self.assertEqual(gathered.tolist(), item_reads) + def test_budget_sizing_and_boot_signature(self): from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedSWAAllocatorBase,