From c10fa03fc43180fcae072c6535f6c6d9c33c3ef1 Mon Sep 17 00:00:00 2001 From: Lianmin Zheng Date: Sat, 26 Sep 2026 11:24:02 -0700 Subject: [PATCH] Make sliding-window caching and speculative batch padding extensible (#41325) Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com> Co-authored-by: jasonjk-park --- python/sglang/srt/managers/schedule_batch.py | 28 +- .../sglang/srt/mem_cache/base_prefix_cache.py | 21 ++ python/sglang/srt/mem_cache/common.py | 35 ++- python/sglang/srt/mem_cache/registry.py | 17 +- .../sglang/srt/mem_cache/swa_memory_pool.py | 6 +- .../mem_cache/unified_cache/components/swa.py | 241 +++++++++--------- .../unified_cache/unified_tree_core.py | 22 ++ .../unified_tree_core_interface.py | 13 + .../srt/mem_cache/unified_memory_pool.py | 4 +- .../srt/model_executor/forward_batch_info.py | 15 +- .../srt/model_executor/pool_configurator.py | 12 +- python/sglang/srt/speculative/eagle_info.py | 23 +- python/sglang/srt/speculative/spec_info.py | 7 + .../unit/mem_cache/test_registry.py | 18 ++ .../mem_cache/test_swa_cpu_copy_filter.py | 8 +- .../mem_cache/test_swa_eviction_interval.py | 129 ++++++++++ .../model_executor/test_mlp_sync_pad_unpad.py | 10 +- 17 files changed, 430 insertions(+), 179 deletions(-) create mode 100644 test/registered/unit/mem_cache/test_swa_eviction_interval.py diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 3394c7751e..f090361921 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -115,7 +115,6 @@ from sglang.srt.mem_cache.common import ( RetractionBackup, backup_kv_cache, evict_from_tree_cache, - free_swa_out_of_window_slots, release_kv_cache, ) from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool @@ -3882,6 +3881,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): def maybe_evict_swa(self): if self.tree_cache.supports_swa(): sliding_window_size = self.tree_cache.sliding_window_size + # Auxiliary windows check their own cursors and prefix locks. + has_auxiliary_swa = self.tree_cache.supports_auxiliary_swa() release_leaf_lock = ( envs.SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW.get() @@ -3901,9 +3902,17 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): if ( req.decode_batch_idx >= 1 and req.kv.holds_kv - and swa_evict_due(req) + and (has_auxiliary_swa or swa_evict_due(req)) ): - self._evict_swa(req, req.seqlen - 1) + # The scheduler already gated the primary window, + # including backends that use a forward-based cadence. + self._evict_swa( + req, + req.seqlen - 1, + eviction_interval=( + eviction_interval if has_auxiliary_swa else 1 + ), + ) # Once the decode position has moved past the sliding window, # the SWA portion of the prefill-time tree lock is no longer @@ -3938,17 +3947,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): self._evict_swa(req, pre_len) self.token_to_kv_pool_allocator.free_group_end() - def _evict_swa(self, req: Req, pre_len: int): + def _evict_swa(self, req: Req, pre_len: int, *, eviction_interval: int = 1): assert self.tree_cache.supports_swa(), "prefix cache must support swa" - free_swa_out_of_window_slots( - req, - pre_len, - sliding_window_size=self.tree_cache.sliding_window_size, - page_size=self.tree_cache.page_size, - req_to_token_pool=self.req_to_token_pool, - token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, - is_chunk_cache=self.tree_cache.is_chunk_cache(), - retain_floor=self.tree_cache.swa_retain_floor(req), + self.tree_cache.evict_sliding_windows( + req, pre_len, eviction_interval=eviction_interval ) def __str__(self): diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index 60f8fea13e..607fa61370 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -605,6 +605,27 @@ class BasePrefixCache(ABC, PrefixCacheTrait): def supports_swa(self) -> bool: return False + def supports_auxiliary_swa(self) -> bool: + return False + + def evict_sliding_windows( + self, req: Req, pre_len: int, *, eviction_interval: int = 1 + ) -> None: + """Slide request-owned windows at the scheduler's safe eviction frontier.""" + from sglang.srt.mem_cache.common import free_swa_out_of_window_slots + + free_swa_out_of_window_slots( + req, + pre_len, + sliding_window_size=self.sliding_window_size, + page_size=self.page_size, + req_to_token_pool=self.req_to_token_pool, + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + is_chunk_cache=self.is_chunk_cache(), + retain_floor=self.swa_retain_floor(req), + eviction_interval=eviction_interval, + ) + def swa_retain_floor(self, req) -> int | None: # A match lands on a state checkpoint rather than on the tail, so a cache # that pairs SWA with mamba/conv checkpoints has to keep the window behind diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 158506db34..508ba0f7de 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, NamedTuple, Optional, cast +from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, cast import numpy as np import torch @@ -62,6 +62,9 @@ def free_swa_out_of_window_slots( token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator, is_chunk_cache: bool = False, retain_floor: int | None = None, + component_type: ComponentType = ComponentType.SWA, + free_segment: Callable[..., None] | None = None, + eviction_interval: int = 1, ) -> None: if not req.kv.holds_kv: return @@ -70,10 +73,17 @@ def free_swa_out_of_window_slots( assert req.kv.cache_protected_len % page_size == 0, ( "cache_protected_len must be page aligned" ) - swa_evicted_seqlen = max( - req.kv.get_evicted_seqlen(ComponentType.SWA), req.kv.swa_dead_lo(page_size) + # Protected rows limit what can be freed, not where the interval starts. + evicted_seqlen = req.kv.get_evicted_seqlen(component_type) + if pre_len - sliding_window_size < evicted_seqlen + eviction_interval: + return + dead_lo = ( + req.kv.swa_dead_lo(page_size) + if component_type == ComponentType.SWA + else req.kv.cache_protected_len ) - req.kv.set_evicted_seqlen(ComponentType.SWA, swa_evicted_seqlen) + evicted_seqlen = max(evicted_seqlen, dead_lo) + req.kv.set_evicted_seqlen(component_type, evicted_seqlen) if is_chunk_cache: # Chunk cache builds no radix tree, so no tombstone-leaf concern; evict @@ -91,19 +101,20 @@ def free_swa_out_of_window_slots( # retained checkpoint could never be matched and holding it is pure cost. evict_threshold = min(evict_threshold, retain_floor) - new_swa_evicted_seqlen = max(swa_evicted_seqlen, evict_threshold) + new_evicted_seqlen = max(evicted_seqlen, evict_threshold) if page_size > 1: - new_swa_evicted_seqlen = (new_swa_evicted_seqlen // page_size) * page_size + new_evicted_seqlen = (new_evicted_seqlen // page_size) * page_size - if new_swa_evicted_seqlen > swa_evicted_seqlen: + if new_evicted_seqlen > evicted_seqlen: free_slots = req_to_token_pool.req_to_token[ - req.kv.req_pool_idx, swa_evicted_seqlen:new_swa_evicted_seqlen + req.kv.req_pool_idx, evicted_seqlen:new_evicted_seqlen ] - token_to_kv_pool_allocator.free_swa_segment( - free_slots, start_pos=swa_evicted_seqlen - ) - req.kv.set_evicted_seqlen(ComponentType.SWA, new_swa_evicted_seqlen) + if free_segment is None: + assert component_type == ComponentType.SWA + free_segment = token_to_kv_pool_allocator.free_swa_segment + free_segment(free_slots, start_pos=evicted_seqlen) + req.kv.set_evicted_seqlen(component_type, new_evicted_seqlen) def coalesce_ranges(ranges: list[tuple[int, int]]) -> list[tuple[int, int]]: diff --git a/python/sglang/srt/mem_cache/registry.py b/python/sglang/srt/mem_cache/registry.py index 732e24d358..d04399bec6 100644 --- a/python/sglang/srt/mem_cache/registry.py +++ b/python/sglang/srt/mem_cache/registry.py @@ -21,6 +21,7 @@ from sglang.srt.runtime_context import get_disagg, get_memory, get_serving if TYPE_CHECKING: from sglang.srt.configs.model_config import ModelConfig + from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.srt.server_args import ServerArgs logger = logging.getLogger(__name__) @@ -78,14 +79,13 @@ def registered_radix_cache_backends() -> list[str]: def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: """Built-in Radix Cache selection chain.""" - server_args = ctx.server_args params = ctx.params if ( ctx.disable_radix_cache and get_disagg().disaggregation_decode_retraction_backup == "host_pool" ): - return _create_unified_radix_cache(ctx, server_args, params) + return create_unified_radix_cache(ctx) if ctx.effective_chunked_prefill_size is not None and ctx.disable_radix_cache: if not ctx.is_hybrid_swa: @@ -124,7 +124,7 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: ) if get_memory().enable_unified_cache_external_linker: - return _create_unified_radix_cache(ctx, server_args, params) + return create_unified_radix_cache(ctx) if ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0: from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache @@ -145,15 +145,16 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache: os.environ["FLEXKV_CONFIG_PATH"] = get_memory().flexkv_config_file return _flexkv_factory(ctx) - return _create_unified_radix_cache(ctx, server_args, params) + return create_unified_radix_cache(ctx) -def _create_unified_radix_cache( +def create_unified_radix_cache( ctx: TreeCacheBuildContext, - server_args: ServerArgs, - params: CacheInitParams, + *, + cache_class: type[UnifiedRadixCache] | None = None, ) -> BasePrefixCache: """Initialize a UnifiedRadixCache with proper components and optional HiCache.""" + server_args, params = ctx.server_args, ctx.params if get_disagg().disaggregation_decode_retraction_backup == "host_pool": if ctx.is_hybrid_ssm: raise ValueError("Host-pool retraction does not support Mamba models.") @@ -189,7 +190,7 @@ def _create_unified_radix_cache( params.component_registry_override = { ComponentType.MAMBA: MlxAuxiliaryStateComponent, } - cache = UnifiedRadixCache(params) + cache = (cache_class or UnifiedRadixCache)(params) if ( ctx.enable_hierarchical_cache or get_disagg().disaggregation_decode_retraction_backup == "host_pool" diff --git a/python/sglang/srt/mem_cache/swa_memory_pool.py b/python/sglang/srt/mem_cache/swa_memory_pool.py index 1206168478..3ccbbaa55a 100644 --- a/python/sglang/srt/mem_cache/swa_memory_pool.py +++ b/python/sglang/srt/mem_cache/swa_memory_pool.py @@ -337,7 +337,7 @@ class SWAKVPool(BaseSWAKVPool): src_loc_swa = self.translate_loc_from_full_to_swa(src_loc) self.swa_kv_pool.move_kv_cache(tgt_loc_swa, src_loc_swa) - def _filter_swa_cpu_copy(self, swa_kv_cpu, row_mask: torch.Tensor): + def _filter_swa_cpu_copy(self, *, swa_kv_cpu, row_mask: torch.Tensor): if swa_kv_cpu is None: return None if row_mask is None or bool(torch.all(row_mask).item()): @@ -419,5 +419,7 @@ class SWAKVPool(BaseSWAKVPool): if swa_indices.numel() == 0: return - swa_kv_cpu = self._filter_swa_cpu_copy(swa_kv_cpu, row_mask) + swa_kv_cpu = self._filter_swa_cpu_copy( + swa_kv_cpu=swa_kv_cpu, row_mask=row_mask + ) self.swa_kv_pool.load_cpu_copy(swa_kv_cpu, swa_indices) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index 6e5709db45..2d2550e45f 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -88,6 +88,7 @@ class SWAComponent(TreeComponent): self._swa_kv_pool_host = None component_type = ComponentType.SWA + _independent = False def _collect_unbacked_swa_nodes( self, node: UnifiedTreeNode @@ -385,6 +386,54 @@ class SWAComponent(TreeComponent): swa_branching_seqlen=branching_seqlen, ) + @staticmethod + def _segments(*, node, start, end): + # Earlier components may have split the original insertion span. + segments = [] + while end > start: + node_start = end - len(node.key) + assert node_start >= start + segments.append((node, node_start, end)) + end, node = node_start, node.parent + return reversed(segments) + + def _clip_to_window(self, *, node, start, end, window_start, cache_actions): + if node.component_data[self.component_type].value is not None: + return None + assert window_start % self.tree_core.page_size == 0, ( + f"{self.component_type}: swa_evicted_seqlen must be page-aligned, {window_start=}" + ) + live_start = max(start, window_start) + if live_start >= end: + return None + if live_start > start: + _, action = self.tree_core._split_node(node.key, node, live_start - start) + if action is not None: + cache_actions.append(action) + return live_start + + def _publish_window(self, *, node, cache_actions): + cache_actions.append( + SWARebuild(node.id, node.component_data[BASE_COMPONENT_TYPE].value) + ) + + def _recover_from_incoming( + self, *, node, incoming_full, start, end, result, cache_actions + ): + full_cd = node.component_data[BASE_COMPONENT_TYPE] + old_full = full_cd.value + # A locked SWA tombstone also has Full locked, so only an unlocked + # Full value can be replaced by the incoming ids. + if full_cd.lock_ref > 0: + cache_actions.append( + RecoverSWAWithLockedFull(node.id, old_full, incoming_full) + ) + return + result.record_adopted_range(BASE_COMPONENT_TYPE, start, end) + full_cd.value = incoming_full.clone() + cache_actions.append(FreeDeviceKVFullOnly([old_full])) + self._publish_window(node=node, cache_actions=cache_actions) + def update_component_on_insert_overlap( self, node: UnifiedTreeNode, @@ -398,72 +447,56 @@ class SWAComponent(TreeComponent): if params.prev_prefix_len >= total_prefix_len + prefix_len: return prefix_len - is_tombstone = node.component_data[self.component_type].value is None - if not is_tombstone: - return prefix_len + window_start = params.get_evicted_seqlen(self.component_type) + if self._independent: + window_start = max(window_start, params.prev_prefix_len) + consumed_from = prefix_len + for fragment, start, end in self._segments( + node=node, start=total_prefix_len, end=total_prefix_len + prefix_len + ): + live_start = self._clip_to_window( + node=fragment, + start=start, + end=end, + window_start=window_start, + cache_actions=cache_actions, + ) + if live_start is None: + continue + result.record_adopted_range(self.component_type, live_start, end) + self._recover_from_incoming( + node=fragment, + incoming_full=value_slice[ + live_start - total_prefix_len : end - total_prefix_len + ], + start=live_start, + end=end, + result=result, + cache_actions=cache_actions, + ) + if not self._independent: + consumed_from = min(consumed_from, live_start - total_prefix_len) + return consumed_from - full_cd = node.component_data[BASE_COMPONENT_TYPE] - swa_evicted_seqlen = params.get_evicted_seqlen(self.component_type) - # A locked tombstone is legal (segment locks count every node); the - # full-value swap below is safe because full lock_ref >= swa - # lock_ref, so a locked-SWA node always takes the Recover branch. - assert swa_evicted_seqlen % self.tree_core.page_size == 0, ( - f"{self.component_type}: swa_evicted_seqlen must be page-aligned, {swa_evicted_seqlen=}" - ) - - if swa_evicted_seqlen <= total_prefix_len: - # Branch 1: entire value_slice is within SWA window — recover - result.record_adopted_range( - self.component_type, - total_prefix_len, - total_prefix_len + prefix_len, + def _populate_range( + self, *, node, start, end, params, result, cap_leaf, cache_actions + ): + for fragment, start, end in self._segments(node=node, start=start, end=end): + live_start = self._clip_to_window( + node=fragment, + start=start, + end=end, + window_start=params.get_evicted_seqlen(self.component_type), + cache_actions=cache_actions, ) - old_full = full_cd.value - if full_cd.lock_ref > 0: - cache_actions.append( - RecoverSWAWithLockedFull(node.id, old_full, value_slice) - ) - return 0 - result.record_adopted_range( - BASE_COMPONENT_TYPE, - total_prefix_len, - total_prefix_len + prefix_len, - ) - full_cd.value = value_slice.clone() - cache_actions.append(FreeDeviceKVFullOnly([old_full])) - cache_actions.append(SWARebuild(node.id, value_slice)) - return 0 - elif swa_evicted_seqlen < total_prefix_len + prefix_len: - # Branch 2: value_slice[start_idx:] is within SWA window — partial recover - start_idx = swa_evicted_seqlen - total_prefix_len - result.record_adopted_range( - self.component_type, - swa_evicted_seqlen, - total_prefix_len + prefix_len, - ) - is_locked = full_cd.lock_ref > 0 - old_full = full_cd.value[start_idx:] - _, action = self.tree_core._split_node(node.key, node, start_idx) - if action is not None: - cache_actions.append(action) - new_full = value_slice[start_idx:] - if is_locked: - cache_actions.append( - RecoverSWAWithLockedFull(node.id, old_full, new_full) - ) - return start_idx - result.record_adopted_range( - BASE_COMPONENT_TYPE, - swa_evicted_seqlen, - total_prefix_len + prefix_len, - ) - node.component_data[BASE_COMPONENT_TYPE].value = new_full.clone() - cache_actions.append(FreeDeviceKVFullOnly([old_full])) - cache_actions.append(SWARebuild(node.id, new_full)) - return start_idx - else: - # Branch 3: entire value_slice is outside SWA window — not consumed - return prefix_len + if live_start is None: + continue + parent = self._maybe_split_leaf_for_swa_lock(fragment) if cap_leaf else None + # Older fragments publish first so the tail remains more-MRU. + if parent is not None: + self._publish_window(node=parent, cache_actions=cache_actions) + self._publish_window(node=fragment, cache_actions=cache_actions) + result.record_adopted_range(self.component_type, live_start, end) def recover_after_unevict( self, @@ -477,33 +510,14 @@ class SWAComponent(TreeComponent): # _unevict_node_on_insert already wrote the request's fresh KV slice # into the base value. We just need to rebuild SWA from that slice for # the in-window portion. There is no old SWA slot to free here. - ct = self.component_type - if node.component_data[ct].value is not None: - return - swa_evicted_seqlen = params.get_evicted_seqlen(self.component_type) - assert swa_evicted_seqlen % self.tree_core.page_size == 0, ( - f"{ct}: swa_evicted_seqlen must be page-aligned, {swa_evicted_seqlen=}" - ) - - if swa_evicted_seqlen <= total_prefix_len: - pass # entire node is within the SWA window - elif swa_evicted_seqlen < total_prefix_len + prefix_len: - start_idx = swa_evicted_seqlen - total_prefix_len - _, action = self.tree_core._split_node(node.key, node, start_idx) - if action is not None: - cache_actions.append(action) - else: - return - result.record_adopted_range( - self.component_type, - max(total_prefix_len, swa_evicted_seqlen), - total_prefix_len + prefix_len, - ) - cache_actions.append( - SWARebuild( - node.id, - node.component_data[BASE_COMPONENT_TYPE].value, - ) + self._populate_range( + node=node, + start=total_prefix_len, + end=total_prefix_len + prefix_len, + params=params, + result=result, + cap_leaf=self._independent, + cache_actions=cache_actions, ) def commit_insert_component_data( @@ -522,39 +536,20 @@ class SWAComponent(TreeComponent): if not is_new_leaf: return - node_start = result.prefix_len - node_end = node_start + len(node.key) - swa_evicted_seqlen = params.get_evicted_seqlen(self.component_type) - split_pos = swa_evicted_seqlen - node_start - if split_pos >= len(node.key): - # Entire leaf is outside the SWA window — left as a tombstone. - return - result.record_adopted_range( - self.component_type, - max(node_start, swa_evicted_seqlen), - node_end, + # Another window may have split this leaf; use the full aligned insert span. + end = ( + len(params.key) // self.tree_core.page_size * self.tree_core.page_size + if self._independent + else result.prefix_len + len(node.key) ) - if split_pos > 0: - # Node straddles the boundary: split into an out-of-window parent - # (tombstone) and an in-window child; `node` becomes the child. - _, action = self.tree_core._split_node(node.key, node, split_pos) - assert action is None, "new leaf cannot be write-through-pending" - # Cap the in-window leaf at one window for lock granularity, then rebuild SWA - # onto the in-window node(s) at apply time; rebuild the older prefix first so - # the in-window tail lands more-MRU. - capped_parent = self._maybe_split_leaf_for_swa_lock(node) - if capped_parent is not None: - cache_actions.append( - SWARebuild( - capped_parent.id, - capped_parent.component_data[BASE_COMPONENT_TYPE].value, - ) - ) - cache_actions.append( - SWARebuild( - node.id, - node.component_data[BASE_COMPONENT_TYPE].value, - ) + self._populate_range( + node=node, + start=result.prefix_len, + end=end, + params=params, + result=result, + cap_leaf=True, + cache_actions=cache_actions, ) def _maybe_split_leaf_for_swa_lock( diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 6cccf44339..4027b80354 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -739,6 +739,25 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): ) self._update_evictable_leaf_sets(node) + def dec_window_lock_only( + self, + node_id: NodeId, + component_type: ComponentType, + params: DecLockRefParams, + ) -> DecSwaLockOnlyResult: + result = DecSwaLockOnlyResult() + node = self.node_by_id(node_id) + self._assert_receipt_anchor(node, params) + if node is self.root_node or component_type in params.skipped_lock_components: + return result + self.components_by_type[component_type].release_window_lock( + node, + params.get_lock_uuid(component_type), + result.device_frees, + result.host_frees, + ) + return result + def dec_swa_lock_only( self, node_id: NodeId, @@ -3038,6 +3057,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): """Evictable token count for one component (0 if the component is absent).""" return self.component_evictable_size_.get(component_type, 0) + def component_protected_size(self, component_type: ComponentType) -> int: + return self.component_protected_size_.get(component_type, 0) + def full_evictable_size(self) -> int: return self.evictable_size() diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py index 8111d0871d..4987b100f3 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py @@ -282,6 +282,15 @@ class UnifiedTreeCoreInterface(ABC): counts; the result carries the freed slots.""" ... + def dec_window_lock_only( + self, + node_id: NodeId, + component_type: ComponentType, + params: DecLockRefParams, + ) -> DecSwaLockOnlyResult: + """Release one window's receipt without releasing peer components.""" + raise NotImplementedError("This tree core does not support independent windows") + # ==== Device eviction (driven step-wise by the Controller's evict()) ==== @abstractmethod @@ -349,6 +358,10 @@ class UnifiedTreeCoreInterface(ABC): """Evictable token count for one component (0 if the component is absent).""" ... + def component_protected_size(self, component_type: ComponentType) -> int: + """Protected token count for one component (0 if absent).""" + raise NotImplementedError("This tree core does not expose per-component sizes") + @abstractmethod def full_evictable_size(self) -> int: ... diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index d8bf163e49..bc5e39a24b 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -1744,7 +1744,9 @@ class UnifiedSWAKVPool(SWAKVPool): swa_phys = swa_phys[old_swa_mask][row_mask.to(indices.device)] if swa_phys.numel() == 0: return - swa_cpu = self._filter_swa_cpu_copy(kv_cache_cpu["swa"], row_mask) + swa_cpu = self._filter_swa_cpu_copy( + swa_kv_cpu=kv_cache_cpu["swa"], row_mask=row_mask + ) self.swa_kv_pool.load_cpu_copy(swa_cpu, swa_phys) diff --git a/python/sglang/srt/model_executor/forward_batch_info.py b/python/sglang/srt/model_executor/forward_batch_info.py index 6b116fd1c0..5ecbb9232e 100644 --- a/python/sglang/srt/model_executor/forward_batch_info.py +++ b/python/sglang/srt/model_executor/forward_batch_info.py @@ -1878,29 +1878,18 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin): spec_info = self.spec_info self.output_cache_loc_backup = self.out_cache_loc self.hidden_states_backup = spec_info.hidden_states - # spec_info is EagleDraftInput | EagleDraftExtendInput; each carries - # a disjoint subset of the fields below, so getattr-guard each one. + # Draft input types carry different subsets of the shared fields. if getattr(spec_info, "topk_p", None) is not None: spec_info.topk_p = self._pad_tensor_to_size(spec_info.topk_p, bs) if getattr(spec_info, "topk_index", None) is not None: spec_info.topk_index = self._pad_tensor_to_size( spec_info.topk_index, bs ) - if getattr(spec_info, "draft_probs", None) is not None: - spec_info.draft_probs = self._pad_tensor_to_size( - spec_info.draft_probs, bs - ) - if getattr(spec_info, "num_correct_drafts", None) is not None: - spec_info.num_correct_drafts = self._pad_tensor_to_size( - spec_info.num_correct_drafts, bs - ) - spec_info.num_accept_tokens = self._pad_tensor_to_size( - spec_info.num_accept_tokens, bs - ) if spec_info.hidden_states is not None: spec_info.hidden_states = self._pad_tensor_to_size( spec_info.hidden_states, num_tokens ) + spec_info.pad_batch(self._pad_tensor_to_size, bs) def _forward_num_tokens(self) -> int: """Token width of this forward (``input_embeds`` over ``input_ids``), as diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py index 24ef6dfcf5..b0417d545f 100644 --- a/python/sglang/srt/model_executor/pool_configurator.py +++ b/python/sglang/srt/model_executor/pool_configurator.py @@ -809,7 +809,13 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator): return self._solve_pool_sizes(max_total_num_tokens, page_size) -def compute_swa_request_cap(*, page_size: int, window: int, attn_dp_size: int) -> int: +def compute_swa_request_cap( + *, + page_size: int, + window: int, + attn_dp_size: int, + max_running_requests: int | None = None, +) -> int: """Worst-case SWA slots the scheduler holds live at max_running_requests.""" draft_tokens = get_spec().speculative_num_draft_tokens or 1 eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get()) @@ -833,7 +839,9 @@ def compute_swa_request_cap(*, page_size: int, window: int, attn_dp_size: int) - decode_alloc = 2 * get_alloc_len_per_decode() per_request = trailing_tokens + decode_alloc - num_reqs = get_schedule().max_running_requests // attn_dp_size + if max_running_requests is None: + max_running_requests = get_schedule().max_running_requests + num_reqs = max_running_requests // attn_dp_size if get_disagg().disaggregation_mode == "decode": return ( per_request * num_reqs diff --git a/python/sglang/srt/speculative/eagle_info.py b/python/sglang/srt/speculative/eagle_info.py index 7c58117b37..80ad187ea8 100644 --- a/python/sglang/srt/speculative/eagle_info.py +++ b/python/sglang/srt/speculative/eagle_info.py @@ -1,6 +1,6 @@ import logging from dataclasses import dataclass, field -from typing import List, Optional +from typing import Callable, List, Optional import torch @@ -180,6 +180,14 @@ class EagleDraftInput(SpecInput): def __post_init__(self): super().__init__(SpecInputType.EAGLE_DRAFT) + def pad_batch( + self, + pad_tensor_to_size: Callable[..., torch.Tensor], + batch_size: int, + ) -> None: + if self.draft_probs is not None: + self.draft_probs = pad_tensor_to_size(self.draft_probs, batch_size) + @classmethod def create_idle_input( cls, @@ -328,6 +336,19 @@ class EagleDraftExtendInput(SpecInput): def __post_init__(self): super().__init__(SpecInputType.EAGLE_DRAFT_EXTEND) + def pad_batch( + self, + pad_tensor_to_size: Callable[..., torch.Tensor], + batch_size: int, + ) -> None: + if self.num_correct_drafts is not None: + self.num_correct_drafts = pad_tensor_to_size( + self.num_correct_drafts, batch_size + ) + self.num_accept_tokens = pad_tensor_to_size( + self.num_accept_tokens, batch_size + ) + @classmethod def create_idle_input( cls, diff --git a/python/sglang/srt/speculative/spec_info.py b/python/sglang/srt/speculative/spec_info.py index b106525963..bc0256cc82 100644 --- a/python/sglang/srt/speculative/spec_info.py +++ b/python/sglang/srt/speculative/spec_info.py @@ -432,6 +432,13 @@ class SpecInput(ABC): SpecInputType.UNO_VERIFY, } + def pad_batch( + self, + pad_tensor_to_size: Callable[..., torch.Tensor], + batch_size: int, + ) -> None: + return None + def spec_scale_global_num_tokens( spec_info: SpecInput, diff --git a/test/registered/unit/mem_cache/test_registry.py b/test/registered/unit/mem_cache/test_registry.py index f0dab16188..d5441e9476 100644 --- a/test/registered/unit/mem_cache/test_registry.py +++ b/test/registered/unit/mem_cache/test_registry.py @@ -12,6 +12,7 @@ from sglang.srt.mem_cache.registry import ( _RADIX_CACHE_REGISTRY, TreeCacheBuildContext, create_tree_cache, + create_unified_radix_cache, default_radix_cache_factory, get_radix_cache_factory, register_radix_cache_backend, @@ -303,6 +304,23 @@ class TestDefaultRadixCacheFactory(CustomTestCase): ) self.assertIs(result, cache) + def test_custom_unified_cache_keeps_host_pool_setup(self): + ctx = _make_ctx(self) + enter_override( + self, + get_context().override_server_args( + disaggregation_decode_retraction_backup="host_pool" + ), + ) + cache_class = MagicMock() + result = create_unified_radix_cache(ctx, cache_class=cache_class) + cache_class.assert_called_once_with(ctx.params) + result.init_hicache.assert_called_once_with(ctx.server_args, ctx.params) + ctx.tp_worker.register_hicache_layer_transfer_counter.assert_called_once_with( + result.cache_controller.layer_done_counter + ) + self.assertIs(result, cache_class.return_value) + def test_pure_swa_radix_cache_when_all_swa(self): ctx = _make_ctx(self, is_hybrid_swa=True, full_tokens_per_layer=0) with patch( diff --git a/test/registered/unit/mem_cache/test_swa_cpu_copy_filter.py b/test/registered/unit/mem_cache/test_swa_cpu_copy_filter.py index bfe0e7412c..3927adad88 100644 --- a/test/registered/unit/mem_cache/test_swa_cpu_copy_filter.py +++ b/test/registered/unit/mem_cache/test_swa_cpu_copy_filter.py @@ -36,7 +36,7 @@ class TestSWACpuCopyFilter(unittest.TestCase): side produced, and a short chunk resumes against the wrong exponents.""" row_mask = torch.tensor([True, False, True, False]) - filtered = _pool()._filter_swa_cpu_copy(_chunk(4), row_mask) + filtered = _pool()._filter_swa_cpu_copy(swa_kv_cpu=_chunk(4), row_mask=row_mask) self.assertEqual(len(filtered[0][0]), 4) for t, tensor in enumerate(filtered[0][0]): @@ -46,7 +46,7 @@ class TestSWACpuCopyFilter(unittest.TestCase): def test_unquantized_chunk_is_unchanged(self): row_mask = torch.tensor([False, True, True, False]) - filtered = _pool()._filter_swa_cpu_copy(_chunk(2), row_mask) + filtered = _pool()._filter_swa_cpu_copy(swa_kv_cpu=_chunk(2), row_mask=row_mask) self.assertEqual(len(filtered[0][0]), 2) self.assertEqual(filtered[0][0][0].shape[0], 2) @@ -55,7 +55,9 @@ class TestSWACpuCopyFilter(unittest.TestCase): original = _chunk(4) self.assertIs( - _pool()._filter_swa_cpu_copy(original, torch.tensor([True] * ROWS)), + _pool()._filter_swa_cpu_copy( + swa_kv_cpu=original, row_mask=torch.tensor([True] * ROWS) + ), original, ) diff --git a/test/registered/unit/mem_cache/test_swa_eviction_interval.py b/test/registered/unit/mem_cache/test_swa_eviction_interval.py new file mode 100644 index 0000000000..6fbf40761c --- /dev/null +++ b/test/registered/unit/mem_cache/test_swa_eviction_interval.py @@ -0,0 +1,129 @@ +import unittest +from itertools import product +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch + +from sglang.srt.environ import envs +from sglang.srt.managers.schedule_batch import ReqKvInfo, ScheduleBatch +from sglang.srt.mem_cache.radix_cache import RadixCache +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class _SwaCache(RadixCache): + def __init__(self, *, auxiliary=False): + self.sliding_window_size = 128 + self.page_size = 16 + self.req_to_token_pool = SimpleNamespace( + req_to_token=torch.arange(1024).reshape(1, -1) + ) + self.token_to_kv_pool_allocator = Mock( + spec=["free_swa_segment", "free_group_begin", "free_group_end"] + ) + self.trim_calls = 0 + self.auxiliary = auxiliary + + def supports_swa(self): + return True + + def supports_auxiliary_swa(self): + return self.auxiliary + + def evict_sliding_windows(self, req, pre_len, *, eviction_interval=1): + self.trim_calls += 1 + super().evict_sliding_windows(req, pre_len, eviction_interval=eviction_interval) + + +class TestSWAEvictionInterval(unittest.TestCase): + def test_scheduler_gates_on_request_cursor_before_protected_prefix(self): + for (protected, shield), auxiliary in product( + [(256, 0), (0, 249)], [False, True] + ): + with ( + self.subTest(protected=protected, shield=shield, auxiliary=auxiliary), + envs.SGLANG_SWA_EVICTION_INTERVAL.override(128), + envs.SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW.override(False), + ): + cache = _SwaCache(auxiliary=auxiliary) + req = SimpleNamespace( + seqlen=401, + decode_batch_idx=1, + kv=ReqKvInfo( + req_pool_idx=0, + cache_protected_len=protected, + swa_evict_floor=shield, + ), + ) + batch = ScheduleBatch( + reqs=[req], + tree_cache=cache, + req_to_token_pool=cache.req_to_token_pool, + token_to_kv_pool_allocator=cache.token_to_kv_pool_allocator, + forward_mode=ForwardMode.DECODE, + ) + free = cache.token_to_kv_pool_allocator.free_swa_segment + + # The raw cursor makes eviction due; the protected prefix only + # limits which rows may be freed, not when the interval starts. + batch.maybe_evict_swa() + self.assertEqual(cache.trim_calls, 1) + free.assert_called_once() + self.assertTrue( + torch.equal(free.call_args.args[0], torch.arange(256, 272)) + ) + self.assertEqual(free.call_args.kwargs, {"start_pos": 256}) + self.assertEqual(req.kv.get_evicted_seqlen(ComponentType.SWA), 272) + free.reset_mock() + + req.seqlen = 528 + batch.maybe_evict_swa() + self.assertEqual(cache.trim_calls, 1 + auxiliary) + free.assert_not_called() + + req.seqlen += 1 + batch.maybe_evict_swa() + self.assertEqual(cache.trim_calls, 2 + auxiliary) + free.assert_called_once() + self.assertTrue( + torch.equal(free.call_args.args[0], torch.arange(272, 400)) + ) + + def test_forward_cadence_is_not_gated_again_by_token_interval(self): + cache = _SwaCache() + req = SimpleNamespace( + seqlen=201, + decode_batch_idx=1, + kv=ReqKvInfo(req_pool_idx=0), + ) + batch = ScheduleBatch( + reqs=[req], + tree_cache=cache, + req_to_token_pool=cache.req_to_token_pool, + token_to_kv_pool_allocator=cache.token_to_kv_pool_allocator, + forward_mode=ForwardMode.DECODE, + forward_iter=127, + ) + free = cache.token_to_kv_pool_allocator.free_swa_segment + with ( + patch("sglang.srt.managers.schedule_batch._is_hip", True), + envs.SGLANG_AMD_USE_FLYDSL_MEGA_MOE.override(True), + envs.SGLANG_SWA_EVICTION_INTERVAL.override(128), + envs.SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW.override(False), + ): + batch.maybe_evict_swa() + free.assert_not_called() + batch.forward_iter += 1 + batch.maybe_evict_swa() + + free.assert_called_once() + self.assertTrue(torch.equal(free.call_args.args[0], torch.arange(64))) + self.assertEqual(req.kv.get_evicted_seqlen(ComponentType.SWA), 64) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py b/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py index 4f80f9c08e..600751cf70 100644 --- a/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py +++ b/test/registered/unit/model_executor/test_mlp_sync_pad_unpad.py @@ -104,10 +104,12 @@ class TestMlpSyncPadUnpad(CustomTestCase): ) self.assertTrue(fb.can_run_decode_cuda_graph) - def test_draft_input_without_hidden_states_can_be_padded(self): + def test_draft_input_without_hidden_states_runs_padding_hook(self): + pad_batch = MagicMock() spec_info = SimpleNamespace( is_draft_input=lambda: True, hidden_states=None, + pad_batch=pad_batch, ) fb = ForwardBatch( forward_mode=ForwardMode.TARGET_VERIFY, @@ -126,6 +128,12 @@ class TestMlpSyncPadUnpad(CustomTestCase): fb._pad_inputs_to_size(_mock_model_runner(), num_tokens=2, bs=1) self.assertIsNone(spec_info.hidden_states) + pad_batch.assert_called_once() + pad_tensor_to_size, batch_size = pad_batch.call_args.args + self.assertEqual(batch_size, 1) + torch.testing.assert_close( + pad_tensor_to_size(torch.tensor([1]), 2), torch.tensor([1, 0]) + ) def test_dp_cuda_graph_batch_size_uses_raw_request_counts(self): fb = SimpleNamespace(original_global_num_tokens_cpu=[3, 11, 7])