Make sliding-window caching and speculative batch padding extensible (#41325)
Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com> Co-authored-by: jasonjk-park <jasonjk@fb.com>
This commit is contained in:
co-authored by
Lu Fang
jasonjk-park
parent
c73f7077eb
commit
c10fa03fc4
@@ -115,7 +115,6 @@ from sglang.srt.mem_cache.common import (
|
||||
RetractionBackup,
|
||||
backup_kv_cache,
|
||||
evict_from_tree_cache,
|
||||
free_swa_out_of_window_slots,
|
||||
release_kv_cache,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
|
||||
@@ -3882,6 +3881,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
def maybe_evict_swa(self):
|
||||
if self.tree_cache.supports_swa():
|
||||
sliding_window_size = self.tree_cache.sliding_window_size
|
||||
# Auxiliary windows check their own cursors and prefix locks.
|
||||
has_auxiliary_swa = self.tree_cache.supports_auxiliary_swa()
|
||||
|
||||
release_leaf_lock = (
|
||||
envs.SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW.get()
|
||||
@@ -3901,9 +3902,17 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
if (
|
||||
req.decode_batch_idx >= 1
|
||||
and req.kv.holds_kv
|
||||
and swa_evict_due(req)
|
||||
and (has_auxiliary_swa or swa_evict_due(req))
|
||||
):
|
||||
self._evict_swa(req, req.seqlen - 1)
|
||||
# The scheduler already gated the primary window,
|
||||
# including backends that use a forward-based cadence.
|
||||
self._evict_swa(
|
||||
req,
|
||||
req.seqlen - 1,
|
||||
eviction_interval=(
|
||||
eviction_interval if has_auxiliary_swa else 1
|
||||
),
|
||||
)
|
||||
|
||||
# Once the decode position has moved past the sliding window,
|
||||
# the SWA portion of the prefill-time tree lock is no longer
|
||||
@@ -3938,17 +3947,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
self._evict_swa(req, pre_len)
|
||||
self.token_to_kv_pool_allocator.free_group_end()
|
||||
|
||||
def _evict_swa(self, req: Req, pre_len: int):
|
||||
def _evict_swa(self, req: Req, pre_len: int, *, eviction_interval: int = 1):
|
||||
assert self.tree_cache.supports_swa(), "prefix cache must support swa"
|
||||
free_swa_out_of_window_slots(
|
||||
req,
|
||||
pre_len,
|
||||
sliding_window_size=self.tree_cache.sliding_window_size,
|
||||
page_size=self.tree_cache.page_size,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
is_chunk_cache=self.tree_cache.is_chunk_cache(),
|
||||
retain_floor=self.tree_cache.swa_retain_floor(req),
|
||||
self.tree_cache.evict_sliding_windows(
|
||||
req, pre_len, eviction_interval=eviction_interval
|
||||
)
|
||||
|
||||
def __str__(self):
|
||||
|
||||
@@ -605,6 +605,27 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
def supports_swa(self) -> bool:
|
||||
return False
|
||||
|
||||
def supports_auxiliary_swa(self) -> bool:
|
||||
return False
|
||||
|
||||
def evict_sliding_windows(
|
||||
self, req: Req, pre_len: int, *, eviction_interval: int = 1
|
||||
) -> None:
|
||||
"""Slide request-owned windows at the scheduler's safe eviction frontier."""
|
||||
from sglang.srt.mem_cache.common import free_swa_out_of_window_slots
|
||||
|
||||
free_swa_out_of_window_slots(
|
||||
req,
|
||||
pre_len,
|
||||
sliding_window_size=self.sliding_window_size,
|
||||
page_size=self.page_size,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
is_chunk_cache=self.is_chunk_cache(),
|
||||
retain_floor=self.swa_retain_floor(req),
|
||||
eviction_interval=eviction_interval,
|
||||
)
|
||||
|
||||
def swa_retain_floor(self, req) -> int | None:
|
||||
# A match lands on a state checkpoint rather than on the tail, so a cache
|
||||
# that pairs SWA with mamba/conv checkpoints has to keep the window behind
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, cast
|
||||
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -62,6 +62,9 @@ def free_swa_out_of_window_slots(
|
||||
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
|
||||
is_chunk_cache: bool = False,
|
||||
retain_floor: int | None = None,
|
||||
component_type: ComponentType = ComponentType.SWA,
|
||||
free_segment: Callable[..., None] | None = None,
|
||||
eviction_interval: int = 1,
|
||||
) -> None:
|
||||
if not req.kv.holds_kv:
|
||||
return
|
||||
@@ -70,10 +73,17 @@ def free_swa_out_of_window_slots(
|
||||
assert req.kv.cache_protected_len % page_size == 0, (
|
||||
"cache_protected_len must be page aligned"
|
||||
)
|
||||
swa_evicted_seqlen = max(
|
||||
req.kv.get_evicted_seqlen(ComponentType.SWA), req.kv.swa_dead_lo(page_size)
|
||||
# Protected rows limit what can be freed, not where the interval starts.
|
||||
evicted_seqlen = req.kv.get_evicted_seqlen(component_type)
|
||||
if pre_len - sliding_window_size < evicted_seqlen + eviction_interval:
|
||||
return
|
||||
dead_lo = (
|
||||
req.kv.swa_dead_lo(page_size)
|
||||
if component_type == ComponentType.SWA
|
||||
else req.kv.cache_protected_len
|
||||
)
|
||||
req.kv.set_evicted_seqlen(ComponentType.SWA, swa_evicted_seqlen)
|
||||
evicted_seqlen = max(evicted_seqlen, dead_lo)
|
||||
req.kv.set_evicted_seqlen(component_type, evicted_seqlen)
|
||||
|
||||
if is_chunk_cache:
|
||||
# Chunk cache builds no radix tree, so no tombstone-leaf concern; evict
|
||||
@@ -91,19 +101,20 @@ def free_swa_out_of_window_slots(
|
||||
# retained checkpoint could never be matched and holding it is pure cost.
|
||||
evict_threshold = min(evict_threshold, retain_floor)
|
||||
|
||||
new_swa_evicted_seqlen = max(swa_evicted_seqlen, evict_threshold)
|
||||
new_evicted_seqlen = max(evicted_seqlen, evict_threshold)
|
||||
|
||||
if page_size > 1:
|
||||
new_swa_evicted_seqlen = (new_swa_evicted_seqlen // page_size) * page_size
|
||||
new_evicted_seqlen = (new_evicted_seqlen // page_size) * page_size
|
||||
|
||||
if new_swa_evicted_seqlen > swa_evicted_seqlen:
|
||||
if new_evicted_seqlen > evicted_seqlen:
|
||||
free_slots = req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, swa_evicted_seqlen:new_swa_evicted_seqlen
|
||||
req.kv.req_pool_idx, evicted_seqlen:new_evicted_seqlen
|
||||
]
|
||||
token_to_kv_pool_allocator.free_swa_segment(
|
||||
free_slots, start_pos=swa_evicted_seqlen
|
||||
)
|
||||
req.kv.set_evicted_seqlen(ComponentType.SWA, new_swa_evicted_seqlen)
|
||||
if free_segment is None:
|
||||
assert component_type == ComponentType.SWA
|
||||
free_segment = token_to_kv_pool_allocator.free_swa_segment
|
||||
free_segment(free_slots, start_pos=evicted_seqlen)
|
||||
req.kv.set_evicted_seqlen(component_type, new_evicted_seqlen)
|
||||
|
||||
|
||||
def coalesce_ranges(ranges: list[tuple[int, int]]) -> list[tuple[int, int]]:
|
||||
|
||||
@@ -21,6 +21,7 @@ from sglang.srt.runtime_context import get_disagg, get_memory, get_serving
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.configs.model_config import ModelConfig
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -78,14 +79,13 @@ def registered_radix_cache_backends() -> list[str]:
|
||||
|
||||
def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
"""Built-in Radix Cache selection chain."""
|
||||
server_args = ctx.server_args
|
||||
params = ctx.params
|
||||
|
||||
if (
|
||||
ctx.disable_radix_cache
|
||||
and get_disagg().disaggregation_decode_retraction_backup == "host_pool"
|
||||
):
|
||||
return _create_unified_radix_cache(ctx, server_args, params)
|
||||
return create_unified_radix_cache(ctx)
|
||||
|
||||
if ctx.effective_chunked_prefill_size is not None and ctx.disable_radix_cache:
|
||||
if not ctx.is_hybrid_swa:
|
||||
@@ -124,7 +124,7 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
)
|
||||
|
||||
if get_memory().enable_unified_cache_external_linker:
|
||||
return _create_unified_radix_cache(ctx, server_args, params)
|
||||
return create_unified_radix_cache(ctx)
|
||||
|
||||
if ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0:
|
||||
from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache
|
||||
@@ -145,15 +145,16 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
|
||||
os.environ["FLEXKV_CONFIG_PATH"] = get_memory().flexkv_config_file
|
||||
return _flexkv_factory(ctx)
|
||||
|
||||
return _create_unified_radix_cache(ctx, server_args, params)
|
||||
return create_unified_radix_cache(ctx)
|
||||
|
||||
|
||||
def _create_unified_radix_cache(
|
||||
def create_unified_radix_cache(
|
||||
ctx: TreeCacheBuildContext,
|
||||
server_args: ServerArgs,
|
||||
params: CacheInitParams,
|
||||
*,
|
||||
cache_class: type[UnifiedRadixCache] | None = None,
|
||||
) -> BasePrefixCache:
|
||||
"""Initialize a UnifiedRadixCache with proper components and optional HiCache."""
|
||||
server_args, params = ctx.server_args, ctx.params
|
||||
if get_disagg().disaggregation_decode_retraction_backup == "host_pool":
|
||||
if ctx.is_hybrid_ssm:
|
||||
raise ValueError("Host-pool retraction does not support Mamba models.")
|
||||
@@ -189,7 +190,7 @@ def _create_unified_radix_cache(
|
||||
params.component_registry_override = {
|
||||
ComponentType.MAMBA: MlxAuxiliaryStateComponent,
|
||||
}
|
||||
cache = UnifiedRadixCache(params)
|
||||
cache = (cache_class or UnifiedRadixCache)(params)
|
||||
if (
|
||||
ctx.enable_hierarchical_cache
|
||||
or get_disagg().disaggregation_decode_retraction_backup == "host_pool"
|
||||
|
||||
@@ -337,7 +337,7 @@ class SWAKVPool(BaseSWAKVPool):
|
||||
src_loc_swa = self.translate_loc_from_full_to_swa(src_loc)
|
||||
self.swa_kv_pool.move_kv_cache(tgt_loc_swa, src_loc_swa)
|
||||
|
||||
def _filter_swa_cpu_copy(self, swa_kv_cpu, row_mask: torch.Tensor):
|
||||
def _filter_swa_cpu_copy(self, *, swa_kv_cpu, row_mask: torch.Tensor):
|
||||
if swa_kv_cpu is None:
|
||||
return None
|
||||
if row_mask is None or bool(torch.all(row_mask).item()):
|
||||
@@ -419,5 +419,7 @@ class SWAKVPool(BaseSWAKVPool):
|
||||
if swa_indices.numel() == 0:
|
||||
return
|
||||
|
||||
swa_kv_cpu = self._filter_swa_cpu_copy(swa_kv_cpu, row_mask)
|
||||
swa_kv_cpu = self._filter_swa_cpu_copy(
|
||||
swa_kv_cpu=swa_kv_cpu, row_mask=row_mask
|
||||
)
|
||||
self.swa_kv_pool.load_cpu_copy(swa_kv_cpu, swa_indices)
|
||||
|
||||
@@ -88,6 +88,7 @@ class SWAComponent(TreeComponent):
|
||||
self._swa_kv_pool_host = None
|
||||
|
||||
component_type = ComponentType.SWA
|
||||
_independent = False
|
||||
|
||||
def _collect_unbacked_swa_nodes(
|
||||
self, node: UnifiedTreeNode
|
||||
@@ -385,6 +386,54 @@ class SWAComponent(TreeComponent):
|
||||
swa_branching_seqlen=branching_seqlen,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _segments(*, node, start, end):
|
||||
# Earlier components may have split the original insertion span.
|
||||
segments = []
|
||||
while end > start:
|
||||
node_start = end - len(node.key)
|
||||
assert node_start >= start
|
||||
segments.append((node, node_start, end))
|
||||
end, node = node_start, node.parent
|
||||
return reversed(segments)
|
||||
|
||||
def _clip_to_window(self, *, node, start, end, window_start, cache_actions):
|
||||
if node.component_data[self.component_type].value is not None:
|
||||
return None
|
||||
assert window_start % self.tree_core.page_size == 0, (
|
||||
f"{self.component_type}: swa_evicted_seqlen must be page-aligned, {window_start=}"
|
||||
)
|
||||
live_start = max(start, window_start)
|
||||
if live_start >= end:
|
||||
return None
|
||||
if live_start > start:
|
||||
_, action = self.tree_core._split_node(node.key, node, live_start - start)
|
||||
if action is not None:
|
||||
cache_actions.append(action)
|
||||
return live_start
|
||||
|
||||
def _publish_window(self, *, node, cache_actions):
|
||||
cache_actions.append(
|
||||
SWARebuild(node.id, node.component_data[BASE_COMPONENT_TYPE].value)
|
||||
)
|
||||
|
||||
def _recover_from_incoming(
|
||||
self, *, node, incoming_full, start, end, result, cache_actions
|
||||
):
|
||||
full_cd = node.component_data[BASE_COMPONENT_TYPE]
|
||||
old_full = full_cd.value
|
||||
# A locked SWA tombstone also has Full locked, so only an unlocked
|
||||
# Full value can be replaced by the incoming ids.
|
||||
if full_cd.lock_ref > 0:
|
||||
cache_actions.append(
|
||||
RecoverSWAWithLockedFull(node.id, old_full, incoming_full)
|
||||
)
|
||||
return
|
||||
result.record_adopted_range(BASE_COMPONENT_TYPE, start, end)
|
||||
full_cd.value = incoming_full.clone()
|
||||
cache_actions.append(FreeDeviceKVFullOnly([old_full]))
|
||||
self._publish_window(node=node, cache_actions=cache_actions)
|
||||
|
||||
def update_component_on_insert_overlap(
|
||||
self,
|
||||
node: UnifiedTreeNode,
|
||||
@@ -398,72 +447,56 @@ class SWAComponent(TreeComponent):
|
||||
if params.prev_prefix_len >= total_prefix_len + prefix_len:
|
||||
return prefix_len
|
||||
|
||||
is_tombstone = node.component_data[self.component_type].value is None
|
||||
if not is_tombstone:
|
||||
return prefix_len
|
||||
window_start = params.get_evicted_seqlen(self.component_type)
|
||||
if self._independent:
|
||||
window_start = max(window_start, params.prev_prefix_len)
|
||||
consumed_from = prefix_len
|
||||
for fragment, start, end in self._segments(
|
||||
node=node, start=total_prefix_len, end=total_prefix_len + prefix_len
|
||||
):
|
||||
live_start = self._clip_to_window(
|
||||
node=fragment,
|
||||
start=start,
|
||||
end=end,
|
||||
window_start=window_start,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
if live_start is None:
|
||||
continue
|
||||
result.record_adopted_range(self.component_type, live_start, end)
|
||||
self._recover_from_incoming(
|
||||
node=fragment,
|
||||
incoming_full=value_slice[
|
||||
live_start - total_prefix_len : end - total_prefix_len
|
||||
],
|
||||
start=live_start,
|
||||
end=end,
|
||||
result=result,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
if not self._independent:
|
||||
consumed_from = min(consumed_from, live_start - total_prefix_len)
|
||||
return consumed_from
|
||||
|
||||
full_cd = node.component_data[BASE_COMPONENT_TYPE]
|
||||
swa_evicted_seqlen = params.get_evicted_seqlen(self.component_type)
|
||||
# A locked tombstone is legal (segment locks count every node); the
|
||||
# full-value swap below is safe because full lock_ref >= swa
|
||||
# lock_ref, so a locked-SWA node always takes the Recover branch.
|
||||
assert swa_evicted_seqlen % self.tree_core.page_size == 0, (
|
||||
f"{self.component_type}: swa_evicted_seqlen must be page-aligned, {swa_evicted_seqlen=}"
|
||||
)
|
||||
|
||||
if swa_evicted_seqlen <= total_prefix_len:
|
||||
# Branch 1: entire value_slice is within SWA window — recover
|
||||
result.record_adopted_range(
|
||||
self.component_type,
|
||||
total_prefix_len,
|
||||
total_prefix_len + prefix_len,
|
||||
def _populate_range(
|
||||
self, *, node, start, end, params, result, cap_leaf, cache_actions
|
||||
):
|
||||
for fragment, start, end in self._segments(node=node, start=start, end=end):
|
||||
live_start = self._clip_to_window(
|
||||
node=fragment,
|
||||
start=start,
|
||||
end=end,
|
||||
window_start=params.get_evicted_seqlen(self.component_type),
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
old_full = full_cd.value
|
||||
if full_cd.lock_ref > 0:
|
||||
cache_actions.append(
|
||||
RecoverSWAWithLockedFull(node.id, old_full, value_slice)
|
||||
)
|
||||
return 0
|
||||
result.record_adopted_range(
|
||||
BASE_COMPONENT_TYPE,
|
||||
total_prefix_len,
|
||||
total_prefix_len + prefix_len,
|
||||
)
|
||||
full_cd.value = value_slice.clone()
|
||||
cache_actions.append(FreeDeviceKVFullOnly([old_full]))
|
||||
cache_actions.append(SWARebuild(node.id, value_slice))
|
||||
return 0
|
||||
elif swa_evicted_seqlen < total_prefix_len + prefix_len:
|
||||
# Branch 2: value_slice[start_idx:] is within SWA window — partial recover
|
||||
start_idx = swa_evicted_seqlen - total_prefix_len
|
||||
result.record_adopted_range(
|
||||
self.component_type,
|
||||
swa_evicted_seqlen,
|
||||
total_prefix_len + prefix_len,
|
||||
)
|
||||
is_locked = full_cd.lock_ref > 0
|
||||
old_full = full_cd.value[start_idx:]
|
||||
_, action = self.tree_core._split_node(node.key, node, start_idx)
|
||||
if action is not None:
|
||||
cache_actions.append(action)
|
||||
new_full = value_slice[start_idx:]
|
||||
if is_locked:
|
||||
cache_actions.append(
|
||||
RecoverSWAWithLockedFull(node.id, old_full, new_full)
|
||||
)
|
||||
return start_idx
|
||||
result.record_adopted_range(
|
||||
BASE_COMPONENT_TYPE,
|
||||
swa_evicted_seqlen,
|
||||
total_prefix_len + prefix_len,
|
||||
)
|
||||
node.component_data[BASE_COMPONENT_TYPE].value = new_full.clone()
|
||||
cache_actions.append(FreeDeviceKVFullOnly([old_full]))
|
||||
cache_actions.append(SWARebuild(node.id, new_full))
|
||||
return start_idx
|
||||
else:
|
||||
# Branch 3: entire value_slice is outside SWA window — not consumed
|
||||
return prefix_len
|
||||
if live_start is None:
|
||||
continue
|
||||
parent = self._maybe_split_leaf_for_swa_lock(fragment) if cap_leaf else None
|
||||
# Older fragments publish first so the tail remains more-MRU.
|
||||
if parent is not None:
|
||||
self._publish_window(node=parent, cache_actions=cache_actions)
|
||||
self._publish_window(node=fragment, cache_actions=cache_actions)
|
||||
result.record_adopted_range(self.component_type, live_start, end)
|
||||
|
||||
def recover_after_unevict(
|
||||
self,
|
||||
@@ -477,33 +510,14 @@ class SWAComponent(TreeComponent):
|
||||
# _unevict_node_on_insert already wrote the request's fresh KV slice
|
||||
# into the base value. We just need to rebuild SWA from that slice for
|
||||
# the in-window portion. There is no old SWA slot to free here.
|
||||
ct = self.component_type
|
||||
if node.component_data[ct].value is not None:
|
||||
return
|
||||
swa_evicted_seqlen = params.get_evicted_seqlen(self.component_type)
|
||||
assert swa_evicted_seqlen % self.tree_core.page_size == 0, (
|
||||
f"{ct}: swa_evicted_seqlen must be page-aligned, {swa_evicted_seqlen=}"
|
||||
)
|
||||
|
||||
if swa_evicted_seqlen <= total_prefix_len:
|
||||
pass # entire node is within the SWA window
|
||||
elif swa_evicted_seqlen < total_prefix_len + prefix_len:
|
||||
start_idx = swa_evicted_seqlen - total_prefix_len
|
||||
_, action = self.tree_core._split_node(node.key, node, start_idx)
|
||||
if action is not None:
|
||||
cache_actions.append(action)
|
||||
else:
|
||||
return
|
||||
result.record_adopted_range(
|
||||
self.component_type,
|
||||
max(total_prefix_len, swa_evicted_seqlen),
|
||||
total_prefix_len + prefix_len,
|
||||
)
|
||||
cache_actions.append(
|
||||
SWARebuild(
|
||||
node.id,
|
||||
node.component_data[BASE_COMPONENT_TYPE].value,
|
||||
)
|
||||
self._populate_range(
|
||||
node=node,
|
||||
start=total_prefix_len,
|
||||
end=total_prefix_len + prefix_len,
|
||||
params=params,
|
||||
result=result,
|
||||
cap_leaf=self._independent,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
|
||||
def commit_insert_component_data(
|
||||
@@ -522,39 +536,20 @@ class SWAComponent(TreeComponent):
|
||||
if not is_new_leaf:
|
||||
return
|
||||
|
||||
node_start = result.prefix_len
|
||||
node_end = node_start + len(node.key)
|
||||
swa_evicted_seqlen = params.get_evicted_seqlen(self.component_type)
|
||||
split_pos = swa_evicted_seqlen - node_start
|
||||
if split_pos >= len(node.key):
|
||||
# Entire leaf is outside the SWA window — left as a tombstone.
|
||||
return
|
||||
result.record_adopted_range(
|
||||
self.component_type,
|
||||
max(node_start, swa_evicted_seqlen),
|
||||
node_end,
|
||||
# Another window may have split this leaf; use the full aligned insert span.
|
||||
end = (
|
||||
len(params.key) // self.tree_core.page_size * self.tree_core.page_size
|
||||
if self._independent
|
||||
else result.prefix_len + len(node.key)
|
||||
)
|
||||
if split_pos > 0:
|
||||
# Node straddles the boundary: split into an out-of-window parent
|
||||
# (tombstone) and an in-window child; `node` becomes the child.
|
||||
_, action = self.tree_core._split_node(node.key, node, split_pos)
|
||||
assert action is None, "new leaf cannot be write-through-pending"
|
||||
# Cap the in-window leaf at one window for lock granularity, then rebuild SWA
|
||||
# onto the in-window node(s) at apply time; rebuild the older prefix first so
|
||||
# the in-window tail lands more-MRU.
|
||||
capped_parent = self._maybe_split_leaf_for_swa_lock(node)
|
||||
if capped_parent is not None:
|
||||
cache_actions.append(
|
||||
SWARebuild(
|
||||
capped_parent.id,
|
||||
capped_parent.component_data[BASE_COMPONENT_TYPE].value,
|
||||
)
|
||||
)
|
||||
cache_actions.append(
|
||||
SWARebuild(
|
||||
node.id,
|
||||
node.component_data[BASE_COMPONENT_TYPE].value,
|
||||
)
|
||||
self._populate_range(
|
||||
node=node,
|
||||
start=result.prefix_len,
|
||||
end=end,
|
||||
params=params,
|
||||
result=result,
|
||||
cap_leaf=True,
|
||||
cache_actions=cache_actions,
|
||||
)
|
||||
|
||||
def _maybe_split_leaf_for_swa_lock(
|
||||
|
||||
@@ -739,6 +739,25 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
)
|
||||
self._update_evictable_leaf_sets(node)
|
||||
|
||||
def dec_window_lock_only(
|
||||
self,
|
||||
node_id: NodeId,
|
||||
component_type: ComponentType,
|
||||
params: DecLockRefParams,
|
||||
) -> DecSwaLockOnlyResult:
|
||||
result = DecSwaLockOnlyResult()
|
||||
node = self.node_by_id(node_id)
|
||||
self._assert_receipt_anchor(node, params)
|
||||
if node is self.root_node or component_type in params.skipped_lock_components:
|
||||
return result
|
||||
self.components_by_type[component_type].release_window_lock(
|
||||
node,
|
||||
params.get_lock_uuid(component_type),
|
||||
result.device_frees,
|
||||
result.host_frees,
|
||||
)
|
||||
return result
|
||||
|
||||
def dec_swa_lock_only(
|
||||
self,
|
||||
node_id: NodeId,
|
||||
@@ -3038,6 +3057,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
"""Evictable token count for one component (0 if the component is absent)."""
|
||||
return self.component_evictable_size_.get(component_type, 0)
|
||||
|
||||
def component_protected_size(self, component_type: ComponentType) -> int:
|
||||
return self.component_protected_size_.get(component_type, 0)
|
||||
|
||||
def full_evictable_size(self) -> int:
|
||||
return self.evictable_size()
|
||||
|
||||
|
||||
@@ -282,6 +282,15 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
counts; the result carries the freed slots."""
|
||||
...
|
||||
|
||||
def dec_window_lock_only(
|
||||
self,
|
||||
node_id: NodeId,
|
||||
component_type: ComponentType,
|
||||
params: DecLockRefParams,
|
||||
) -> DecSwaLockOnlyResult:
|
||||
"""Release one window's receipt without releasing peer components."""
|
||||
raise NotImplementedError("This tree core does not support independent windows")
|
||||
|
||||
# ==== Device eviction (driven step-wise by the Controller's evict()) ====
|
||||
|
||||
@abstractmethod
|
||||
@@ -349,6 +358,10 @@ class UnifiedTreeCoreInterface(ABC):
|
||||
"""Evictable token count for one component (0 if the component is absent)."""
|
||||
...
|
||||
|
||||
def component_protected_size(self, component_type: ComponentType) -> int:
|
||||
"""Protected token count for one component (0 if absent)."""
|
||||
raise NotImplementedError("This tree core does not expose per-component sizes")
|
||||
|
||||
@abstractmethod
|
||||
def full_evictable_size(self) -> int: ...
|
||||
|
||||
|
||||
@@ -1744,7 +1744,9 @@ class UnifiedSWAKVPool(SWAKVPool):
|
||||
swa_phys = swa_phys[old_swa_mask][row_mask.to(indices.device)]
|
||||
if swa_phys.numel() == 0:
|
||||
return
|
||||
swa_cpu = self._filter_swa_cpu_copy(kv_cache_cpu["swa"], row_mask)
|
||||
swa_cpu = self._filter_swa_cpu_copy(
|
||||
swa_kv_cpu=kv_cache_cpu["swa"], row_mask=row_mask
|
||||
)
|
||||
self.swa_kv_pool.load_cpu_copy(swa_cpu, swa_phys)
|
||||
|
||||
|
||||
|
||||
@@ -1878,29 +1878,18 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
spec_info = self.spec_info
|
||||
self.output_cache_loc_backup = self.out_cache_loc
|
||||
self.hidden_states_backup = spec_info.hidden_states
|
||||
# spec_info is EagleDraftInput | EagleDraftExtendInput; each carries
|
||||
# a disjoint subset of the fields below, so getattr-guard each one.
|
||||
# Draft input types carry different subsets of the shared fields.
|
||||
if getattr(spec_info, "topk_p", None) is not None:
|
||||
spec_info.topk_p = self._pad_tensor_to_size(spec_info.topk_p, bs)
|
||||
if getattr(spec_info, "topk_index", None) is not None:
|
||||
spec_info.topk_index = self._pad_tensor_to_size(
|
||||
spec_info.topk_index, bs
|
||||
)
|
||||
if getattr(spec_info, "draft_probs", None) is not None:
|
||||
spec_info.draft_probs = self._pad_tensor_to_size(
|
||||
spec_info.draft_probs, bs
|
||||
)
|
||||
if getattr(spec_info, "num_correct_drafts", None) is not None:
|
||||
spec_info.num_correct_drafts = self._pad_tensor_to_size(
|
||||
spec_info.num_correct_drafts, bs
|
||||
)
|
||||
spec_info.num_accept_tokens = self._pad_tensor_to_size(
|
||||
spec_info.num_accept_tokens, bs
|
||||
)
|
||||
if spec_info.hidden_states is not None:
|
||||
spec_info.hidden_states = self._pad_tensor_to_size(
|
||||
spec_info.hidden_states, num_tokens
|
||||
)
|
||||
spec_info.pad_batch(self._pad_tensor_to_size, bs)
|
||||
|
||||
def _forward_num_tokens(self) -> int:
|
||||
"""Token width of this forward (``input_embeds`` over ``input_ids``), as
|
||||
|
||||
@@ -809,7 +809,13 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
|
||||
return self._solve_pool_sizes(max_total_num_tokens, page_size)
|
||||
|
||||
|
||||
def compute_swa_request_cap(*, page_size: int, window: int, attn_dp_size: int) -> int:
|
||||
def compute_swa_request_cap(
|
||||
*,
|
||||
page_size: int,
|
||||
window: int,
|
||||
attn_dp_size: int,
|
||||
max_running_requests: int | None = None,
|
||||
) -> int:
|
||||
"""Worst-case SWA slots the scheduler holds live at max_running_requests."""
|
||||
draft_tokens = get_spec().speculative_num_draft_tokens or 1
|
||||
eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get())
|
||||
@@ -833,7 +839,9 @@ def compute_swa_request_cap(*, page_size: int, window: int, attn_dp_size: int) -
|
||||
decode_alloc = 2 * get_alloc_len_per_decode()
|
||||
per_request = trailing_tokens + decode_alloc
|
||||
|
||||
num_reqs = get_schedule().max_running_requests // attn_dp_size
|
||||
if max_running_requests is None:
|
||||
max_running_requests = get_schedule().max_running_requests
|
||||
num_reqs = max_running_requests // attn_dp_size
|
||||
if get_disagg().disaggregation_mode == "decode":
|
||||
return (
|
||||
per_request * num_reqs
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Optional
|
||||
from typing import Callable, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
@@ -180,6 +180,14 @@ class EagleDraftInput(SpecInput):
|
||||
def __post_init__(self):
|
||||
super().__init__(SpecInputType.EAGLE_DRAFT)
|
||||
|
||||
def pad_batch(
|
||||
self,
|
||||
pad_tensor_to_size: Callable[..., torch.Tensor],
|
||||
batch_size: int,
|
||||
) -> None:
|
||||
if self.draft_probs is not None:
|
||||
self.draft_probs = pad_tensor_to_size(self.draft_probs, batch_size)
|
||||
|
||||
@classmethod
|
||||
def create_idle_input(
|
||||
cls,
|
||||
@@ -328,6 +336,19 @@ class EagleDraftExtendInput(SpecInput):
|
||||
def __post_init__(self):
|
||||
super().__init__(SpecInputType.EAGLE_DRAFT_EXTEND)
|
||||
|
||||
def pad_batch(
|
||||
self,
|
||||
pad_tensor_to_size: Callable[..., torch.Tensor],
|
||||
batch_size: int,
|
||||
) -> None:
|
||||
if self.num_correct_drafts is not None:
|
||||
self.num_correct_drafts = pad_tensor_to_size(
|
||||
self.num_correct_drafts, batch_size
|
||||
)
|
||||
self.num_accept_tokens = pad_tensor_to_size(
|
||||
self.num_accept_tokens, batch_size
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def create_idle_input(
|
||||
cls,
|
||||
|
||||
@@ -432,6 +432,13 @@ class SpecInput(ABC):
|
||||
SpecInputType.UNO_VERIFY,
|
||||
}
|
||||
|
||||
def pad_batch(
|
||||
self,
|
||||
pad_tensor_to_size: Callable[..., torch.Tensor],
|
||||
batch_size: int,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def spec_scale_global_num_tokens(
|
||||
spec_info: SpecInput,
|
||||
|
||||
Reference in New Issue
Block a user