Make sliding-window caching and speculative batch padding extensible (#41325)

Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
Co-authored-by: jasonjk-park <jasonjk@fb.com>
This commit is contained in:
Lianmin Zheng
2026-09-26 11:24:02 -07:00
committed by GitHub
co-authored by Lu Fang jasonjk-park
parent c73f7077eb
commit c10fa03fc4
17 changed files with 430 additions and 179 deletions
+15 -13
View File
@@ -115,7 +115,6 @@ from sglang.srt.mem_cache.common import (
RetractionBackup,
backup_kv_cache,
evict_from_tree_cache,
free_swa_out_of_window_slots,
release_kv_cache,
)
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
@@ -3882,6 +3881,8 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
def maybe_evict_swa(self):
if self.tree_cache.supports_swa():
sliding_window_size = self.tree_cache.sliding_window_size
# Auxiliary windows check their own cursors and prefix locks.
has_auxiliary_swa = self.tree_cache.supports_auxiliary_swa()
release_leaf_lock = (
envs.SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW.get()
@@ -3901,9 +3902,17 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
if (
req.decode_batch_idx >= 1
and req.kv.holds_kv
and swa_evict_due(req)
and (has_auxiliary_swa or swa_evict_due(req))
):
self._evict_swa(req, req.seqlen - 1)
# The scheduler already gated the primary window,
# including backends that use a forward-based cadence.
self._evict_swa(
req,
req.seqlen - 1,
eviction_interval=(
eviction_interval if has_auxiliary_swa else 1
),
)
# Once the decode position has moved past the sliding window,
# the SWA portion of the prefill-time tree lock is no longer
@@ -3938,17 +3947,10 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
self._evict_swa(req, pre_len)
self.token_to_kv_pool_allocator.free_group_end()
def _evict_swa(self, req: Req, pre_len: int):
def _evict_swa(self, req: Req, pre_len: int, *, eviction_interval: int = 1):
assert self.tree_cache.supports_swa(), "prefix cache must support swa"
free_swa_out_of_window_slots(
req,
pre_len,
sliding_window_size=self.tree_cache.sliding_window_size,
page_size=self.tree_cache.page_size,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
is_chunk_cache=self.tree_cache.is_chunk_cache(),
retain_floor=self.tree_cache.swa_retain_floor(req),
self.tree_cache.evict_sliding_windows(
req, pre_len, eviction_interval=eviction_interval
)
def __str__(self):
@@ -605,6 +605,27 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
def supports_swa(self) -> bool:
return False
def supports_auxiliary_swa(self) -> bool:
return False
def evict_sliding_windows(
self, req: Req, pre_len: int, *, eviction_interval: int = 1
) -> None:
"""Slide request-owned windows at the scheduler's safe eviction frontier."""
from sglang.srt.mem_cache.common import free_swa_out_of_window_slots
free_swa_out_of_window_slots(
req,
pre_len,
sliding_window_size=self.sliding_window_size,
page_size=self.page_size,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
is_chunk_cache=self.is_chunk_cache(),
retain_floor=self.swa_retain_floor(req),
eviction_interval=eviction_interval,
)
def swa_retain_floor(self, req) -> int | None:
# A match lands on a state checkpoint rather than on the tail, so a cache
# that pairs SWA with mamba/conv checkpoints has to keep the window behind
+23 -12
View File
@@ -1,7 +1,7 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any, NamedTuple, Optional, cast
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, cast
import numpy as np
import torch
@@ -62,6 +62,9 @@ def free_swa_out_of_window_slots(
token_to_kv_pool_allocator: BaseTokenToKVPoolAllocator,
is_chunk_cache: bool = False,
retain_floor: int | None = None,
component_type: ComponentType = ComponentType.SWA,
free_segment: Callable[..., None] | None = None,
eviction_interval: int = 1,
) -> None:
if not req.kv.holds_kv:
return
@@ -70,10 +73,17 @@ def free_swa_out_of_window_slots(
assert req.kv.cache_protected_len % page_size == 0, (
"cache_protected_len must be page aligned"
)
swa_evicted_seqlen = max(
req.kv.get_evicted_seqlen(ComponentType.SWA), req.kv.swa_dead_lo(page_size)
# Protected rows limit what can be freed, not where the interval starts.
evicted_seqlen = req.kv.get_evicted_seqlen(component_type)
if pre_len - sliding_window_size < evicted_seqlen + eviction_interval:
return
dead_lo = (
req.kv.swa_dead_lo(page_size)
if component_type == ComponentType.SWA
else req.kv.cache_protected_len
)
req.kv.set_evicted_seqlen(ComponentType.SWA, swa_evicted_seqlen)
evicted_seqlen = max(evicted_seqlen, dead_lo)
req.kv.set_evicted_seqlen(component_type, evicted_seqlen)
if is_chunk_cache:
# Chunk cache builds no radix tree, so no tombstone-leaf concern; evict
@@ -91,19 +101,20 @@ def free_swa_out_of_window_slots(
# retained checkpoint could never be matched and holding it is pure cost.
evict_threshold = min(evict_threshold, retain_floor)
new_swa_evicted_seqlen = max(swa_evicted_seqlen, evict_threshold)
new_evicted_seqlen = max(evicted_seqlen, evict_threshold)
if page_size > 1:
new_swa_evicted_seqlen = (new_swa_evicted_seqlen // page_size) * page_size
new_evicted_seqlen = (new_evicted_seqlen // page_size) * page_size
if new_swa_evicted_seqlen > swa_evicted_seqlen:
if new_evicted_seqlen > evicted_seqlen:
free_slots = req_to_token_pool.req_to_token[
req.kv.req_pool_idx, swa_evicted_seqlen:new_swa_evicted_seqlen
req.kv.req_pool_idx, evicted_seqlen:new_evicted_seqlen
]
token_to_kv_pool_allocator.free_swa_segment(
free_slots, start_pos=swa_evicted_seqlen
)
req.kv.set_evicted_seqlen(ComponentType.SWA, new_swa_evicted_seqlen)
if free_segment is None:
assert component_type == ComponentType.SWA
free_segment = token_to_kv_pool_allocator.free_swa_segment
free_segment(free_slots, start_pos=evicted_seqlen)
req.kv.set_evicted_seqlen(component_type, new_evicted_seqlen)
def coalesce_ranges(ranges: list[tuple[int, int]]) -> list[tuple[int, int]]:
+9 -8
View File
@@ -21,6 +21,7 @@ from sglang.srt.runtime_context import get_disagg, get_memory, get_serving
if TYPE_CHECKING:
from sglang.srt.configs.model_config import ModelConfig
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.server_args import ServerArgs
logger = logging.getLogger(__name__)
@@ -78,14 +79,13 @@ def registered_radix_cache_backends() -> list[str]:
def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
"""Built-in Radix Cache selection chain."""
server_args = ctx.server_args
params = ctx.params
if (
ctx.disable_radix_cache
and get_disagg().disaggregation_decode_retraction_backup == "host_pool"
):
return _create_unified_radix_cache(ctx, server_args, params)
return create_unified_radix_cache(ctx)
if ctx.effective_chunked_prefill_size is not None and ctx.disable_radix_cache:
if not ctx.is_hybrid_swa:
@@ -124,7 +124,7 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
)
if get_memory().enable_unified_cache_external_linker:
return _create_unified_radix_cache(ctx, server_args, params)
return create_unified_radix_cache(ctx)
if ctx.is_hybrid_swa and ctx.full_tokens_per_layer == 0:
from sglang.srt.mem_cache.pure_swa_radix_cache import PureSWARadixCache
@@ -145,15 +145,16 @@ def default_radix_cache_factory(ctx: TreeCacheBuildContext) -> BasePrefixCache:
os.environ["FLEXKV_CONFIG_PATH"] = get_memory().flexkv_config_file
return _flexkv_factory(ctx)
return _create_unified_radix_cache(ctx, server_args, params)
return create_unified_radix_cache(ctx)
def _create_unified_radix_cache(
def create_unified_radix_cache(
ctx: TreeCacheBuildContext,
server_args: ServerArgs,
params: CacheInitParams,
*,
cache_class: type[UnifiedRadixCache] | None = None,
) -> BasePrefixCache:
"""Initialize a UnifiedRadixCache with proper components and optional HiCache."""
server_args, params = ctx.server_args, ctx.params
if get_disagg().disaggregation_decode_retraction_backup == "host_pool":
if ctx.is_hybrid_ssm:
raise ValueError("Host-pool retraction does not support Mamba models.")
@@ -189,7 +190,7 @@ def _create_unified_radix_cache(
params.component_registry_override = {
ComponentType.MAMBA: MlxAuxiliaryStateComponent,
}
cache = UnifiedRadixCache(params)
cache = (cache_class or UnifiedRadixCache)(params)
if (
ctx.enable_hierarchical_cache
or get_disagg().disaggregation_decode_retraction_backup == "host_pool"
@@ -337,7 +337,7 @@ class SWAKVPool(BaseSWAKVPool):
src_loc_swa = self.translate_loc_from_full_to_swa(src_loc)
self.swa_kv_pool.move_kv_cache(tgt_loc_swa, src_loc_swa)
def _filter_swa_cpu_copy(self, swa_kv_cpu, row_mask: torch.Tensor):
def _filter_swa_cpu_copy(self, *, swa_kv_cpu, row_mask: torch.Tensor):
if swa_kv_cpu is None:
return None
if row_mask is None or bool(torch.all(row_mask).item()):
@@ -419,5 +419,7 @@ class SWAKVPool(BaseSWAKVPool):
if swa_indices.numel() == 0:
return
swa_kv_cpu = self._filter_swa_cpu_copy(swa_kv_cpu, row_mask)
swa_kv_cpu = self._filter_swa_cpu_copy(
swa_kv_cpu=swa_kv_cpu, row_mask=row_mask
)
self.swa_kv_pool.load_cpu_copy(swa_kv_cpu, swa_indices)
@@ -88,6 +88,7 @@ class SWAComponent(TreeComponent):
self._swa_kv_pool_host = None
component_type = ComponentType.SWA
_independent = False
def _collect_unbacked_swa_nodes(
self, node: UnifiedTreeNode
@@ -385,6 +386,54 @@ class SWAComponent(TreeComponent):
swa_branching_seqlen=branching_seqlen,
)
@staticmethod
def _segments(*, node, start, end):
# Earlier components may have split the original insertion span.
segments = []
while end > start:
node_start = end - len(node.key)
assert node_start >= start
segments.append((node, node_start, end))
end, node = node_start, node.parent
return reversed(segments)
def _clip_to_window(self, *, node, start, end, window_start, cache_actions):
if node.component_data[self.component_type].value is not None:
return None
assert window_start % self.tree_core.page_size == 0, (
f"{self.component_type}: swa_evicted_seqlen must be page-aligned, {window_start=}"
)
live_start = max(start, window_start)
if live_start >= end:
return None
if live_start > start:
_, action = self.tree_core._split_node(node.key, node, live_start - start)
if action is not None:
cache_actions.append(action)
return live_start
def _publish_window(self, *, node, cache_actions):
cache_actions.append(
SWARebuild(node.id, node.component_data[BASE_COMPONENT_TYPE].value)
)
def _recover_from_incoming(
self, *, node, incoming_full, start, end, result, cache_actions
):
full_cd = node.component_data[BASE_COMPONENT_TYPE]
old_full = full_cd.value
# A locked SWA tombstone also has Full locked, so only an unlocked
# Full value can be replaced by the incoming ids.
if full_cd.lock_ref > 0:
cache_actions.append(
RecoverSWAWithLockedFull(node.id, old_full, incoming_full)
)
return
result.record_adopted_range(BASE_COMPONENT_TYPE, start, end)
full_cd.value = incoming_full.clone()
cache_actions.append(FreeDeviceKVFullOnly([old_full]))
self._publish_window(node=node, cache_actions=cache_actions)
def update_component_on_insert_overlap(
self,
node: UnifiedTreeNode,
@@ -398,72 +447,56 @@ class SWAComponent(TreeComponent):
if params.prev_prefix_len >= total_prefix_len + prefix_len:
return prefix_len
is_tombstone = node.component_data[self.component_type].value is None
if not is_tombstone:
return prefix_len
window_start = params.get_evicted_seqlen(self.component_type)
if self._independent:
window_start = max(window_start, params.prev_prefix_len)
consumed_from = prefix_len
for fragment, start, end in self._segments(
node=node, start=total_prefix_len, end=total_prefix_len + prefix_len
):
live_start = self._clip_to_window(
node=fragment,
start=start,
end=end,
window_start=window_start,
cache_actions=cache_actions,
)
if live_start is None:
continue
result.record_adopted_range(self.component_type, live_start, end)
self._recover_from_incoming(
node=fragment,
incoming_full=value_slice[
live_start - total_prefix_len : end - total_prefix_len
],
start=live_start,
end=end,
result=result,
cache_actions=cache_actions,
)
if not self._independent:
consumed_from = min(consumed_from, live_start - total_prefix_len)
return consumed_from
full_cd = node.component_data[BASE_COMPONENT_TYPE]
swa_evicted_seqlen = params.get_evicted_seqlen(self.component_type)
# A locked tombstone is legal (segment locks count every node); the
# full-value swap below is safe because full lock_ref >= swa
# lock_ref, so a locked-SWA node always takes the Recover branch.
assert swa_evicted_seqlen % self.tree_core.page_size == 0, (
f"{self.component_type}: swa_evicted_seqlen must be page-aligned, {swa_evicted_seqlen=}"
)
if swa_evicted_seqlen <= total_prefix_len:
# Branch 1: entire value_slice is within SWA window — recover
result.record_adopted_range(
self.component_type,
total_prefix_len,
total_prefix_len + prefix_len,
def _populate_range(
self, *, node, start, end, params, result, cap_leaf, cache_actions
):
for fragment, start, end in self._segments(node=node, start=start, end=end):
live_start = self._clip_to_window(
node=fragment,
start=start,
end=end,
window_start=params.get_evicted_seqlen(self.component_type),
cache_actions=cache_actions,
)
old_full = full_cd.value
if full_cd.lock_ref > 0:
cache_actions.append(
RecoverSWAWithLockedFull(node.id, old_full, value_slice)
)
return 0
result.record_adopted_range(
BASE_COMPONENT_TYPE,
total_prefix_len,
total_prefix_len + prefix_len,
)
full_cd.value = value_slice.clone()
cache_actions.append(FreeDeviceKVFullOnly([old_full]))
cache_actions.append(SWARebuild(node.id, value_slice))
return 0
elif swa_evicted_seqlen < total_prefix_len + prefix_len:
# Branch 2: value_slice[start_idx:] is within SWA window — partial recover
start_idx = swa_evicted_seqlen - total_prefix_len
result.record_adopted_range(
self.component_type,
swa_evicted_seqlen,
total_prefix_len + prefix_len,
)
is_locked = full_cd.lock_ref > 0
old_full = full_cd.value[start_idx:]
_, action = self.tree_core._split_node(node.key, node, start_idx)
if action is not None:
cache_actions.append(action)
new_full = value_slice[start_idx:]
if is_locked:
cache_actions.append(
RecoverSWAWithLockedFull(node.id, old_full, new_full)
)
return start_idx
result.record_adopted_range(
BASE_COMPONENT_TYPE,
swa_evicted_seqlen,
total_prefix_len + prefix_len,
)
node.component_data[BASE_COMPONENT_TYPE].value = new_full.clone()
cache_actions.append(FreeDeviceKVFullOnly([old_full]))
cache_actions.append(SWARebuild(node.id, new_full))
return start_idx
else:
# Branch 3: entire value_slice is outside SWA window — not consumed
return prefix_len
if live_start is None:
continue
parent = self._maybe_split_leaf_for_swa_lock(fragment) if cap_leaf else None
# Older fragments publish first so the tail remains more-MRU.
if parent is not None:
self._publish_window(node=parent, cache_actions=cache_actions)
self._publish_window(node=fragment, cache_actions=cache_actions)
result.record_adopted_range(self.component_type, live_start, end)
def recover_after_unevict(
self,
@@ -477,33 +510,14 @@ class SWAComponent(TreeComponent):
# _unevict_node_on_insert already wrote the request's fresh KV slice
# into the base value. We just need to rebuild SWA from that slice for
# the in-window portion. There is no old SWA slot to free here.
ct = self.component_type
if node.component_data[ct].value is not None:
return
swa_evicted_seqlen = params.get_evicted_seqlen(self.component_type)
assert swa_evicted_seqlen % self.tree_core.page_size == 0, (
f"{ct}: swa_evicted_seqlen must be page-aligned, {swa_evicted_seqlen=}"
)
if swa_evicted_seqlen <= total_prefix_len:
pass # entire node is within the SWA window
elif swa_evicted_seqlen < total_prefix_len + prefix_len:
start_idx = swa_evicted_seqlen - total_prefix_len
_, action = self.tree_core._split_node(node.key, node, start_idx)
if action is not None:
cache_actions.append(action)
else:
return
result.record_adopted_range(
self.component_type,
max(total_prefix_len, swa_evicted_seqlen),
total_prefix_len + prefix_len,
)
cache_actions.append(
SWARebuild(
node.id,
node.component_data[BASE_COMPONENT_TYPE].value,
)
self._populate_range(
node=node,
start=total_prefix_len,
end=total_prefix_len + prefix_len,
params=params,
result=result,
cap_leaf=self._independent,
cache_actions=cache_actions,
)
def commit_insert_component_data(
@@ -522,39 +536,20 @@ class SWAComponent(TreeComponent):
if not is_new_leaf:
return
node_start = result.prefix_len
node_end = node_start + len(node.key)
swa_evicted_seqlen = params.get_evicted_seqlen(self.component_type)
split_pos = swa_evicted_seqlen - node_start
if split_pos >= len(node.key):
# Entire leaf is outside the SWA window — left as a tombstone.
return
result.record_adopted_range(
self.component_type,
max(node_start, swa_evicted_seqlen),
node_end,
# Another window may have split this leaf; use the full aligned insert span.
end = (
len(params.key) // self.tree_core.page_size * self.tree_core.page_size
if self._independent
else result.prefix_len + len(node.key)
)
if split_pos > 0:
# Node straddles the boundary: split into an out-of-window parent
# (tombstone) and an in-window child; `node` becomes the child.
_, action = self.tree_core._split_node(node.key, node, split_pos)
assert action is None, "new leaf cannot be write-through-pending"
# Cap the in-window leaf at one window for lock granularity, then rebuild SWA
# onto the in-window node(s) at apply time; rebuild the older prefix first so
# the in-window tail lands more-MRU.
capped_parent = self._maybe_split_leaf_for_swa_lock(node)
if capped_parent is not None:
cache_actions.append(
SWARebuild(
capped_parent.id,
capped_parent.component_data[BASE_COMPONENT_TYPE].value,
)
)
cache_actions.append(
SWARebuild(
node.id,
node.component_data[BASE_COMPONENT_TYPE].value,
)
self._populate_range(
node=node,
start=result.prefix_len,
end=end,
params=params,
result=result,
cap_leaf=True,
cache_actions=cache_actions,
)
def _maybe_split_leaf_for_swa_lock(
@@ -739,6 +739,25 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
)
self._update_evictable_leaf_sets(node)
def dec_window_lock_only(
self,
node_id: NodeId,
component_type: ComponentType,
params: DecLockRefParams,
) -> DecSwaLockOnlyResult:
result = DecSwaLockOnlyResult()
node = self.node_by_id(node_id)
self._assert_receipt_anchor(node, params)
if node is self.root_node or component_type in params.skipped_lock_components:
return result
self.components_by_type[component_type].release_window_lock(
node,
params.get_lock_uuid(component_type),
result.device_frees,
result.host_frees,
)
return result
def dec_swa_lock_only(
self,
node_id: NodeId,
@@ -3038,6 +3057,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
"""Evictable token count for one component (0 if the component is absent)."""
return self.component_evictable_size_.get(component_type, 0)
def component_protected_size(self, component_type: ComponentType) -> int:
return self.component_protected_size_.get(component_type, 0)
def full_evictable_size(self) -> int:
return self.evictable_size()
@@ -282,6 +282,15 @@ class UnifiedTreeCoreInterface(ABC):
counts; the result carries the freed slots."""
...
def dec_window_lock_only(
self,
node_id: NodeId,
component_type: ComponentType,
params: DecLockRefParams,
) -> DecSwaLockOnlyResult:
"""Release one window's receipt without releasing peer components."""
raise NotImplementedError("This tree core does not support independent windows")
# ==== Device eviction (driven step-wise by the Controller's evict()) ====
@abstractmethod
@@ -349,6 +358,10 @@ class UnifiedTreeCoreInterface(ABC):
"""Evictable token count for one component (0 if the component is absent)."""
...
def component_protected_size(self, component_type: ComponentType) -> int:
"""Protected token count for one component (0 if absent)."""
raise NotImplementedError("This tree core does not expose per-component sizes")
@abstractmethod
def full_evictable_size(self) -> int: ...
@@ -1744,7 +1744,9 @@ class UnifiedSWAKVPool(SWAKVPool):
swa_phys = swa_phys[old_swa_mask][row_mask.to(indices.device)]
if swa_phys.numel() == 0:
return
swa_cpu = self._filter_swa_cpu_copy(kv_cache_cpu["swa"], row_mask)
swa_cpu = self._filter_swa_cpu_copy(
swa_kv_cpu=kv_cache_cpu["swa"], row_mask=row_mask
)
self.swa_kv_pool.load_cpu_copy(swa_cpu, swa_phys)
@@ -1878,29 +1878,18 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
spec_info = self.spec_info
self.output_cache_loc_backup = self.out_cache_loc
self.hidden_states_backup = spec_info.hidden_states
# spec_info is EagleDraftInput | EagleDraftExtendInput; each carries
# a disjoint subset of the fields below, so getattr-guard each one.
# Draft input types carry different subsets of the shared fields.
if getattr(spec_info, "topk_p", None) is not None:
spec_info.topk_p = self._pad_tensor_to_size(spec_info.topk_p, bs)
if getattr(spec_info, "topk_index", None) is not None:
spec_info.topk_index = self._pad_tensor_to_size(
spec_info.topk_index, bs
)
if getattr(spec_info, "draft_probs", None) is not None:
spec_info.draft_probs = self._pad_tensor_to_size(
spec_info.draft_probs, bs
)
if getattr(spec_info, "num_correct_drafts", None) is not None:
spec_info.num_correct_drafts = self._pad_tensor_to_size(
spec_info.num_correct_drafts, bs
)
spec_info.num_accept_tokens = self._pad_tensor_to_size(
spec_info.num_accept_tokens, bs
)
if spec_info.hidden_states is not None:
spec_info.hidden_states = self._pad_tensor_to_size(
spec_info.hidden_states, num_tokens
)
spec_info.pad_batch(self._pad_tensor_to_size, bs)
def _forward_num_tokens(self) -> int:
"""Token width of this forward (``input_embeds`` over ``input_ids``), as
@@ -809,7 +809,13 @@ class HybridSWAPoolConfigurator(MemoryPoolConfigurator):
return self._solve_pool_sizes(max_total_num_tokens, page_size)
def compute_swa_request_cap(*, page_size: int, window: int, attn_dp_size: int) -> int:
def compute_swa_request_cap(
*,
page_size: int,
window: int,
attn_dp_size: int,
max_running_requests: int | None = None,
) -> int:
"""Worst-case SWA slots the scheduler holds live at max_running_requests."""
draft_tokens = get_spec().speculative_num_draft_tokens or 1
eviction_interval = max(1, envs.SGLANG_SWA_EVICTION_INTERVAL.get())
@@ -833,7 +839,9 @@ def compute_swa_request_cap(*, page_size: int, window: int, attn_dp_size: int) -
decode_alloc = 2 * get_alloc_len_per_decode()
per_request = trailing_tokens + decode_alloc
num_reqs = get_schedule().max_running_requests // attn_dp_size
if max_running_requests is None:
max_running_requests = get_schedule().max_running_requests
num_reqs = max_running_requests // attn_dp_size
if get_disagg().disaggregation_mode == "decode":
return (
per_request * num_reqs
+22 -1
View File
@@ -1,6 +1,6 @@
import logging
from dataclasses import dataclass, field
from typing import List, Optional
from typing import Callable, List, Optional
import torch
@@ -180,6 +180,14 @@ class EagleDraftInput(SpecInput):
def __post_init__(self):
super().__init__(SpecInputType.EAGLE_DRAFT)
def pad_batch(
self,
pad_tensor_to_size: Callable[..., torch.Tensor],
batch_size: int,
) -> None:
if self.draft_probs is not None:
self.draft_probs = pad_tensor_to_size(self.draft_probs, batch_size)
@classmethod
def create_idle_input(
cls,
@@ -328,6 +336,19 @@ class EagleDraftExtendInput(SpecInput):
def __post_init__(self):
super().__init__(SpecInputType.EAGLE_DRAFT_EXTEND)
def pad_batch(
self,
pad_tensor_to_size: Callable[..., torch.Tensor],
batch_size: int,
) -> None:
if self.num_correct_drafts is not None:
self.num_correct_drafts = pad_tensor_to_size(
self.num_correct_drafts, batch_size
)
self.num_accept_tokens = pad_tensor_to_size(
self.num_accept_tokens, batch_size
)
@classmethod
def create_idle_input(
cls,
@@ -432,6 +432,13 @@ class SpecInput(ABC):
SpecInputType.UNO_VERIFY,
}
def pad_batch(
self,
pad_tensor_to_size: Callable[..., torch.Tensor],
batch_size: int,
) -> None:
return None
def spec_scale_global_num_tokens(
spec_info: SpecInput,