[Scheduler] Refresh waiting requests' cached prefixes under FCFS-like policies with LRU eviction (#41593)
Signed-off-by: Pei Li <peili@nvidia.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Po-Han Huang <pohanh@nvidia.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
co-authored by
Po-Han Huang
Po-Han Huang
Claude Opus 5.5
hnyls2002
parent
2b94cb4eb5
commit
20d1f3f56a
@@ -602,6 +602,7 @@ class Envs:
|
||||
# ===================================================================
|
||||
# Scheduler token budgeting and admission
|
||||
# ===================================================================
|
||||
SGLANG_ENABLE_WAITING_PREFIX_REFRESH = EnvBool(True)
|
||||
SGLANG_INIT_NEW_TOKEN_RATIO = EnvFloat(0.7)
|
||||
SGLANG_MIN_NEW_TOKEN_RATIO_FACTOR = EnvFloat(0.14)
|
||||
SGLANG_NEW_TOKEN_RATIO_DECAY_STEPS = EnvInt(600)
|
||||
|
||||
@@ -7,6 +7,7 @@ from sglang.srt.managers.prefill_delayer import PrefillDelayerSinglePassExecutor
|
||||
from sglang.srt.runtime_context import (
|
||||
get_disagg,
|
||||
get_exec,
|
||||
get_memory,
|
||||
get_schedule,
|
||||
)
|
||||
from sglang.srt.utils import get_bool_env_var, is_gfx95_supported, is_hip
|
||||
@@ -31,6 +32,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from collections import Counter
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
@@ -66,7 +68,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
MatchPrefixParams,
|
||||
zero_match_result,
|
||||
)
|
||||
from sglang.srt.mem_cache.common import match_kv_cache
|
||||
from sglang.srt.mem_cache.common import match_kv_cache, touch_waiting_prefix
|
||||
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
|
||||
from sglang.srt.mem_cache.unified_cache.components import (
|
||||
CacheTransferPhase,
|
||||
@@ -128,6 +130,11 @@ if PREFILL_TILE_BUDGET_MODE not in {"legacy", "compact"}:
|
||||
)
|
||||
PREFILL_TILE_BUDGET_MODE = "compact"
|
||||
|
||||
# Arbitrary; bounds per-round prefix work on the waiting queue.
|
||||
WAITING_QUEUE_PREFIX_MATCH_MAX = 128
|
||||
# Arbitrary; bounds the walk cost of touching waiting prefixes.
|
||||
WAITING_PREFIX_REFRESH_INTERVAL_S = 0.5
|
||||
|
||||
|
||||
def _ceil_div(value: int, divisor: int) -> int:
|
||||
return -(-value // divisor)
|
||||
@@ -195,6 +202,7 @@ class SchedulePolicy:
|
||||
self.schedule_low_priority_values_first = schedule_low_priority_values_first
|
||||
self.priority_sign = 1 if schedule_low_priority_values_first else -1
|
||||
self._shortest_prefill_calls = 0
|
||||
self._last_waiting_prefix_refresh = float("-inf")
|
||||
|
||||
# It is used to find the matching prefix for in-batch prefix caching.
|
||||
self.waiting_queue_radix_tree = RadixCache.create_simulated()
|
||||
@@ -222,9 +230,7 @@ class SchedulePolicy:
|
||||
SchedulePolicy._sort_by_priority_and_fcfs(
|
||||
waiting_queue, self.priority_sign
|
||||
)
|
||||
return
|
||||
|
||||
if isinstance(policy, CacheAwarePolicy):
|
||||
elif isinstance(policy, CacheAwarePolicy):
|
||||
temporary_deprioritized = self._compute_prefix_matches(
|
||||
waiting_queue, policy
|
||||
)
|
||||
@@ -268,6 +274,28 @@ class SchedulePolicy:
|
||||
else:
|
||||
raise ValueError(f"Unknown CacheAgnostic Policy: {policy=}")
|
||||
|
||||
self._touch_waiting_prefixes(policy, waiting_queue)
|
||||
|
||||
def _touch_waiting_prefixes(self, policy: Policy, waiting_queue: List[Req]) -> None:
|
||||
# Head last, so LRU evicts in reverse admission order; cache-aware policies
|
||||
# already touch while matching, and other eviction strategies ignore recency.
|
||||
if (
|
||||
isinstance(policy, CacheAwarePolicy)
|
||||
or not waiting_queue
|
||||
or not envs.SGLANG_ENABLE_WAITING_PREFIX_REFRESH.get()
|
||||
or get_disagg().disaggregation_mode == "decode"
|
||||
or get_memory().radix_eviction_policy.lower() != "lru"
|
||||
or self.tree_cache.supports_fast_match_prefix()
|
||||
or not self.tree_cache.supports_prefix_sharing()
|
||||
):
|
||||
return
|
||||
now = time.monotonic()
|
||||
if now - self._last_waiting_prefix_refresh < WAITING_PREFIX_REFRESH_INTERVAL_S:
|
||||
return
|
||||
self._last_waiting_prefix_refresh = now
|
||||
for r in reversed(waiting_queue[:WAITING_QUEUE_PREFIX_MATCH_MAX]):
|
||||
touch_waiting_prefix(r, self.tree_cache)
|
||||
|
||||
def _determine_active_policy(self, waiting_queue: List[Req]) -> Policy:
|
||||
if (
|
||||
self.policy
|
||||
@@ -275,7 +303,7 @@ class SchedulePolicy:
|
||||
CacheAwarePolicy.LPM,
|
||||
CacheAwarePolicy.HRRN,
|
||||
)
|
||||
and len(waiting_queue) > 128
|
||||
and len(waiting_queue) > WAITING_QUEUE_PREFIX_MATCH_MAX
|
||||
):
|
||||
# Turn off the expensive prefix matching and sorting when the #queue is large.
|
||||
return CacheAgnosticPolicy.FCFS
|
||||
|
||||
@@ -440,6 +440,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
def supports_fast_match_prefix(self) -> bool:
|
||||
return False
|
||||
|
||||
def touch_prefix(self, key: RadixKey) -> None:
|
||||
"""Bump LRU recency along ``key``'s cached path. Unlike ``match_prefix``,
|
||||
never allocates, loads, or queries external tiers."""
|
||||
|
||||
def dfs_weight_order(self, node_handles: Sequence[Any]) -> list[int]:
|
||||
"""Return request indices in depth-first, subtree-weight order."""
|
||||
return _dfs_weight_order(self.root_node, node_handles, self.resolve_node_handle)
|
||||
|
||||
@@ -172,6 +172,34 @@ def free_kv_row_segments(
|
||||
allocator.free_segments(swa_alive)
|
||||
|
||||
|
||||
def _req_radix_key(
|
||||
tree_cache: BasePrefixCache,
|
||||
req: Req,
|
||||
token_ids: array[int],
|
||||
max_prefix_len: Optional[int] = None,
|
||||
) -> RadixKey:
|
||||
# unified_kv SWA lives in a per-request ring the tree never stores, so a reused
|
||||
# prefix carries stale SWA; cap the match so the window is re-prefilled.
|
||||
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
|
||||
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
|
||||
if max_prefix_len is not None:
|
||||
key_limit = (
|
||||
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
|
||||
)
|
||||
return RadixKey(
|
||||
token_ids=token_ids,
|
||||
extra_key=req.extra_key,
|
||||
limit=key_limit,
|
||||
cache_salt=req.cache_salt,
|
||||
)
|
||||
|
||||
|
||||
def touch_waiting_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
tree_cache.touch_prefix(
|
||||
_req_radix_key(tree_cache, req, req.origin_input_ids + req.output_ids)
|
||||
)
|
||||
|
||||
|
||||
def match_kv_cache(
|
||||
req: Req,
|
||||
tree_cache: BasePrefixCache,
|
||||
@@ -184,23 +212,9 @@ def match_kv_cache(
|
||||
if token_ids is None:
|
||||
token_ids = req.origin_input_ids + req.output_ids
|
||||
|
||||
# unified_kv SWA lives in a per-request ring the tree never stores, so a reused
|
||||
# prefix carries stale SWA; cap the match so the window is re-prefilled.
|
||||
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
|
||||
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
|
||||
if max_prefix_len is not None:
|
||||
key_limit = (
|
||||
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
|
||||
)
|
||||
|
||||
match_result = tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids=token_ids,
|
||||
extra_key=req.extra_key,
|
||||
limit=key_limit,
|
||||
cache_salt=req.cache_salt,
|
||||
),
|
||||
key=_req_radix_key(tree_cache, req, token_ids, max_prefix_len),
|
||||
cow_mamba=cow_mamba,
|
||||
req=req,
|
||||
)
|
||||
|
||||
@@ -406,6 +406,15 @@ class RadixCache(BasePrefixCache):
|
||||
best_match_node=last_node,
|
||||
)
|
||||
|
||||
def touch_prefix(self, key: RadixKey) -> None:
|
||||
key, _ = key.maybe_to_bigram_view(self.is_eagle)
|
||||
if self.disable or len(key) == 0:
|
||||
return
|
||||
key = key.page_aligned(self.page_size)
|
||||
if len(key) == 0:
|
||||
return
|
||||
self._match_prefix_helper(self.root_node, key)
|
||||
|
||||
def insert(self, params: InsertParams) -> InsertResult:
|
||||
if self.disable:
|
||||
return InsertResult(prefix_len=0)
|
||||
|
||||
@@ -599,6 +599,13 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def supports_fast_match_prefix(self) -> bool:
|
||||
return self.tree_core.supports_fast_match_prefix()
|
||||
|
||||
def touch_prefix(self, key: RadixKey) -> None:
|
||||
# Tree walk only; skips match_prefix's session shortcut, finalizers and linker.
|
||||
if self.disable:
|
||||
return
|
||||
result = self.tree_core.match_prefix(MatchPrefixParams(key=key))
|
||||
self._apply_cache_actions(result.cache_actions)
|
||||
|
||||
def supports_prefix_sharing(self) -> bool:
|
||||
return not self.disable
|
||||
|
||||
|
||||
Reference in New Issue
Block a user