[Scheduler] Refresh waiting requests' cached prefixes under FCFS-like policies with LRU eviction (#41593)

Signed-off-by: Pei Li <peili@nvidia.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
Pei Li
2026-10-08 17:05:04 -07:00
committed by GitHub
co-authored by Po-Han Huang Po-Han Huang Claude Opus 5.5 hnyls2002
parent 2b94cb4eb5
commit 20d1f3f56a
8 changed files with 191 additions and 20 deletions
+1
View File
@@ -602,6 +602,7 @@ class Envs:
# ===================================================================
# Scheduler token budgeting and admission
# ===================================================================
SGLANG_ENABLE_WAITING_PREFIX_REFRESH = EnvBool(True)
SGLANG_INIT_NEW_TOKEN_RATIO = EnvFloat(0.7)
SGLANG_MIN_NEW_TOKEN_RATIO_FACTOR = EnvFloat(0.14)
SGLANG_NEW_TOKEN_RATIO_DECAY_STEPS = EnvInt(600)
+33 -5
View File
@@ -7,6 +7,7 @@ from sglang.srt.managers.prefill_delayer import PrefillDelayerSinglePassExecutor
from sglang.srt.runtime_context import (
get_disagg,
get_exec,
get_memory,
get_schedule,
)
from sglang.srt.utils import get_bool_env_var, is_gfx95_supported, is_hip
@@ -31,6 +32,7 @@ logger = logging.getLogger(__name__)
import os
import random
import time
from collections import Counter
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
@@ -66,7 +68,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
MatchPrefixParams,
zero_match_result,
)
from sglang.srt.mem_cache.common import match_kv_cache
from sglang.srt.mem_cache.common import match_kv_cache, touch_waiting_prefix
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
from sglang.srt.mem_cache.unified_cache.components import (
CacheTransferPhase,
@@ -128,6 +130,11 @@ if PREFILL_TILE_BUDGET_MODE not in {"legacy", "compact"}:
)
PREFILL_TILE_BUDGET_MODE = "compact"
# Arbitrary; bounds per-round prefix work on the waiting queue.
WAITING_QUEUE_PREFIX_MATCH_MAX = 128
# Arbitrary; bounds the walk cost of touching waiting prefixes.
WAITING_PREFIX_REFRESH_INTERVAL_S = 0.5
def _ceil_div(value: int, divisor: int) -> int:
return -(-value // divisor)
@@ -195,6 +202,7 @@ class SchedulePolicy:
self.schedule_low_priority_values_first = schedule_low_priority_values_first
self.priority_sign = 1 if schedule_low_priority_values_first else -1
self._shortest_prefill_calls = 0
self._last_waiting_prefix_refresh = float("-inf")
# It is used to find the matching prefix for in-batch prefix caching.
self.waiting_queue_radix_tree = RadixCache.create_simulated()
@@ -222,9 +230,7 @@ class SchedulePolicy:
SchedulePolicy._sort_by_priority_and_fcfs(
waiting_queue, self.priority_sign
)
return
if isinstance(policy, CacheAwarePolicy):
elif isinstance(policy, CacheAwarePolicy):
temporary_deprioritized = self._compute_prefix_matches(
waiting_queue, policy
)
@@ -268,6 +274,28 @@ class SchedulePolicy:
else:
raise ValueError(f"Unknown CacheAgnostic Policy: {policy=}")
self._touch_waiting_prefixes(policy, waiting_queue)
def _touch_waiting_prefixes(self, policy: Policy, waiting_queue: List[Req]) -> None:
# Head last, so LRU evicts in reverse admission order; cache-aware policies
# already touch while matching, and other eviction strategies ignore recency.
if (
isinstance(policy, CacheAwarePolicy)
or not waiting_queue
or not envs.SGLANG_ENABLE_WAITING_PREFIX_REFRESH.get()
or get_disagg().disaggregation_mode == "decode"
or get_memory().radix_eviction_policy.lower() != "lru"
or self.tree_cache.supports_fast_match_prefix()
or not self.tree_cache.supports_prefix_sharing()
):
return
now = time.monotonic()
if now - self._last_waiting_prefix_refresh < WAITING_PREFIX_REFRESH_INTERVAL_S:
return
self._last_waiting_prefix_refresh = now
for r in reversed(waiting_queue[:WAITING_QUEUE_PREFIX_MATCH_MAX]):
touch_waiting_prefix(r, self.tree_cache)
def _determine_active_policy(self, waiting_queue: List[Req]) -> Policy:
if (
self.policy
@@ -275,7 +303,7 @@ class SchedulePolicy:
CacheAwarePolicy.LPM,
CacheAwarePolicy.HRRN,
)
and len(waiting_queue) > 128
and len(waiting_queue) > WAITING_QUEUE_PREFIX_MATCH_MAX
):
# Turn off the expensive prefix matching and sorting when the #queue is large.
return CacheAgnosticPolicy.FCFS
@@ -440,6 +440,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
def supports_fast_match_prefix(self) -> bool:
return False
def touch_prefix(self, key: RadixKey) -> None:
"""Bump LRU recency along ``key``'s cached path. Unlike ``match_prefix``,
never allocates, loads, or queries external tiers."""
def dfs_weight_order(self, node_handles: Sequence[Any]) -> list[int]:
"""Return request indices in depth-first, subtree-weight order."""
return _dfs_weight_order(self.root_node, node_handles, self.resolve_node_handle)
+29 -15
View File
@@ -172,6 +172,34 @@ def free_kv_row_segments(
allocator.free_segments(swa_alive)
def _req_radix_key(
tree_cache: BasePrefixCache,
req: Req,
token_ids: array[int],
max_prefix_len: Optional[int] = None,
) -> RadixKey:
# unified_kv SWA lives in a per-request ring the tree never stores, so a reused
# prefix carries stale SWA; cap the match so the window is re-prefilled.
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
if max_prefix_len is not None:
key_limit = (
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
)
return RadixKey(
token_ids=token_ids,
extra_key=req.extra_key,
limit=key_limit,
cache_salt=req.cache_salt,
)
def touch_waiting_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
tree_cache.touch_prefix(
_req_radix_key(tree_cache, req, req.origin_input_ids + req.output_ids)
)
def match_kv_cache(
req: Req,
tree_cache: BasePrefixCache,
@@ -184,23 +212,9 @@ def match_kv_cache(
if token_ids is None:
token_ids = req.origin_input_ids + req.output_ids
# unified_kv SWA lives in a per-request ring the tree never stores, so a reused
# prefix carries stale SWA; cap the match so the window is re-prefilled.
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
if max_prefix_len is not None:
key_limit = (
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
)
match_result = tree_cache.match_prefix(
MatchPrefixParams(
key=RadixKey(
token_ids=token_ids,
extra_key=req.extra_key,
limit=key_limit,
cache_salt=req.cache_salt,
),
key=_req_radix_key(tree_cache, req, token_ids, max_prefix_len),
cow_mamba=cow_mamba,
req=req,
)
@@ -406,6 +406,15 @@ class RadixCache(BasePrefixCache):
best_match_node=last_node,
)
def touch_prefix(self, key: RadixKey) -> None:
key, _ = key.maybe_to_bigram_view(self.is_eagle)
if self.disable or len(key) == 0:
return
key = key.page_aligned(self.page_size)
if len(key) == 0:
return
self._match_prefix_helper(self.root_node, key)
def insert(self, params: InsertParams) -> InsertResult:
if self.disable:
return InsertResult(prefix_len=0)
@@ -599,6 +599,13 @@ class UnifiedRadixCache(BasePrefixCache):
def supports_fast_match_prefix(self) -> bool:
return self.tree_core.supports_fast_match_prefix()
def touch_prefix(self, key: RadixKey) -> None:
# Tree walk only; skips match_prefix's session shortcut, finalizers and linker.
if self.disable:
return
result = self.tree_core.match_prefix(MatchPrefixParams(key=key))
self._apply_cache_actions(result.cache_actions)
def supports_prefix_sharing(self) -> bool:
return not self.disable