[Scheduler] Fold match write-back into match_kv_cache (#42823)

This commit is contained in:
Liangsheng Yin
2026-10-07 15:43:05 -07:00
committed by GitHub
parent b1ffe5c16a
commit 4bf8f28d15
6 changed files with 104 additions and 162 deletions
+3 -8
View File
@@ -77,7 +77,6 @@ from sglang.srt.managers.schedule_batch import (
NextBatchPlan,
ScheduleBatch,
)
from sglang.srt.managers.schedule_policy import match_prefix_for_req
from sglang.srt.managers.utils import GenerationBatchResult
from sglang.srt.mem_cache.allocation import ensure_mamba_capacity
from sglang.srt.mem_cache.allocation_sizing import get_mamba_tracking_slots
@@ -93,6 +92,7 @@ from sglang.srt.mem_cache.common import (
discard_kv_cache_backup,
dsv41_dspark_needs_rebootstrap,
kv_to_page_indices,
match_kv_cache,
page_align_floor,
release_kv_cache,
restore_kv_cache,
@@ -782,22 +782,17 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
self.pending_reqs.append(decode_req)
def _match_prefix_and_lock(self, req: Req) -> DecodePrefixMatch:
"""
Match a request against the decode-side radix cache, lock the matched
node to prevent eviction, and return the matched prefix information.
"""
max_prefix_len = None
if self._uses_swa_tail_prealloc():
fill_len = self._pre_alloc_fill_len(req)
max_prefix_len = fill_len - self._swa_tail_len(fill_len)
# Match and lock only reusable FULL KV. The entire SWA tail must be
# freshly allocated, including when the prefix comes from L2/L3.
result = match_prefix_for_req(
self.tree_cache,
result = match_kv_cache(
req,
self.tree_cache,
req.origin_input_ids,
cow_mamba=self.tree_cache.supports_mamba(),
include_req=True,
max_prefix_len=max_prefix_len,
)
req.lock = self.tree_cache.lock(result.last_device_node)
@@ -10,11 +10,11 @@ from typing import TYPE_CHECKING, Any, List, Optional
import torch
from sglang.srt.disaggregation.base import KVPoll
from sglang.srt.managers.schedule_policy import match_prefix_for_req
from sglang.srt.mem_cache.base_prefix_cache import (
CacheRequestOutcome,
InitLoadBackParams,
)
from sglang.srt.mem_cache.common import match_kv_cache
if TYPE_CHECKING:
from sglang.srt.disaggregation.decode import DecodeRequest
@@ -64,7 +64,7 @@ class DecodeHiCachePreallocMixin:
def _build_decode_prefix_match(
self, req: Req, result: Any, *, max_prefix_len: Optional[int] = None
) -> DecodePrefixMatch:
"""Convert a ``match_prefix_for_req`` result into ``DecodePrefixMatch``.
"""Convert a ``match_kv_cache`` result into ``DecodePrefixMatch``.
Performs the optional L3 storage hit length query when decode-side
HiCache is enabled and the last host node is backed up.
@@ -212,12 +212,11 @@ class DecodeHiCacheTransferMixin:
self.tree_cache.pop_prefetch_loaded_tokens(dr.req.cache_request_handle)
# Re-match: req.last_node / prefix_indices updated to current device state.
rematch = match_prefix_for_req(
self.tree_cache,
rematch = match_kv_cache(
dr.req,
self.tree_cache,
dr.req.origin_input_ids,
cow_mamba=False,
include_req=True,
max_prefix_len=pm.decode_prefix_len,
)
new_indices, restored_node = self.tree_cache.init_load_back(
+9 -60
View File
@@ -107,18 +107,16 @@ from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import (
BasePrefixCache,
CacheRequestHandle,
MatchPrefixParams,
TreeLock,
zero_match_result,
)
from sglang.srt.mem_cache.common import (
RetractionBackup,
backup_kv_cache,
evict_from_tree_cache,
match_kv_cache,
release_kv_cache,
)
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
@@ -1624,16 +1622,6 @@ class Req(ReqDllmMixin):
token_ids_to_match = self.full_untruncated_fill_ids
key_limit: Optional[int] = self._compute_max_prefix_len(input_len)
# SWA lives in a per-request ring that's not content-stable and is never
# stored in the radix tree, so a reused prefix carries stale SWA. Cap the
# match by the trailing sliding window so it gets re-prefilled, rewriting
# this request's SWA ring. No-op for other layouts.
if tree_cache is not None:
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
if reprefill_tail:
capped = max(0, input_len - reprefill_tail)
key_limit = capped if key_limit is None else min(key_limit, capped)
# Disable prefix caching when embed overrides are present: same token IDs
# with different override vectors must not share cached KV values.
if self.positional_embed_overrides is not None:
@@ -1643,54 +1631,15 @@ class Req(ReqDllmMixin):
if tree_cache is not None:
if cow_mamba is None:
cow_mamba = tree_cache.supports_mamba()
# unified_kv SWA lives in a per-request ring that is not content-stable
# and never cached in the radix tree, so a reused prefix carries stale
# SWA. Cap the match by the trailing sliding window so it is re-prefilled
# into this request's ring. No-op for other layouts (returns 0).
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
if reprefill_tail:
capped = max(0, input_len - reprefill_tail)
key_limit = capped if key_limit is None else min(key_limit, capped)
match_result = tree_cache.match_prefix(
MatchPrefixParams(
key=RadixKey(
token_ids=token_ids_to_match,
extra_key=self.extra_key,
limit=key_limit,
cache_salt=self.cache_salt,
),
req=self,
cow_mamba=cow_mamba,
)
match_result = match_kv_cache(
self,
tree_cache,
token_ids_to_match,
cow_mamba=cow_mamba,
max_prefix_len=key_limit,
)
if envs.SGLANG_RADIX_FORCE_MISS.get():
match_result = zero_match_result(
tree_cache, match_result, extra_key=self.extra_key
)
(
self.prefix_indices,
self.last_node,
self.last_host_node,
self.best_match_node,
self.host_hit_length,
self.swa_host_hit_length,
self.swa_branching_seqlen,
self.mamba_host_hit_length,
self.mamba_branching_seqlen,
) = (
match_result.device_indices,
match_result.last_device_node,
match_result.last_host_node,
match_result.best_match_node,
match_result.host_hit_length,
match_result.swa_host_hit_length,
match_result.swa_branching_seqlen,
match_result.mamba_host_hit_length,
match_result.mamba_branching_seqlen,
)
if match_result.cache_protected_len is not None:
self.kv.cache_protected_len = match_result.cache_protected_len
else:
self.mamba_branching_seqlen = match_result.mamba_branching_seqlen
if match_result.cache_protected_len is None:
self.kv.cache_protected_len = len(self.prefix_indices)
if self.is_dllm():
+5 -81
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import logging
from array import array
from sglang.srt.environ import envs
from sglang.srt.managers.prefill_delayer import PrefillDelayerSinglePassExecutor
@@ -67,6 +66,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
MatchPrefixParams,
zero_match_result,
)
from sglang.srt.mem_cache.common import match_kv_cache
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
from sglang.srt.mem_cache.unified_cache.components import (
CacheTransferPhase,
@@ -159,74 +159,6 @@ def estimate_prefill_extend_tile_metrics(
}
def match_prefix_for_req(
tree_cache: BasePrefixCache,
req: Req,
token_ids: Optional[array[int]] = None,
*,
cow_mamba: bool = False,
include_req: bool = False,
max_prefix_len: Optional[int] = None,
):
if token_ids is None:
token_ids = req.origin_input_ids + req.output_ids
# unified_kv SWA lives in a per-request ring that's not content-stable and is
# never stored in the radix tree, so a reused prefix carries stale SWA. Cap
# the match by the trailing sliding window so it gets re-prefilled, rewriting
# this request's SWA ring. No-op for other layouts.
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
if max_prefix_len is not None:
key_limit = (
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
)
match_result = tree_cache.match_prefix(
MatchPrefixParams(
key=RadixKey(
token_ids=token_ids,
extra_key=req.extra_key,
limit=key_limit,
cache_salt=req.cache_salt,
),
cow_mamba=cow_mamba,
req=req if include_req else None,
)
)
if envs.SGLANG_RADIX_FORCE_MISS.get():
match_result = zero_match_result(
tree_cache, match_result, extra_key=req.extra_key
)
(
req.prefix_indices,
req.last_node,
req.last_host_node,
req.best_match_node,
req.host_hit_length,
req.swa_host_hit_length,
req.mamba_host_hit_length,
) = (
match_result.device_indices,
match_result.last_device_node,
match_result.last_host_node,
match_result.best_match_node,
match_result.host_hit_length,
match_result.swa_host_hit_length,
match_result.mamba_host_hit_length,
)
max_len = req._compute_max_prefix_len(len(token_ids))
req.num_matched_prefix_tokens = min(
len(req.prefix_indices) + req.host_hit_length, max_len
)
req.swa_branching_seqlen = match_result.swa_branching_seqlen
if match_result.mamba_branching_seqlen is not None:
req.mamba_branching_seqlen = match_result.mamba_branching_seqlen
if match_result.cache_protected_len is not None:
req.kv.cache_protected_len = match_result.cache_protected_len
return match_result
class CacheAwarePolicy(Enum):
"""Scheduling policies that are aware of the tree cache."""
@@ -275,17 +207,15 @@ class SchedulePolicy:
) -> None:
policy = self._determine_active_policy(waiting_queue)
# Populate req.num_matched_prefix_tokens at schedule time. Cache-aware policies
# set it in _compute_prefix_matches; do the same full match for
# cache-agnostic policies when the radix supports it, so the load
# snapshot has it. Skip on decode (never prefills).
# Cache-agnostic policies sort without matching; match anyway so the load
# snapshot sees num_matched_prefix_tokens. Decode never prefills.
if (
not isinstance(policy, CacheAwarePolicy)
and self.tree_cache.supports_fast_match_prefix()
and get_disagg().disaggregation_mode != "decode"
):
for r in waiting_queue:
match_prefix_for_req(self.tree_cache, r, include_req=True)
match_kv_cache(r, self.tree_cache)
if self.policy == CacheAgnosticPolicy.FCFS:
if self.enable_priority_scheduling:
@@ -383,10 +313,6 @@ class SchedulePolicy:
def _compute_prefix_matches(
self, waiting_queue: List[Req], policy: CacheAwarePolicy
) -> Set[int]:
"""
Computes and caches the matching prefixes for requests in the waiting queue,
and handles in-batch prefix caching logic.
"""
temporary_deprioritized: Set[int] = set()
self.waiting_queue_radix_tree.reset()
@@ -394,9 +320,7 @@ class SchedulePolicy:
prefix_ids = r.origin_input_ids + r.output_ids
extra_key = r.extra_key
cache_salt = r.cache_salt
match_result = match_prefix_for_req(
self.tree_cache, r, prefix_ids, include_req=True
)
match_result = match_kv_cache(r, self.tree_cache, prefix_ids)
# NOTE(sang): This logic is for in-batch prefix caching;
# If there are more than 1 request that have small matching prefix from
+77 -1
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import logging
from array import array
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, cast
import numpy as np
@@ -10,10 +11,18 @@ from sglang.kernels.ops.memory.common import (
_get_last_loc_safe_kernel as _get_last_loc_safe_kernel,
)
from sglang.kernels.ops.memory.common import get_last_loc_kernel as get_last_loc_kernel
from sglang.srt.environ import envs
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache, EvictParams
from sglang.srt.mem_cache.base_prefix_cache import (
BasePrefixCache,
EvictParams,
MatchPrefixParams,
MatchResult,
zero_match_result,
)
from sglang.srt.mem_cache.hicache_storage import PoolTransfer
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.runtime_context import get_serving, get_spec
from sglang.srt.utils.common import ceil_align
@@ -163,6 +172,73 @@ def free_kv_row_segments(
allocator.free_segments(swa_alive)
def match_kv_cache(
req: Req,
tree_cache: BasePrefixCache,
token_ids: Optional[array] = None,
*,
cow_mamba: bool = False,
max_prefix_len: Optional[int] = None,
) -> MatchResult:
"""Match token_ids against tree_cache and adopt the hit as req's prefix."""
if token_ids is None:
token_ids = req.origin_input_ids + req.output_ids
# unified_kv SWA lives in a per-request ring the tree never stores, so a reused
# prefix carries stale SWA; cap the match so the window is re-prefilled.
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
if max_prefix_len is not None:
key_limit = (
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
)
match_result = tree_cache.match_prefix(
MatchPrefixParams(
key=RadixKey(
token_ids=token_ids,
extra_key=req.extra_key,
limit=key_limit,
cache_salt=req.cache_salt,
),
cow_mamba=cow_mamba,
req=req,
)
)
if envs.SGLANG_RADIX_FORCE_MISS.get():
match_result = zero_match_result(
tree_cache, match_result, extra_key=req.extra_key
)
(
req.prefix_indices,
req.last_node,
req.last_host_node,
req.best_match_node,
req.host_hit_length,
req.swa_host_hit_length,
req.mamba_host_hit_length,
) = (
match_result.device_indices,
match_result.last_device_node,
match_result.last_host_node,
match_result.best_match_node,
match_result.host_hit_length,
match_result.swa_host_hit_length,
match_result.mamba_host_hit_length,
)
max_len = req._compute_max_prefix_len(len(token_ids))
req.num_matched_prefix_tokens = min(
len(req.prefix_indices) + req.host_hit_length, max_len
)
req.swa_branching_seqlen = match_result.swa_branching_seqlen
# A probe match keeps what it did not report; a new round resets both.
if match_result.mamba_branching_seqlen is not None:
req.mamba_branching_seqlen = match_result.mamba_branching_seqlen
if match_result.cache_protected_len is not None:
req.kv.cache_protected_len = match_result.cache_protected_len
return match_result
def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
"""Publish what the running request has computed so far, unless it is
barred from the tree."""