[Scheduler] Fold match write-back into match_kv_cache (#42823)
This commit is contained in:
@@ -77,7 +77,6 @@ from sglang.srt.managers.schedule_batch import (
|
||||
NextBatchPlan,
|
||||
ScheduleBatch,
|
||||
)
|
||||
from sglang.srt.managers.schedule_policy import match_prefix_for_req
|
||||
from sglang.srt.managers.utils import GenerationBatchResult
|
||||
from sglang.srt.mem_cache.allocation import ensure_mamba_capacity
|
||||
from sglang.srt.mem_cache.allocation_sizing import get_mamba_tracking_slots
|
||||
@@ -93,6 +92,7 @@ from sglang.srt.mem_cache.common import (
|
||||
discard_kv_cache_backup,
|
||||
dsv41_dspark_needs_rebootstrap,
|
||||
kv_to_page_indices,
|
||||
match_kv_cache,
|
||||
page_align_floor,
|
||||
release_kv_cache,
|
||||
restore_kv_cache,
|
||||
@@ -782,22 +782,17 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
self.pending_reqs.append(decode_req)
|
||||
|
||||
def _match_prefix_and_lock(self, req: Req) -> DecodePrefixMatch:
|
||||
"""
|
||||
Match a request against the decode-side radix cache, lock the matched
|
||||
node to prevent eviction, and return the matched prefix information.
|
||||
"""
|
||||
max_prefix_len = None
|
||||
if self._uses_swa_tail_prealloc():
|
||||
fill_len = self._pre_alloc_fill_len(req)
|
||||
max_prefix_len = fill_len - self._swa_tail_len(fill_len)
|
||||
# Match and lock only reusable FULL KV. The entire SWA tail must be
|
||||
# freshly allocated, including when the prefix comes from L2/L3.
|
||||
result = match_prefix_for_req(
|
||||
self.tree_cache,
|
||||
result = match_kv_cache(
|
||||
req,
|
||||
self.tree_cache,
|
||||
req.origin_input_ids,
|
||||
cow_mamba=self.tree_cache.supports_mamba(),
|
||||
include_req=True,
|
||||
max_prefix_len=max_prefix_len,
|
||||
)
|
||||
req.lock = self.tree_cache.lock(result.last_device_node)
|
||||
|
||||
@@ -10,11 +10,11 @@ from typing import TYPE_CHECKING, Any, List, Optional
|
||||
import torch
|
||||
|
||||
from sglang.srt.disaggregation.base import KVPoll
|
||||
from sglang.srt.managers.schedule_policy import match_prefix_for_req
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
CacheRequestOutcome,
|
||||
InitLoadBackParams,
|
||||
)
|
||||
from sglang.srt.mem_cache.common import match_kv_cache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.disaggregation.decode import DecodeRequest
|
||||
@@ -64,7 +64,7 @@ class DecodeHiCachePreallocMixin:
|
||||
def _build_decode_prefix_match(
|
||||
self, req: Req, result: Any, *, max_prefix_len: Optional[int] = None
|
||||
) -> DecodePrefixMatch:
|
||||
"""Convert a ``match_prefix_for_req`` result into ``DecodePrefixMatch``.
|
||||
"""Convert a ``match_kv_cache`` result into ``DecodePrefixMatch``.
|
||||
|
||||
Performs the optional L3 storage hit length query when decode-side
|
||||
HiCache is enabled and the last host node is backed up.
|
||||
@@ -212,12 +212,11 @@ class DecodeHiCacheTransferMixin:
|
||||
self.tree_cache.pop_prefetch_loaded_tokens(dr.req.cache_request_handle)
|
||||
|
||||
# Re-match: req.last_node / prefix_indices updated to current device state.
|
||||
rematch = match_prefix_for_req(
|
||||
self.tree_cache,
|
||||
rematch = match_kv_cache(
|
||||
dr.req,
|
||||
self.tree_cache,
|
||||
dr.req.origin_input_ids,
|
||||
cow_mamba=False,
|
||||
include_req=True,
|
||||
max_prefix_len=pm.decode_prefix_len,
|
||||
)
|
||||
new_indices, restored_node = self.tree_cache.init_load_back(
|
||||
|
||||
@@ -107,18 +107,16 @@ from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
BasePrefixCache,
|
||||
CacheRequestHandle,
|
||||
MatchPrefixParams,
|
||||
TreeLock,
|
||||
zero_match_result,
|
||||
)
|
||||
from sglang.srt.mem_cache.common import (
|
||||
RetractionBackup,
|
||||
backup_kv_cache,
|
||||
evict_from_tree_cache,
|
||||
match_kv_cache,
|
||||
release_kv_cache,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
@@ -1624,16 +1622,6 @@ class Req(ReqDllmMixin):
|
||||
token_ids_to_match = self.full_untruncated_fill_ids
|
||||
key_limit: Optional[int] = self._compute_max_prefix_len(input_len)
|
||||
|
||||
# SWA lives in a per-request ring that's not content-stable and is never
|
||||
# stored in the radix tree, so a reused prefix carries stale SWA. Cap the
|
||||
# match by the trailing sliding window so it gets re-prefilled, rewriting
|
||||
# this request's SWA ring. No-op for other layouts.
|
||||
if tree_cache is not None:
|
||||
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
|
||||
if reprefill_tail:
|
||||
capped = max(0, input_len - reprefill_tail)
|
||||
key_limit = capped if key_limit is None else min(key_limit, capped)
|
||||
|
||||
# Disable prefix caching when embed overrides are present: same token IDs
|
||||
# with different override vectors must not share cached KV values.
|
||||
if self.positional_embed_overrides is not None:
|
||||
@@ -1643,54 +1631,15 @@ class Req(ReqDllmMixin):
|
||||
if tree_cache is not None:
|
||||
if cow_mamba is None:
|
||||
cow_mamba = tree_cache.supports_mamba()
|
||||
# unified_kv SWA lives in a per-request ring that is not content-stable
|
||||
# and never cached in the radix tree, so a reused prefix carries stale
|
||||
# SWA. Cap the match by the trailing sliding window so it is re-prefilled
|
||||
# into this request's ring. No-op for other layouts (returns 0).
|
||||
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
|
||||
if reprefill_tail:
|
||||
capped = max(0, input_len - reprefill_tail)
|
||||
key_limit = capped if key_limit is None else min(key_limit, capped)
|
||||
match_result = tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids=token_ids_to_match,
|
||||
extra_key=self.extra_key,
|
||||
limit=key_limit,
|
||||
cache_salt=self.cache_salt,
|
||||
),
|
||||
req=self,
|
||||
cow_mamba=cow_mamba,
|
||||
)
|
||||
match_result = match_kv_cache(
|
||||
self,
|
||||
tree_cache,
|
||||
token_ids_to_match,
|
||||
cow_mamba=cow_mamba,
|
||||
max_prefix_len=key_limit,
|
||||
)
|
||||
if envs.SGLANG_RADIX_FORCE_MISS.get():
|
||||
match_result = zero_match_result(
|
||||
tree_cache, match_result, extra_key=self.extra_key
|
||||
)
|
||||
(
|
||||
self.prefix_indices,
|
||||
self.last_node,
|
||||
self.last_host_node,
|
||||
self.best_match_node,
|
||||
self.host_hit_length,
|
||||
self.swa_host_hit_length,
|
||||
self.swa_branching_seqlen,
|
||||
self.mamba_host_hit_length,
|
||||
self.mamba_branching_seqlen,
|
||||
) = (
|
||||
match_result.device_indices,
|
||||
match_result.last_device_node,
|
||||
match_result.last_host_node,
|
||||
match_result.best_match_node,
|
||||
match_result.host_hit_length,
|
||||
match_result.swa_host_hit_length,
|
||||
match_result.swa_branching_seqlen,
|
||||
match_result.mamba_host_hit_length,
|
||||
match_result.mamba_branching_seqlen,
|
||||
)
|
||||
if match_result.cache_protected_len is not None:
|
||||
self.kv.cache_protected_len = match_result.cache_protected_len
|
||||
else:
|
||||
self.mamba_branching_seqlen = match_result.mamba_branching_seqlen
|
||||
if match_result.cache_protected_len is None:
|
||||
self.kv.cache_protected_len = len(self.prefix_indices)
|
||||
|
||||
if self.is_dllm():
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.prefill_delayer import PrefillDelayerSinglePassExecutor
|
||||
@@ -67,6 +66,7 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
MatchPrefixParams,
|
||||
zero_match_result,
|
||||
)
|
||||
from sglang.srt.mem_cache.common import match_kv_cache
|
||||
from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode
|
||||
from sglang.srt.mem_cache.unified_cache.components import (
|
||||
CacheTransferPhase,
|
||||
@@ -159,74 +159,6 @@ def estimate_prefill_extend_tile_metrics(
|
||||
}
|
||||
|
||||
|
||||
def match_prefix_for_req(
|
||||
tree_cache: BasePrefixCache,
|
||||
req: Req,
|
||||
token_ids: Optional[array[int]] = None,
|
||||
*,
|
||||
cow_mamba: bool = False,
|
||||
include_req: bool = False,
|
||||
max_prefix_len: Optional[int] = None,
|
||||
):
|
||||
if token_ids is None:
|
||||
token_ids = req.origin_input_ids + req.output_ids
|
||||
|
||||
# unified_kv SWA lives in a per-request ring that's not content-stable and is
|
||||
# never stored in the radix tree, so a reused prefix carries stale SWA. Cap
|
||||
# the match by the trailing sliding window so it gets re-prefilled, rewriting
|
||||
# this request's SWA ring. No-op for other layouts.
|
||||
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
|
||||
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
|
||||
if max_prefix_len is not None:
|
||||
key_limit = (
|
||||
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
|
||||
)
|
||||
|
||||
match_result = tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids=token_ids,
|
||||
extra_key=req.extra_key,
|
||||
limit=key_limit,
|
||||
cache_salt=req.cache_salt,
|
||||
),
|
||||
cow_mamba=cow_mamba,
|
||||
req=req if include_req else None,
|
||||
)
|
||||
)
|
||||
if envs.SGLANG_RADIX_FORCE_MISS.get():
|
||||
match_result = zero_match_result(
|
||||
tree_cache, match_result, extra_key=req.extra_key
|
||||
)
|
||||
(
|
||||
req.prefix_indices,
|
||||
req.last_node,
|
||||
req.last_host_node,
|
||||
req.best_match_node,
|
||||
req.host_hit_length,
|
||||
req.swa_host_hit_length,
|
||||
req.mamba_host_hit_length,
|
||||
) = (
|
||||
match_result.device_indices,
|
||||
match_result.last_device_node,
|
||||
match_result.last_host_node,
|
||||
match_result.best_match_node,
|
||||
match_result.host_hit_length,
|
||||
match_result.swa_host_hit_length,
|
||||
match_result.mamba_host_hit_length,
|
||||
)
|
||||
max_len = req._compute_max_prefix_len(len(token_ids))
|
||||
req.num_matched_prefix_tokens = min(
|
||||
len(req.prefix_indices) + req.host_hit_length, max_len
|
||||
)
|
||||
req.swa_branching_seqlen = match_result.swa_branching_seqlen
|
||||
if match_result.mamba_branching_seqlen is not None:
|
||||
req.mamba_branching_seqlen = match_result.mamba_branching_seqlen
|
||||
if match_result.cache_protected_len is not None:
|
||||
req.kv.cache_protected_len = match_result.cache_protected_len
|
||||
return match_result
|
||||
|
||||
|
||||
class CacheAwarePolicy(Enum):
|
||||
"""Scheduling policies that are aware of the tree cache."""
|
||||
|
||||
@@ -275,17 +207,15 @@ class SchedulePolicy:
|
||||
) -> None:
|
||||
policy = self._determine_active_policy(waiting_queue)
|
||||
|
||||
# Populate req.num_matched_prefix_tokens at schedule time. Cache-aware policies
|
||||
# set it in _compute_prefix_matches; do the same full match for
|
||||
# cache-agnostic policies when the radix supports it, so the load
|
||||
# snapshot has it. Skip on decode (never prefills).
|
||||
# Cache-agnostic policies sort without matching; match anyway so the load
|
||||
# snapshot sees num_matched_prefix_tokens. Decode never prefills.
|
||||
if (
|
||||
not isinstance(policy, CacheAwarePolicy)
|
||||
and self.tree_cache.supports_fast_match_prefix()
|
||||
and get_disagg().disaggregation_mode != "decode"
|
||||
):
|
||||
for r in waiting_queue:
|
||||
match_prefix_for_req(self.tree_cache, r, include_req=True)
|
||||
match_kv_cache(r, self.tree_cache)
|
||||
|
||||
if self.policy == CacheAgnosticPolicy.FCFS:
|
||||
if self.enable_priority_scheduling:
|
||||
@@ -383,10 +313,6 @@ class SchedulePolicy:
|
||||
def _compute_prefix_matches(
|
||||
self, waiting_queue: List[Req], policy: CacheAwarePolicy
|
||||
) -> Set[int]:
|
||||
"""
|
||||
Computes and caches the matching prefixes for requests in the waiting queue,
|
||||
and handles in-batch prefix caching logic.
|
||||
"""
|
||||
temporary_deprioritized: Set[int] = set()
|
||||
self.waiting_queue_radix_tree.reset()
|
||||
|
||||
@@ -394,9 +320,7 @@ class SchedulePolicy:
|
||||
prefix_ids = r.origin_input_ids + r.output_ids
|
||||
extra_key = r.extra_key
|
||||
cache_salt = r.cache_salt
|
||||
match_result = match_prefix_for_req(
|
||||
self.tree_cache, r, prefix_ids, include_req=True
|
||||
)
|
||||
match_result = match_kv_cache(r, self.tree_cache, prefix_ids)
|
||||
|
||||
# NOTE(sang): This logic is for in-batch prefix caching;
|
||||
# If there are more than 1 request that have small matching prefix from
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from array import array
|
||||
from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, cast
|
||||
|
||||
import numpy as np
|
||||
@@ -10,10 +11,18 @@ from sglang.kernels.ops.memory.common import (
|
||||
_get_last_loc_safe_kernel as _get_last_loc_safe_kernel,
|
||||
)
|
||||
from sglang.kernels.ops.memory.common import get_last_loc_kernel as get_last_loc_kernel
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache, EvictParams
|
||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
BasePrefixCache,
|
||||
EvictParams,
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
zero_match_result,
|
||||
)
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolTransfer
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.runtime_context import get_serving, get_spec
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
@@ -163,6 +172,73 @@ def free_kv_row_segments(
|
||||
allocator.free_segments(swa_alive)
|
||||
|
||||
|
||||
def match_kv_cache(
|
||||
req: Req,
|
||||
tree_cache: BasePrefixCache,
|
||||
token_ids: Optional[array] = None,
|
||||
*,
|
||||
cow_mamba: bool = False,
|
||||
max_prefix_len: Optional[int] = None,
|
||||
) -> MatchResult:
|
||||
"""Match token_ids against tree_cache and adopt the hit as req's prefix."""
|
||||
if token_ids is None:
|
||||
token_ids = req.origin_input_ids + req.output_ids
|
||||
|
||||
# unified_kv SWA lives in a per-request ring the tree never stores, so a reused
|
||||
# prefix carries stale SWA; cap the match so the window is re-prefilled.
|
||||
reprefill_tail = tree_cache.swa_reprefill_tail_tokens()
|
||||
key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None
|
||||
if max_prefix_len is not None:
|
||||
key_limit = (
|
||||
max_prefix_len if key_limit is None else min(key_limit, max_prefix_len)
|
||||
)
|
||||
|
||||
match_result = tree_cache.match_prefix(
|
||||
MatchPrefixParams(
|
||||
key=RadixKey(
|
||||
token_ids=token_ids,
|
||||
extra_key=req.extra_key,
|
||||
limit=key_limit,
|
||||
cache_salt=req.cache_salt,
|
||||
),
|
||||
cow_mamba=cow_mamba,
|
||||
req=req,
|
||||
)
|
||||
)
|
||||
if envs.SGLANG_RADIX_FORCE_MISS.get():
|
||||
match_result = zero_match_result(
|
||||
tree_cache, match_result, extra_key=req.extra_key
|
||||
)
|
||||
(
|
||||
req.prefix_indices,
|
||||
req.last_node,
|
||||
req.last_host_node,
|
||||
req.best_match_node,
|
||||
req.host_hit_length,
|
||||
req.swa_host_hit_length,
|
||||
req.mamba_host_hit_length,
|
||||
) = (
|
||||
match_result.device_indices,
|
||||
match_result.last_device_node,
|
||||
match_result.last_host_node,
|
||||
match_result.best_match_node,
|
||||
match_result.host_hit_length,
|
||||
match_result.swa_host_hit_length,
|
||||
match_result.mamba_host_hit_length,
|
||||
)
|
||||
max_len = req._compute_max_prefix_len(len(token_ids))
|
||||
req.num_matched_prefix_tokens = min(
|
||||
len(req.prefix_indices) + req.host_hit_length, max_len
|
||||
)
|
||||
req.swa_branching_seqlen = match_result.swa_branching_seqlen
|
||||
# A probe match keeps what it did not report; a new round resets both.
|
||||
if match_result.mamba_branching_seqlen is not None:
|
||||
req.mamba_branching_seqlen = match_result.mamba_branching_seqlen
|
||||
if match_result.cache_protected_len is not None:
|
||||
req.kv.cache_protected_len = match_result.cache_protected_len
|
||||
return match_result
|
||||
|
||||
|
||||
def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
"""Publish what the running request has computed so far, unless it is
|
||||
barred from the tree."""
|
||||
|
||||
Reference in New Issue
Block a user