From 4bf8f28d15b01843a290c33fa918c001f8f32afd Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Wed, 7 Oct 2026 15:43:05 -0700 Subject: [PATCH] [Scheduler] Fold match write-back into `match_kv_cache` (#42823) --- python/sglang/srt/disaggregation/decode.py | 11 +-- .../disaggregation/decode_hicache_mixin.py | 9 +- python/sglang/srt/managers/schedule_batch.py | 69 ++------------- python/sglang/srt/managers/schedule_policy.py | 86 ++----------------- python/sglang/srt/mem_cache/common.py | 78 ++++++++++++++++- .../unit/mem_cache/test_radix_force_miss.py | 13 ++- 6 files changed, 104 insertions(+), 162 deletions(-) diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index c4cf6ec4f5..f1b2ac80b4 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -77,7 +77,6 @@ from sglang.srt.managers.schedule_batch import ( NextBatchPlan, ScheduleBatch, ) -from sglang.srt.managers.schedule_policy import match_prefix_for_req from sglang.srt.managers.utils import GenerationBatchResult from sglang.srt.mem_cache.allocation import ensure_mamba_capacity from sglang.srt.mem_cache.allocation_sizing import get_mamba_tracking_slots @@ -93,6 +92,7 @@ from sglang.srt.mem_cache.common import ( discard_kv_cache_backup, dsv41_dspark_needs_rebootstrap, kv_to_page_indices, + match_kv_cache, page_align_floor, release_kv_cache, restore_kv_cache, @@ -782,22 +782,17 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): self.pending_reqs.append(decode_req) def _match_prefix_and_lock(self, req: Req) -> DecodePrefixMatch: - """ - Match a request against the decode-side radix cache, lock the matched - node to prevent eviction, and return the matched prefix information. - """ max_prefix_len = None if self._uses_swa_tail_prealloc(): fill_len = self._pre_alloc_fill_len(req) max_prefix_len = fill_len - self._swa_tail_len(fill_len) # Match and lock only reusable FULL KV. The entire SWA tail must be # freshly allocated, including when the prefix comes from L2/L3. - result = match_prefix_for_req( - self.tree_cache, + result = match_kv_cache( req, + self.tree_cache, req.origin_input_ids, cow_mamba=self.tree_cache.supports_mamba(), - include_req=True, max_prefix_len=max_prefix_len, ) req.lock = self.tree_cache.lock(result.last_device_node) diff --git a/python/sglang/srt/disaggregation/decode_hicache_mixin.py b/python/sglang/srt/disaggregation/decode_hicache_mixin.py index 9e1bcbdd4c..36084e4937 100644 --- a/python/sglang/srt/disaggregation/decode_hicache_mixin.py +++ b/python/sglang/srt/disaggregation/decode_hicache_mixin.py @@ -10,11 +10,11 @@ from typing import TYPE_CHECKING, Any, List, Optional import torch from sglang.srt.disaggregation.base import KVPoll -from sglang.srt.managers.schedule_policy import match_prefix_for_req from sglang.srt.mem_cache.base_prefix_cache import ( CacheRequestOutcome, InitLoadBackParams, ) +from sglang.srt.mem_cache.common import match_kv_cache if TYPE_CHECKING: from sglang.srt.disaggregation.decode import DecodeRequest @@ -64,7 +64,7 @@ class DecodeHiCachePreallocMixin: def _build_decode_prefix_match( self, req: Req, result: Any, *, max_prefix_len: Optional[int] = None ) -> DecodePrefixMatch: - """Convert a ``match_prefix_for_req`` result into ``DecodePrefixMatch``. + """Convert a ``match_kv_cache`` result into ``DecodePrefixMatch``. Performs the optional L3 storage hit length query when decode-side HiCache is enabled and the last host node is backed up. @@ -212,12 +212,11 @@ class DecodeHiCacheTransferMixin: self.tree_cache.pop_prefetch_loaded_tokens(dr.req.cache_request_handle) # Re-match: req.last_node / prefix_indices updated to current device state. - rematch = match_prefix_for_req( - self.tree_cache, + rematch = match_kv_cache( dr.req, + self.tree_cache, dr.req.origin_input_ids, cow_mamba=False, - include_req=True, max_prefix_len=pm.decode_prefix_len, ) new_indices, restored_node = self.tree_cache.init_load_back( diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index ebccdb1629..a5ab74ae39 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -107,18 +107,16 @@ from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import ( BasePrefixCache, CacheRequestHandle, - MatchPrefixParams, TreeLock, - zero_match_result, ) from sglang.srt.mem_cache.common import ( RetractionBackup, backup_kv_cache, evict_from_tree_cache, + match_kv_cache, release_kv_cache, ) from sglang.srt.mem_cache.memory_pool import HybridReqToTokenPool, ReqToTokenPool -from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, @@ -1624,16 +1622,6 @@ class Req(ReqDllmMixin): token_ids_to_match = self.full_untruncated_fill_ids key_limit: Optional[int] = self._compute_max_prefix_len(input_len) - # SWA lives in a per-request ring that's not content-stable and is never - # stored in the radix tree, so a reused prefix carries stale SWA. Cap the - # match by the trailing sliding window so it gets re-prefilled, rewriting - # this request's SWA ring. No-op for other layouts. - if tree_cache is not None: - reprefill_tail = tree_cache.swa_reprefill_tail_tokens() - if reprefill_tail: - capped = max(0, input_len - reprefill_tail) - key_limit = capped if key_limit is None else min(key_limit, capped) - # Disable prefix caching when embed overrides are present: same token IDs # with different override vectors must not share cached KV values. if self.positional_embed_overrides is not None: @@ -1643,54 +1631,15 @@ class Req(ReqDllmMixin): if tree_cache is not None: if cow_mamba is None: cow_mamba = tree_cache.supports_mamba() - # unified_kv SWA lives in a per-request ring that is not content-stable - # and never cached in the radix tree, so a reused prefix carries stale - # SWA. Cap the match by the trailing sliding window so it is re-prefilled - # into this request's ring. No-op for other layouts (returns 0). - reprefill_tail = tree_cache.swa_reprefill_tail_tokens() - if reprefill_tail: - capped = max(0, input_len - reprefill_tail) - key_limit = capped if key_limit is None else min(key_limit, capped) - match_result = tree_cache.match_prefix( - MatchPrefixParams( - key=RadixKey( - token_ids=token_ids_to_match, - extra_key=self.extra_key, - limit=key_limit, - cache_salt=self.cache_salt, - ), - req=self, - cow_mamba=cow_mamba, - ) + match_result = match_kv_cache( + self, + tree_cache, + token_ids_to_match, + cow_mamba=cow_mamba, + max_prefix_len=key_limit, ) - if envs.SGLANG_RADIX_FORCE_MISS.get(): - match_result = zero_match_result( - tree_cache, match_result, extra_key=self.extra_key - ) - ( - self.prefix_indices, - self.last_node, - self.last_host_node, - self.best_match_node, - self.host_hit_length, - self.swa_host_hit_length, - self.swa_branching_seqlen, - self.mamba_host_hit_length, - self.mamba_branching_seqlen, - ) = ( - match_result.device_indices, - match_result.last_device_node, - match_result.last_host_node, - match_result.best_match_node, - match_result.host_hit_length, - match_result.swa_host_hit_length, - match_result.swa_branching_seqlen, - match_result.mamba_host_hit_length, - match_result.mamba_branching_seqlen, - ) - if match_result.cache_protected_len is not None: - self.kv.cache_protected_len = match_result.cache_protected_len - else: + self.mamba_branching_seqlen = match_result.mamba_branching_seqlen + if match_result.cache_protected_len is None: self.kv.cache_protected_len = len(self.prefix_indices) if self.is_dllm(): diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index ada0d210b7..62305baf74 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -1,7 +1,6 @@ from __future__ import annotations import logging -from array import array from sglang.srt.environ import envs from sglang.srt.managers.prefill_delayer import PrefillDelayerSinglePassExecutor @@ -67,6 +66,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( MatchPrefixParams, zero_match_result, ) +from sglang.srt.mem_cache.common import match_kv_cache from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey, TreeNode from sglang.srt.mem_cache.unified_cache.components import ( CacheTransferPhase, @@ -159,74 +159,6 @@ def estimate_prefill_extend_tile_metrics( } -def match_prefix_for_req( - tree_cache: BasePrefixCache, - req: Req, - token_ids: Optional[array[int]] = None, - *, - cow_mamba: bool = False, - include_req: bool = False, - max_prefix_len: Optional[int] = None, -): - if token_ids is None: - token_ids = req.origin_input_ids + req.output_ids - - # unified_kv SWA lives in a per-request ring that's not content-stable and is - # never stored in the radix tree, so a reused prefix carries stale SWA. Cap - # the match by the trailing sliding window so it gets re-prefilled, rewriting - # this request's SWA ring. No-op for other layouts. - reprefill_tail = tree_cache.swa_reprefill_tail_tokens() - key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None - if max_prefix_len is not None: - key_limit = ( - max_prefix_len if key_limit is None else min(key_limit, max_prefix_len) - ) - - match_result = tree_cache.match_prefix( - MatchPrefixParams( - key=RadixKey( - token_ids=token_ids, - extra_key=req.extra_key, - limit=key_limit, - cache_salt=req.cache_salt, - ), - cow_mamba=cow_mamba, - req=req if include_req else None, - ) - ) - if envs.SGLANG_RADIX_FORCE_MISS.get(): - match_result = zero_match_result( - tree_cache, match_result, extra_key=req.extra_key - ) - ( - req.prefix_indices, - req.last_node, - req.last_host_node, - req.best_match_node, - req.host_hit_length, - req.swa_host_hit_length, - req.mamba_host_hit_length, - ) = ( - match_result.device_indices, - match_result.last_device_node, - match_result.last_host_node, - match_result.best_match_node, - match_result.host_hit_length, - match_result.swa_host_hit_length, - match_result.mamba_host_hit_length, - ) - max_len = req._compute_max_prefix_len(len(token_ids)) - req.num_matched_prefix_tokens = min( - len(req.prefix_indices) + req.host_hit_length, max_len - ) - req.swa_branching_seqlen = match_result.swa_branching_seqlen - if match_result.mamba_branching_seqlen is not None: - req.mamba_branching_seqlen = match_result.mamba_branching_seqlen - if match_result.cache_protected_len is not None: - req.kv.cache_protected_len = match_result.cache_protected_len - return match_result - - class CacheAwarePolicy(Enum): """Scheduling policies that are aware of the tree cache.""" @@ -275,17 +207,15 @@ class SchedulePolicy: ) -> None: policy = self._determine_active_policy(waiting_queue) - # Populate req.num_matched_prefix_tokens at schedule time. Cache-aware policies - # set it in _compute_prefix_matches; do the same full match for - # cache-agnostic policies when the radix supports it, so the load - # snapshot has it. Skip on decode (never prefills). + # Cache-agnostic policies sort without matching; match anyway so the load + # snapshot sees num_matched_prefix_tokens. Decode never prefills. if ( not isinstance(policy, CacheAwarePolicy) and self.tree_cache.supports_fast_match_prefix() and get_disagg().disaggregation_mode != "decode" ): for r in waiting_queue: - match_prefix_for_req(self.tree_cache, r, include_req=True) + match_kv_cache(r, self.tree_cache) if self.policy == CacheAgnosticPolicy.FCFS: if self.enable_priority_scheduling: @@ -383,10 +313,6 @@ class SchedulePolicy: def _compute_prefix_matches( self, waiting_queue: List[Req], policy: CacheAwarePolicy ) -> Set[int]: - """ - Computes and caches the matching prefixes for requests in the waiting queue, - and handles in-batch prefix caching logic. - """ temporary_deprioritized: Set[int] = set() self.waiting_queue_radix_tree.reset() @@ -394,9 +320,7 @@ class SchedulePolicy: prefix_ids = r.origin_input_ids + r.output_ids extra_key = r.extra_key cache_salt = r.cache_salt - match_result = match_prefix_for_req( - self.tree_cache, r, prefix_ids, include_req=True - ) + match_result = match_kv_cache(r, self.tree_cache, prefix_ids) # NOTE(sang): This logic is for in-batch prefix caching; # If there are more than 1 request that have small matching prefix from diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 55cdcd4028..d8dd3c0a9a 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from array import array from typing import TYPE_CHECKING, Any, Callable, NamedTuple, Optional, cast import numpy as np @@ -10,10 +11,18 @@ from sglang.kernels.ops.memory.common import ( _get_last_loc_safe_kernel as _get_last_loc_safe_kernel, ) from sglang.kernels.ops.memory.common import get_last_loc_kernel as get_last_loc_kernel +from sglang.srt.environ import envs from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size -from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache, EvictParams +from sglang.srt.mem_cache.base_prefix_cache import ( + BasePrefixCache, + EvictParams, + MatchPrefixParams, + MatchResult, + zero_match_result, +) from sglang.srt.mem_cache.hicache_storage import PoolTransfer from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.runtime_context import get_serving, get_spec from sglang.srt.utils.common import ceil_align @@ -163,6 +172,73 @@ def free_kv_row_segments( allocator.free_segments(swa_alive) +def match_kv_cache( + req: Req, + tree_cache: BasePrefixCache, + token_ids: Optional[array] = None, + *, + cow_mamba: bool = False, + max_prefix_len: Optional[int] = None, +) -> MatchResult: + """Match token_ids against tree_cache and adopt the hit as req's prefix.""" + if token_ids is None: + token_ids = req.origin_input_ids + req.output_ids + + # unified_kv SWA lives in a per-request ring the tree never stores, so a reused + # prefix carries stale SWA; cap the match so the window is re-prefilled. + reprefill_tail = tree_cache.swa_reprefill_tail_tokens() + key_limit = max(0, len(token_ids) - reprefill_tail) if reprefill_tail else None + if max_prefix_len is not None: + key_limit = ( + max_prefix_len if key_limit is None else min(key_limit, max_prefix_len) + ) + + match_result = tree_cache.match_prefix( + MatchPrefixParams( + key=RadixKey( + token_ids=token_ids, + extra_key=req.extra_key, + limit=key_limit, + cache_salt=req.cache_salt, + ), + cow_mamba=cow_mamba, + req=req, + ) + ) + if envs.SGLANG_RADIX_FORCE_MISS.get(): + match_result = zero_match_result( + tree_cache, match_result, extra_key=req.extra_key + ) + ( + req.prefix_indices, + req.last_node, + req.last_host_node, + req.best_match_node, + req.host_hit_length, + req.swa_host_hit_length, + req.mamba_host_hit_length, + ) = ( + match_result.device_indices, + match_result.last_device_node, + match_result.last_host_node, + match_result.best_match_node, + match_result.host_hit_length, + match_result.swa_host_hit_length, + match_result.mamba_host_hit_length, + ) + max_len = req._compute_max_prefix_len(len(token_ids)) + req.num_matched_prefix_tokens = min( + len(req.prefix_indices) + req.host_hit_length, max_len + ) + req.swa_branching_seqlen = match_result.swa_branching_seqlen + # A probe match keeps what it did not report; a new round resets both. + if match_result.mamba_branching_seqlen is not None: + req.mamba_branching_seqlen = match_result.mamba_branching_seqlen + if match_result.cache_protected_len is not None: + req.kv.cache_protected_len = match_result.cache_protected_len + return match_result + + def checkpoint_kv_cache(req: Req, tree_cache: BasePrefixCache) -> None: """Publish what the running request has computed so far, unless it is barred from the tree.""" diff --git a/test/registered/unit/mem_cache/test_radix_force_miss.py b/test/registered/unit/mem_cache/test_radix_force_miss.py index 8209214179..071a391587 100644 --- a/test/registered/unit/mem_cache/test_radix_force_miss.py +++ b/test/registered/unit/mem_cache/test_radix_force_miss.py @@ -1,7 +1,7 @@ """Unit tests for SGLANG_RADIX_FORCE_MISS. The flag is gated at the scheduler boundary, so we test the helper directly -plus an end-to-end check of `match_prefix_for_req` driving a populated +plus an end-to-end check of `match_kv_cache` driving a populated RadixCache. """ @@ -10,20 +10,19 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=10, suite="base-a-test-cpu") import unittest -import unittest.mock from array import array from types import SimpleNamespace import torch from sglang.srt.environ import envs -from sglang.srt.managers.schedule_policy import match_prefix_for_req from sglang.srt.mem_cache.base_prefix_cache import ( InsertParams, MatchPrefixParams, MatchResult, zero_match_result, ) +from sglang.srt.mem_cache.common import match_kv_cache from sglang.srt.mem_cache.radix_cache import RadixCache, RadixKey @@ -79,7 +78,7 @@ class TestZeroMatchResult(unittest.TestCase): self.assertIs(zero_match_result(_StubChunkCache(), original), original) -class TestMatchPrefixForReqForceMiss(unittest.TestCase): +class TestMatchKvCacheForceMiss(unittest.TestCase): def test_swa_branching_seqlen_is_cleared_without_new_branch(self): class _StubTreeCache: def swa_reprefill_tail_tokens(self): @@ -98,7 +97,7 @@ class TestMatchPrefixForReqForceMiss(unittest.TestCase): req = _StubReq([1, 2, 3, 4]) req.swa_branching_seqlen = 8 - match_prefix_for_req(_StubTreeCache(), req) + match_kv_cache(req, _StubTreeCache()) self.assertIsNone(req.swa_branching_seqlen) @@ -113,14 +112,14 @@ class TestMatchPrefixForReqForceMiss(unittest.TestCase): # Sanity: without the flag, the same lookup hits. baseline_req = _StubReq([10, 11, 12, 13, 99, 100]) with envs.SGLANG_RADIX_FORCE_MISS.override(False): - match_prefix_for_req(tree, baseline_req) + match_kv_cache(baseline_req, tree) self.assertGreater(int(baseline_req.prefix_indices.numel()), 0) self.assertIsNot(baseline_req.last_node, tree.root_node) # With the flag, the same lookup is forced to miss. forced_req = _StubReq([10, 11, 12, 13, 99, 100]) with envs.SGLANG_RADIX_FORCE_MISS.override(True): - match_prefix_for_req(tree, forced_req) + match_kv_cache(forced_req, tree) self.assertEqual(int(forced_req.prefix_indices.numel()), 0) self.assertIs(forced_req.last_node, tree.root_node) self.assertIs(forced_req.last_host_node, tree.root_node)