From 94e4611eacb9312ae1a952e110b00d07819a8528 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Thu, 22 Oct 2026 18:58:15 -0700 Subject: [PATCH] [Refactor] unified-cache: Factor match validators and match completion into helpers (#45714) Co-authored-by: Jialin --- .../unified_cache/unified_tree_core.py | 32 ++++---- .../srt/mem_cache/unified_radix_cache.py | 5 ++ .../test_unified_radix_cache_core.py | 81 +++++++++++++++++++ 3 files changed, 103 insertions(+), 15 deletions(-) diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 9877d231be..7f73fe2d54 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -881,6 +881,14 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): return self._match_prefix_helper(key, params) + def _create_match_validators( + self, params: MatchPrefixParams, match_device_only: bool + ) -> tuple[Callable[[UnifiedTreeNode], bool], ...]: + return tuple( + comp.create_match_validator(match_device_only=match_device_only) + for comp in self.components + ) + def _match_prefix_helper( self, key: RadixKey, params: MatchPrefixParams ) -> MatchResult: @@ -898,23 +906,17 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): full_kv_hit_length = 0 action: Optional[CacheAction | ComponentAction] = None separate_device_match = self.enable_hicache - if separate_device_match: - validators = tuple( - comp.create_match_validator() for comp in self.components - ) - device_validators = tuple( - comp.create_match_validator(match_device_only=True) - for comp in self.components - ) - else: - validators = tuple( - comp.create_match_validator(match_device_only=True) - for comp in self.components - ) - device_validators = validators + validators = self._create_match_validators( + params, match_device_only=not separate_device_match + ) + device_validators = ( + self._create_match_validators(params, match_device_only=True) + if separate_device_match + else validators + ) def _all_valid(validators, node): - return all([v(node) for v in validators]) + return all([validator(node) for validator in validators]) def _update_best_if_valid(node): nonlocal best_match_node diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 2c5b195dda..8e797ab7d4 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -621,6 +621,11 @@ class UnifiedRadixCache(BasePrefixCache): # Apply the walk's actions (e.g. a pending write-through relocation on # a split) before the finalizers, which can evict or raise. self._apply_cache_actions(result.cache_actions) + return self._finalize_match_result(params, result) + + def _finalize_match_result( + self, params: MatchPrefixParams, result: MatchResult + ) -> MatchResult: for component in self._components_tuple: result = component.finalize_match_result_in_cache(params, result) # Finalizers must not emit actions; the walk's were applied above. diff --git a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py index 6270f26a0a..0312abe349 100644 --- a/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py +++ b/test/registered/unit/mem_cache/unified_radix_cache/test_unified_radix_cache_core.py @@ -1,6 +1,7 @@ import unittest from array import array from dataclasses import replace +from types import SimpleNamespace from unittest import mock import torch @@ -14,6 +15,7 @@ from sglang.srt.mem_cache.base_prefix_cache import ( EvictParams, InsertParams, MatchPrefixParams, + MatchResult, TreeLock, zero_match_result, ) @@ -28,6 +30,7 @@ from sglang.srt.mem_cache.memory_pool import ( ) from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.cache_action import ( + FreeDeviceKV, RecoverSWAWithLockedFull, UpdatePendingBackupOnNodeSplit, ) @@ -47,8 +50,11 @@ from sglang.srt.mem_cache.unified_radix_cache import ( ) from sglang.srt.runtime_context import ( get_serving, + publish, + reset_context, ) from sglang.srt.sampling.sampling_params import SamplingParams +from sglang.srt.server_args import ServerArgs from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci from sglang.test.kits.unified_radix_cache.config import ( CacheConfig, @@ -1972,6 +1978,81 @@ class TestUnifiedRadixCacheActionRouting(CustomTestCase): cache._apply_cache_actions(result.cache_actions) self.assertEqual(cache._apply_cache_action.call_count, 2) + def test_match_completion_follows_actions_and_preserves_component_order(self): + # Match completion reads the disaggregation role from the config bags. + reset_context() + self.addCleanup(reset_context) + publish(ServerArgs(model_path="dummy"), role="scheduler") + cache = object.__new__(UnifiedRadixCache) + cache.disable = False + cache.session = mock.Mock() + cache.session.try_match_prefix.return_value = None + cache.tree_core = mock.Mock() + order = mock.Mock() + cache.linker = order.linker + cache.token_to_kv_pool_allocator = order.allocator + cache.components = { + ComponentType.SWA: order.swa, + ComponentType.FULL: order.full, + ComponentType.MAMBA: order.mamba, + } + + def complete_match(params, result): + return UnifiedRadixCache._finalize_match_result(cache, params, finalized) + + order.complete.side_effect = complete_match + cache._finalize_match_result = order.complete + indices = torch.tensor([4, 5]) + for names in (("swa", "full", "mamba"), ("mamba", "swa", "full")): + cache._components_tuple = tuple(getattr(order, name) for name in names) + for req in (SimpleNamespace(rid="first"), SimpleNamespace(rid="second")): + with self.subTest(components=names, req=req.rid): + order.reset_mock() + params = MatchPrefixParams( + key=RadixKey(array("q", [1, 2])), req=req + ) + split = UpdatePendingBackupOnNodeSplit(None, 1, 2, 1) + result = MatchResult( + indices.numel(), + 1, + 1, + 1, + cache_actions=[split, FreeDeviceKV([indices])], + last_visited_node_id=1, + full_device_match_len=1, + ) + finalized = result._replace(device_prefix_len=1) + cache.tree_core.match_prefix.return_value = result + for component in cache.components.values(): + component.finalize_match_result_in_cache.return_value = ( + finalized + ) + order.linker.match.return_value = finalized + + self.assertIs(cache.match_prefix(params), finalized) + expected = [ + mock.call.full.apply_component_action(split), + mock.call.allocator.free_segment(indices, start_pos=0), + mock.call.complete(params, result), + ] + expected.extend( + getattr(mock.call, name).finalize_match_result_in_cache( + params, finalized + ) + for name in names + ) + expected.append(mock.call.linker.match(params.key, req, finalized)) + self.assertEqual(order.mock_calls, expected) + received = order.complete.call_args.args + self.assertIs(received[0], params) + self.assertIs(received[0].req, req) + self.assertIs(received[1], result) + self.assertEqual(finalized.last_visited_node_id, 1) + self.assertEqual(finalized.full_device_match_len, 1) + self.assertEqual(result.cache_actions, []) + cache._apply_cache_actions(result.cache_actions) + self.assertEqual(order.mock_calls, expected) + def test_chained_replace_write_through_requires_list_order(self): # A pending node split twice in one walk emits two chained Replaces: # the second one's old_node_id only enters the publish list when the