From 33a22b1b083a5e3ece8c425cdc80095d1d947189 Mon Sep 17 00:00:00 2001 From: cctry Date: Thu, 3 Sep 2026 10:33:30 -0700 Subject: [PATCH] [Cache] Forward fast prefix matching capability (#37844) --- .../mem_cache/unified_cache/unified_tree_core_interface.py | 4 ++++ python/sglang/srt/mem_cache/unified_radix_cache.py | 3 +++ test/registered/unit/managers/test_load_inquirer.py | 5 ++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py index 213e7f3fa..57bc54b8c 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core_interface.py @@ -369,6 +369,10 @@ class UnifiedTreeCoreInterface(ABC): """Match a key against the tree; returns device indices + boundary NodeIds.""" ... + def supports_fast_match_prefix(self) -> bool: + """Whether matching every waiting request is cheap enough for scheduling.""" + return False + @property @abstractmethod def empty_match_result(self) -> MatchResult: diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 77e556914..1c603cac3 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -538,6 +538,9 @@ class UnifiedRadixCache(BasePrefixCache): result = self.linker.match(params.key, params.req, result) return result + def supports_fast_match_prefix(self) -> bool: + return self.tree_core.supports_fast_match_prefix() + def is_chunk_cache(self) -> bool: return self.disable diff --git a/test/registered/unit/managers/test_load_inquirer.py b/test/registered/unit/managers/test_load_inquirer.py index 73f3f1478..133c4eef7 100644 --- a/test/registered/unit/managers/test_load_inquirer.py +++ b/test/registered/unit/managers/test_load_inquirer.py @@ -31,11 +31,14 @@ class TestSchedulePolicyWaitingQueueMatching(unittest.TestCase): policy.tree_cache = SimpleNamespace(supports_fast_match_prefix=lambda: True) self.assertTrue(policy.waiting_queue_prefix_matched([])) - def test_lpm_queue_limit_can_disable_matching(self): + def test_lpm_queue_limit_respects_fast_matching_capability(self): policy = self.make_policy(CacheAwarePolicy.LPM, False) self.assertTrue(policy.waiting_queue_prefix_matched([None] * 128)) self.assertFalse(policy.waiting_queue_prefix_matched([None] * 129)) + policy.tree_cache = SimpleNamespace(supports_fast_match_prefix=lambda: True) + self.assertTrue(policy.waiting_queue_prefix_matched([None] * 129)) + class TestSchedulerLoadInquirer(unittest.TestCase): def make_inquirer(self, waiting_queue_prefix_matched):