From a0534f8cca44779f94be6da9e1bfd50915d2789d Mon Sep 17 00:00:00 2001 From: Zhiqiang Xie Date: Fri, 18 Sep 2026 15:33:58 -0700 Subject: [PATCH] [HiCache] Stop arming a prefetch retry for a too-short storage span (#40042) --- python/sglang/srt/mem_cache/unified_radix_cache.py | 6 +++--- .../mem_cache/test_unified_radix_cache_unittest.py | 11 +++++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index dd3dc60ae..7b08faa6c 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1981,9 +1981,9 @@ class UnifiedRadixCache(BasePrefixCache): if prefetch_length < self.prefetch_threshold: if prefetch_length > 0: stats["declined_too_short"] += 1 - # A too-short/fully-matched suffix can become a full recompute if - # the device match evicts while queued; arm the retry. - self.storage_prefetch_retries.poll_miss(req_id) + # No lookup was issued, so no retry is armed: polling here would + # spend the re-issue budget the admission-time device-hit-loss + # re-query needs once the device match evicts while queued. return if not buffer_mode and self.cache_controller.prefetch_rate_limited(): stats["declined_rate_limited"] += 1 diff --git a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py index bde262818..08b3a097c 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py +++ b/test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py @@ -4276,8 +4276,9 @@ class UnifiedRadixCacheSuite: cons.finish(aborted_rid, CacheRequestOutcome.ABORT) self.assertNotIn(aborted_rid.rid, cons.storage_prefetch_retries._pending) - # A fully-device-matched (empty-suffix) decline also arms the retry: - # the device match can evict while the request waits in the queue. + # A fully-device-matched (empty-suffix) decline issues no lookup and + # arms no retry; a queue-time eviction of that device match is + # re-queried at admission with the re-issue budget intact. cons.prefetch_from_storage( CacheRequestHandle("fully-matched", 0), cons.root_node_handle(), @@ -4285,7 +4286,7 @@ class UnifiedRadixCacheSuite: None, None, ) - self.assertIn("fully-matched", cons.storage_prefetch_retries._pending) + self.assertNotIn("fully-matched", cons.storage_prefetch_retries._pending) cons.sanity_check() def test_buffer_only_anchor_lock_cap_clamped_by_context_headroom(self): @@ -5115,7 +5116,9 @@ class UnifiedRadixCacheSuite: CacheRequestHandle("subwin-req", 0), cons2.ongoing_prefetch ) self.assertEqual(cons2._prefetch_outcome_stats["declined_too_short"], 1) - self.assertIn("subwin-req", cons2.storage_prefetch_retries._pending) + # A declined span issues no lookup, so it must not spend the + # re-issue budget on polls. + self.assertNotIn("subwin-req", cons2.storage_prefetch_retries._pending) cons2.prefetch_from_storage( CacheRequestHandle("window-req", 0),