minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[HiCache] Stop arming a prefetch retry for a too-short storage span (#40042)

Browse Source
This commit is contained in:
Zhiqiang Xie
2026-09-18 15:33:58 -07:00
committed by GitHub
parent d346b214fb
commit a0534f8cca
2 changed files with 10 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/mem_cache/unified_radix_cache.py
+3 -3
View File
@@ -1981,9 +1981,9 @@ class UnifiedRadixCache(BasePrefixCache):
if prefetch_length < self.prefetch_threshold: if prefetch_length < self.prefetch_threshold:
if prefetch_length > 0: if prefetch_length > 0:
stats["declined_too_short"] += 1 stats["declined_too_short"] += 1
# A too-short/fully-matched suffix can become a full recompute if # No lookup was issued, so no retry is armed: polling here would
# the device match evicts while queued; arm the retry. # spend the re-issue budget the admission-time device-hit-loss
self.storage_prefetch_retries.poll_miss(req_id) # re-query needs once the device match evicts while queued.
return return
if not buffer_mode and self.cache_controller.prefetch_rate_limited(): if not buffer_mode and self.cache_controller.prefetch_rate_limited():
stats["declined_rate_limited"] += 1 stats["declined_rate_limited"] += 1
test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py
+7 -4
View File
@@ -4276,8 +4276,9 @@ class UnifiedRadixCacheSuite:
cons.finish(aborted_rid, CacheRequestOutcome.ABORT) cons.finish(aborted_rid, CacheRequestOutcome.ABORT)
self.assertNotIn(aborted_rid.rid, cons.storage_prefetch_retries._pending) self.assertNotIn(aborted_rid.rid, cons.storage_prefetch_retries._pending)
# A fully-device-matched (empty-suffix) decline also arms the retry: # A fully-device-matched (empty-suffix) decline issues no lookup and
# the device match can evict while the request waits in the queue. # arms no retry; a queue-time eviction of that device match is
# re-queried at admission with the re-issue budget intact.
cons.prefetch_from_storage( cons.prefetch_from_storage(
CacheRequestHandle("fully-matched", 0), CacheRequestHandle("fully-matched", 0),
cons.root_node_handle(), cons.root_node_handle(),
@@ -4285,7 +4286,7 @@ class UnifiedRadixCacheSuite:
None, None,
None, None,
) )
self.assertIn("fully-matched", cons.storage_prefetch_retries._pending) self.assertNotIn("fully-matched", cons.storage_prefetch_retries._pending)
cons.sanity_check() cons.sanity_check()
def test_buffer_only_anchor_lock_cap_clamped_by_context_headroom(self): def test_buffer_only_anchor_lock_cap_clamped_by_context_headroom(self):
@@ -5115,7 +5116,9 @@ class UnifiedRadixCacheSuite:
CacheRequestHandle("subwin-req", 0), cons2.ongoing_prefetch CacheRequestHandle("subwin-req", 0), cons2.ongoing_prefetch
) )
self.assertEqual(cons2._prefetch_outcome_stats["declined_too_short"], 1) self.assertEqual(cons2._prefetch_outcome_stats["declined_too_short"], 1)
self.assertIn("subwin-req", cons2.storage_prefetch_retries._pending) # A declined span issues no lookup, so it must not spend the
# re-issue budget on polls.
self.assertNotIn("subwin-req", cons2.storage_prefetch_retries._pending)
cons2.prefetch_from_storage( cons2.prefetch_from_storage(
CacheRequestHandle("window-req", 0), CacheRequestHandle("window-req", 0),