minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[HiCache] Account for newly pinned ancestors in load-back quota (#38481)

Browse Source
This commit is contained in:
Shuwen Wang
2026-09-10 23:54:08 +08:00
committed by GitHub
parent 92dffebe16
commit cc7e43ad22
2 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/mem_cache/unified_radix_cache.py
+1 -1
View File
@@ -1594,7 +1594,7 @@ class UnifiedRadixCache(BasePrefixCache):
# entirely empty spec (e.g. foreign-pin rejection) must never report # entirely empty spec (e.g. foreign-pin rejection) must never report
# success, even at load_back_threshold <= 0. # success, even at load_back_threshold <= 0.
if (kv_tokens < max(1, self.load_back_threshold) and not comp_xfers) or ( if (kv_tokens < max(1, self.load_back_threshold) and not comp_xfers) or (
mem_quota is not None and kv_tokens > mem_quota + result.delta mem_quota is not None and kv_tokens + result.delta > mem_quota
): ):
self.dec_lock_ref(node_id, ancestor_lock_params) self.dec_lock_ref(node_id, ancestor_lock_params)
self.dec_host_lock_ref(node_id, host_anchor_params) self.dec_host_lock_ref(node_id, host_anchor_params)
test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py
+2 -1
View File
@@ -6281,7 +6281,8 @@ class UnifiedRadixCacheSuite:
best_match_node=req.best_match_node, best_match_node=req.best_match_node,
host_hit_length=req.host_hit_length, host_hit_length=req.host_hit_length,
req=req, req=req,
mem_quota=-1_000_000, # Pinning the resident ancestors leaves too little for the tail.
mem_quota=len(tokens) - 1,
) )
) )