[mem_cache] skip duplicates host evict via environ (#38462)

This commit is contained in:
Liwansi
2026-09-09 14:31:34 +08:00
committed by GitHub
parent a27be5ff62
commit 78da625190
2 changed files with 9 additions and 2 deletions
+1
View File
@@ -730,6 +730,7 @@ class Envs:
SGLANG_MLA_DEDUP_CHUNK_TOKENS = EnvInt(2048)
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)
SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM = EnvBool(False)
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
@@ -27,6 +27,7 @@ import msgspec
import torch
from sglang.srt.disaggregation.kv_events import StorageMedium
from sglang.srt.environ import envs
from sglang.srt.mem_cache.base_prefix_cache import (
DecLockRefParams,
DecLockRefResult,
@@ -1490,11 +1491,16 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
self, component_type: ComponentType, num_tokens: int
) -> DriveHostEvictionResult:
"""Evict a component's host-side resources; no-op if absent. Under
write_back, FULL pressure reclaims redundant Full host copies first."""
write_back, FULL pressure reclaims redundant Full host copies first
(skipped if SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM=1)."""
result = DriveHostEvictionResult()
comp = self.components_by_type.get(component_type)
if comp is not None:
if self.is_write_back and component_type == BASE_COMPONENT_TYPE:
if (
not envs.SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM.get()
and self.is_write_back
and component_type == BASE_COMPONENT_TYPE
):
self._reclaim_full_host_duplicates(
num_tokens,
result.tracker,