[mem_cache] skip duplicates host evict via environ (#38462)
This commit is contained in:
@@ -730,6 +730,7 @@ class Envs:
|
|||||||
SGLANG_MLA_DEDUP_CHUNK_TOKENS = EnvInt(2048)
|
SGLANG_MLA_DEDUP_CHUNK_TOKENS = EnvInt(2048)
|
||||||
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
|
SGLANG_HICACHE_HF3FS_CONFIG_PATH = EnvStr(None)
|
||||||
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)
|
SGLANG_HICACHE_DECODE_OFFLOAD_STRIDE = EnvInt(None)
|
||||||
|
SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM = EnvBool(False)
|
||||||
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
SGLANG_HICACHE_FILE_BACKEND_STORAGE_DIR = EnvStr(None)
|
||||||
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
|
# File-backend LRU eviction (opt-in; sizes accept SI/IEC suffixes, "0" disables).
|
||||||
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
|
SGLANG_HICACHE_FILE_BACKEND_MAX_SIZE = EnvStr(None)
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import msgspec
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
from sglang.srt.disaggregation.kv_events import StorageMedium
|
from sglang.srt.disaggregation.kv_events import StorageMedium
|
||||||
|
from sglang.srt.environ import envs
|
||||||
from sglang.srt.mem_cache.base_prefix_cache import (
|
from sglang.srt.mem_cache.base_prefix_cache import (
|
||||||
DecLockRefParams,
|
DecLockRefParams,
|
||||||
DecLockRefResult,
|
DecLockRefResult,
|
||||||
@@ -1490,11 +1491,16 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
|||||||
self, component_type: ComponentType, num_tokens: int
|
self, component_type: ComponentType, num_tokens: int
|
||||||
) -> DriveHostEvictionResult:
|
) -> DriveHostEvictionResult:
|
||||||
"""Evict a component's host-side resources; no-op if absent. Under
|
"""Evict a component's host-side resources; no-op if absent. Under
|
||||||
write_back, FULL pressure reclaims redundant Full host copies first."""
|
write_back, FULL pressure reclaims redundant Full host copies first
|
||||||
|
(skipped if SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM=1)."""
|
||||||
result = DriveHostEvictionResult()
|
result = DriveHostEvictionResult()
|
||||||
comp = self.components_by_type.get(component_type)
|
comp = self.components_by_type.get(component_type)
|
||||||
if comp is not None:
|
if comp is not None:
|
||||||
if self.is_write_back and component_type == BASE_COMPONENT_TYPE:
|
if (
|
||||||
|
not envs.SGLANG_HICACHE_SKIP_HOST_DUPLICATE_RECLAIM.get()
|
||||||
|
and self.is_write_back
|
||||||
|
and component_type == BASE_COMPONENT_TYPE
|
||||||
|
):
|
||||||
self._reclaim_full_host_duplicates(
|
self._reclaim_full_host_duplicates(
|
||||||
num_tokens,
|
num_tokens,
|
||||||
result.tracker,
|
result.tracker,
|
||||||
|
|||||||
Reference in New Issue
Block a user