[mem_cache] Clean up SWA/Mamba radix cache leftovers and drop SGLANG_ENABLE_UNIFIED_RADIX_TREE (#40780)
This commit is contained in:
@@ -11,7 +11,6 @@ from sglang.srt.arg_groups.model_override_base import (
|
||||
is_attention_backend_not_set,
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -76,5 +75,4 @@ def _inkling_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
f"Use {inkling_attn_backend} as the attention backend for Inkling "
|
||||
"(requires fa4 or triton)."
|
||||
)
|
||||
envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.set(True)
|
||||
return overrides
|
||||
|
||||
@@ -671,10 +671,6 @@ class Envs:
|
||||
SGLANG_OPT_UNIFIED_CACHE_FREE_OUT_OF_WINDOW_SLOTS = EnvBool(True)
|
||||
# Decode batches between SWA out-of-window evictions.
|
||||
SGLANG_SWA_EVICTION_INTERVAL = EnvInt(128)
|
||||
# Deprecated: the unified radix tree is the default tree cache now, so the
|
||||
# registry no longer reads this. Kept because a few model/arch call sites
|
||||
# still assert on it; do not use in new code.
|
||||
SGLANG_ENABLE_UNIFIED_RADIX_TREE = EnvBool(False)
|
||||
# Registered TreeCore backend serving the unified radix cache.
|
||||
SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND = EnvStr("python")
|
||||
SGLANG_OPT_SWA_RELEASE_LEAF_LOCK_AFTER_WINDOW = EnvBool(False)
|
||||
@@ -1975,8 +1971,7 @@ _DEPRECATED_ENVS: Dict[str, _DeprecatedEnv] = {
|
||||
"To override the threshold, use '--min-free-slots-delay'."
|
||||
),
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": _DeprecatedEnv(
|
||||
note="The unified radix tree is the default tree cache now; unset this "
|
||||
"env. The field is still defined for legacy call sites."
|
||||
note="The unified radix tree is the default tree cache now; unset this env."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -38,13 +38,14 @@ def walk_radix_cache_for_canary(
|
||||
f"walk_radix_cache_for_canary does not support {cache_type.__name__}"
|
||||
)
|
||||
|
||||
# RadixCache has no SWA tier, so every node it holds is SWA-resident and
|
||||
# swa_resident_only is a no-op here.
|
||||
slot_buf: list[int] = []
|
||||
position_buf: list[int] = []
|
||||
prev_slot_buf: list[int] = []
|
||||
|
||||
_walk_radix_subtree(
|
||||
node=radix_cache.root_node,
|
||||
radix_cache=radix_cache,
|
||||
depth=0,
|
||||
parent_last_slot=-1,
|
||||
slot_buf=slot_buf,
|
||||
@@ -52,7 +53,6 @@ def walk_radix_cache_for_canary(
|
||||
prev_slot_buf=prev_slot_buf,
|
||||
is_root=True,
|
||||
unlocked_only=unlocked_only,
|
||||
swa_resident_only=swa_resident_only,
|
||||
)
|
||||
|
||||
slot_tensor = torch.tensor(slot_buf, dtype=torch.int64)
|
||||
@@ -68,7 +68,6 @@ def walk_radix_cache_for_canary(
|
||||
def _walk_radix_subtree(
|
||||
*,
|
||||
node: TreeNode,
|
||||
radix_cache: BasePrefixCache,
|
||||
depth: int,
|
||||
parent_last_slot: int,
|
||||
slot_buf: list[int],
|
||||
@@ -76,21 +75,10 @@ def _walk_radix_subtree(
|
||||
prev_slot_buf: list[int],
|
||||
is_root: bool,
|
||||
unlocked_only: bool,
|
||||
swa_resident_only: bool,
|
||||
) -> None:
|
||||
node_slots = _node_slots_for_canary(node=node)
|
||||
|
||||
if unlocked_only:
|
||||
emit_slots = not is_root and _node_is_unlocked_for_canary(
|
||||
node=node, radix_cache=radix_cache
|
||||
)
|
||||
else:
|
||||
emit_slots = not is_root
|
||||
if swa_resident_only:
|
||||
emit_slots = emit_slots and _node_is_swa_resident_for_canary(
|
||||
node=node,
|
||||
radix_cache=radix_cache,
|
||||
)
|
||||
emit_slots = not is_root and (not unlocked_only or node.lock_ref == 0)
|
||||
|
||||
chain_last_slot = parent_last_slot
|
||||
for j, slot in enumerate(node_slots):
|
||||
@@ -105,7 +93,6 @@ def _walk_radix_subtree(
|
||||
for child in node.children.values():
|
||||
_walk_radix_subtree(
|
||||
node=child,
|
||||
radix_cache=radix_cache,
|
||||
depth=child_depth,
|
||||
parent_last_slot=chain_last_slot,
|
||||
slot_buf=slot_buf,
|
||||
@@ -113,7 +100,6 @@ def _walk_radix_subtree(
|
||||
prev_slot_buf=prev_slot_buf,
|
||||
is_root=False,
|
||||
unlocked_only=unlocked_only,
|
||||
swa_resident_only=swa_resident_only,
|
||||
)
|
||||
|
||||
|
||||
@@ -122,25 +108,3 @@ def _node_slots_for_canary(*, node: TreeNode) -> list[int]:
|
||||
if isinstance(value, torch.Tensor):
|
||||
return [int(s) for s in value.tolist()]
|
||||
return []
|
||||
|
||||
|
||||
def _node_is_unlocked_for_canary(
|
||||
*,
|
||||
node: TreeNode,
|
||||
radix_cache: BasePrefixCache,
|
||||
) -> bool:
|
||||
if type(radix_cache) is RadixCache:
|
||||
return node.lock_ref == 0
|
||||
|
||||
raise NotImplementedError(
|
||||
f"walk_radix_cache_for_canary does not support {type(radix_cache).__name__}"
|
||||
)
|
||||
|
||||
|
||||
def _node_is_swa_resident_for_canary(
|
||||
*,
|
||||
node: TreeNode,
|
||||
radix_cache: BasePrefixCache,
|
||||
) -> bool:
|
||||
# RadixCache has no SWA tier, so every node it holds is resident.
|
||||
return True
|
||||
|
||||
@@ -65,7 +65,7 @@ def free_swa_out_of_window_slots(
|
||||
if not req.kv.holds_kv:
|
||||
return
|
||||
|
||||
# For swa radix cache, we need to evict the tokens that are not in the tree cache and also not in the sliding window
|
||||
# For SWA-capable tree caches, we need to evict the tokens that are not in the tree cache and also not in the sliding window
|
||||
assert req.kv.cache_protected_len % page_size == 0, (
|
||||
"cache_protected_len must be page aligned"
|
||||
)
|
||||
|
||||
@@ -36,7 +36,6 @@ from sglang.srt.configs.hybrid_arch import (
|
||||
)
|
||||
from sglang.srt.configs.model_config import ModelImpl, is_deepseek_dsa
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.hardware_backend.mlx.runtime import use_mlx
|
||||
from sglang.srt.managers.mm_schedule import init_mm_embedding_cache
|
||||
from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
@@ -314,12 +313,6 @@ def build_kv_cache(
|
||||
and get_disagg().disaggregation_mode == "decode"
|
||||
):
|
||||
if is_hybrid_swa:
|
||||
if not (envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.get() or use_mlx()):
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache with sliding "
|
||||
"window attention (SWA) models requires the unified radix "
|
||||
"tree (set SGLANG_ENABLE_UNIFIED_RADIX_TREE=1)."
|
||||
)
|
||||
if enable_hierarchical_cache:
|
||||
raise ValueError(
|
||||
"--disaggregation-decode-enable-radix-cache with sliding "
|
||||
|
||||
@@ -367,18 +367,6 @@ class UnifiedLRUList:
|
||||
return None
|
||||
return x
|
||||
|
||||
def get_prev_leaf_no_lock(self, node: UnifiedTreeNode, check_id: bool = True):
|
||||
if check_id:
|
||||
assert node.id in self.cache
|
||||
pt = self._pt
|
||||
ct = self.component_type
|
||||
x = node.lru_prev[pt]
|
||||
while x.component_data[ct].lock_ref > 0 or len(x.children) > 0:
|
||||
x = x.lru_prev[pt]
|
||||
if x == self.head:
|
||||
return None
|
||||
return x
|
||||
|
||||
def get_prev_no_host_lock(self, node: UnifiedTreeNode, check_id: bool = True):
|
||||
"""Host-LRU walker: skip nodes whose component host_lock_ref > 0."""
|
||||
if check_id:
|
||||
@@ -395,9 +383,6 @@ class UnifiedLRUList:
|
||||
def get_lru_no_lock(self):
|
||||
return self.get_prev_no_lock(self.tail, check_id=False)
|
||||
|
||||
def get_leaf_lru_no_lock(self):
|
||||
return self.get_prev_leaf_no_lock(self.tail, check_id=False)
|
||||
|
||||
def get_lru_no_host_lock(self):
|
||||
return self.get_prev_no_host_lock(self.tail, check_id=False)
|
||||
|
||||
|
||||
@@ -1014,7 +1014,6 @@ class InklingForConditionalGeneration(nn.Module):
|
||||
self.config = config
|
||||
self.text_config = config.text_config
|
||||
|
||||
assert envs.SGLANG_ENABLE_UNIFIED_RADIX_TREE.get()
|
||||
if get_disagg().disaggregation_mode != "decode":
|
||||
assert not get_memory().disable_radix_cache
|
||||
assert not get_schedule().disable_hybrid_swa_memory
|
||||
|
||||
@@ -668,7 +668,6 @@ def unified_radix_tree_server_env(
|
||||
**os.environ,
|
||||
**extra_env,
|
||||
"SGLANG_ENABLE_RANK_CONSENSUS_CHECKER": "1",
|
||||
"SGLANG_ENABLE_UNIFIED_RADIX_TREE": "1",
|
||||
"SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND": tree_core_backend,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user