From 4b44a1cde2c80fe2df78fab031ac268c84bc21b5 Mon Sep 17 00:00:00 2001 From: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:48:30 +0800 Subject: [PATCH] [Refactor] Let eviction policies take construction parameters (#37795) Co-authored-by: Claude Opus 5 (1M context) --- docs/docs.json | 1 + .../radix_eviction_policy.mdx | 72 +++++++++++++++++++ .../sglang/srt/mem_cache/cache_init_params.py | 5 +- .../sglang/srt/mem_cache/kv_cache_builder.py | 1 + python/sglang/srt/mem_cache/radix_cache.py | 4 +- .../srt/mem_cache/rust_tree_core/adapter.py | 6 ++ .../unified_cache/unified_tree_core.py | 4 +- python/sglang/srt/mem_cache/utils.py | 10 ++- python/sglang/srt/server_args.py | 20 +++++- 9 files changed, 116 insertions(+), 7 deletions(-) create mode 100644 docs/docs/advanced_features/radix_eviction_policy.mdx diff --git a/docs/docs.json b/docs/docs.json index 0ce559710..5edc8765d 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -925,6 +925,7 @@ "docs/advanced_features/overview", "docs/advanced_features/server_arguments", "docs/advanced_features/session_radix_cache", + "docs/advanced_features/radix_eviction_policy", "docs/advanced_features/hyperparameter_tuning", "docs/advanced_features/attention_backend", "docs/advanced_features/hisparse_guide", diff --git a/docs/docs/advanced_features/radix_eviction_policy.mdx b/docs/docs/advanced_features/radix_eviction_policy.mdx new file mode 100644 index 000000000..368f2b2e4 --- /dev/null +++ b/docs/docs/advanced_features/radix_eviction_policy.mdx @@ -0,0 +1,72 @@ +--- +title: "Radix Cache Eviction Policies" +metatags: + description: "Choose how the radix cache picks which prefix KV to drop under memory pressure, and tune a policy with --radix-eviction-policy-config." +--- + +When the KV cache pool is full, the radix cache reclaims space by evicting cached prefixes. The eviction policy decides which prefix goes first. The default, `lru`, is the right choice for most workloads; the others trade recency for hit frequency, reuse history, or request priority. + +## How eviction picks a victim + +Eviction only ever considers **evictable leaves**: nodes whose KV is present, unlocked (no in-flight request holds them), and not shadowed by a child that still holds KV. The root is never evictable. + +The policy scores each candidate and the **lowest score is evicted first**. Once a leaf is evicted, its parent may become a leaf and re-enter the candidate set, so eviction walks a branch from its tip toward the root. + +A policy only scores. It does not decide how much to free, and it cannot pin KV in memory — a node protected by the policy is still evicted if reclaiming everything else is not enough. + +## Available policies + +Select one with `--radix-eviction-policy`. All of them fall back to least-recently-used order within a tie. + +| Policy | Evicts first | Use when | +|---|---|---| +| `lru` (default) | The prefix unused for longest. | General serving. Matches how prefix reuse decays with time. | +| `lfu` | The prefix with the fewest cache hits, then the least recent. | A small set of prompts is reused far more than the rest, and you want those to survive bursts of one-off traffic. | +| `slru` | Prefixes still in the probationary segment, then the least recent within a segment. | Like `lfu`, but you want a hard floor on how much a one-off prefix can displace a proven one. See [`slru` parameters](#slru-parameters). | +| `priority` | The prefix belonging to the lowest-priority request, then the least recent. | You run [priority scheduling](/docs/advanced_features/server_arguments) and want cache retention to follow the same ranking as admission. | + +Notes on the scoring inputs: + +- **Hit count** (`lfu`, `slru`) counts how many times a node was matched by a later request. It is not incremented for chunked-prefill steps, for evicted nodes, or under write-back HiCache. +- **Request priority** (`priority`) is the `priority` field of the request that inserted the prefix; a node reached by several requests keeps the highest priority among them. Without priority scheduling every node is priority `0` and this policy is equivalent to `lru`. + +`fifo`, `mru`, and `filo` also exist in the policy registry but are not offered on the command line. They are reachable only by out-of-tree code that extends the choice list, and exist for experiments rather than serving. + +## Tuning a policy + +Some policies take parameters. Pass them as a json object to `--radix-eviction-policy-config`; the keys are the policy's own, so they are only valid for the policy you selected. + +```bash Command +python3 -m sglang.launch_server \ + --model-path MODEL_PATH \ + --radix-eviction-policy slru \ + --radix-eviction-policy-config '{"protected_threshold": 4}' +``` + +Omit the flag to accept every default. An unrecognized key fails at startup rather than being ignored: + +``` +TypeError: SLRUStrategy.__init__() got an unexpected keyword argument 'protected_treshold' +``` + + +`--radix-eviction-policy-config` is not supported by the experimental Rust tree core (`SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND=rust`), which builds its strategy from the policy name alone. Passing both fails at startup. + + +### Policy parameters + +Only `slru` currently takes a parameter. `lru`, `lfu`, and `priority` take none, so `--radix-eviction-policy-config` has no effect with them and any key is an error. + +#### `slru` parameters + +`slru` splits the cache into a **probationary** segment and a **protected** segment. A prefix enters probationary, and is promoted to protected once it has been hit enough times. Everything probationary is evicted before anything protected. + +| Key | Type | Default | Meaning | +|---|---|---|---| +| `protected_threshold` | int | `2` | Hit count at which a prefix is promoted to the protected segment. | + +Raising it makes promotion harder, so the protected set stays small and closer to your genuinely hot prefixes; a prefix hit three times stays probationary at `4` but is protected at the default `2`. Lowering it to `1` promotes any prefix that is reused even once, which approaches `lru` with a one-hit grace period. + +## Which policy to pick + +Start with `lru` and change it only against a measured cache hit rate — the counters are exposed under `--enable-metrics`. `lfu` and `slru` help when your traffic has a stable hot set that a recency-only policy keeps flushing; they hurt when prefix popularity shifts over time, because a prefix that earned a high hit count keeps its advantage after it stops being useful. diff --git a/python/sglang/srt/mem_cache/cache_init_params.py b/python/sglang/srt/mem_cache/cache_init_params.py index a5eb8122f..b3aa2a636 100644 --- a/python/sglang/srt/mem_cache/cache_init_params.py +++ b/python/sglang/srt/mem_cache/cache_init_params.py @@ -1,7 +1,7 @@ from __future__ import annotations import dataclasses -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any, Optional import torch @@ -27,6 +27,9 @@ class CacheInitParams: attn_tp_cache_group: Optional[torch.distributed.ProcessGroup] = None pp_cache_group: Optional[torch.distributed.ProcessGroup] = None eviction_policy: str = "lru" + # Keyword arguments for the eviction policy's constructor; see the strategy + # classes in evict_policy.py for what each policy accepts. + eviction_policy_config: Optional[dict[str, Any]] = None disable_finished_insert: bool = False enable_metrics: bool = False diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index ac030ac0e..65b591acd 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -306,6 +306,7 @@ def build_kv_cache( attn_tp_cache_group=attn_tp_cpu_group, pp_cache_group=pp_group.cpu_group, eviction_policy=get_memory().radix_eviction_policy, + eviction_policy_config=get_memory().radix_eviction_policy_config, enable_metrics=enable_metrics, enable_kv_cache_events=enable_kv_cache_events, enable_session_radix_cache=get_memory().enable_session_radix_cache, diff --git a/python/sglang/srt/mem_cache/radix_cache.py b/python/sglang/srt/mem_cache/radix_cache.py index dd5342f4b..10c527331 100644 --- a/python/sglang/srt/mem_cache/radix_cache.py +++ b/python/sglang/srt/mem_cache/radix_cache.py @@ -346,7 +346,9 @@ class RadixCache(BasePrefixCache): else: self.device = torch.device("cpu") - self.eviction_strategy = get_eviction_strategy(self.eviction_policy) + self.eviction_strategy = get_eviction_strategy( + self.eviction_policy, params.eviction_policy_config + ) self.evictable_leaves = set() self.reset() diff --git a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py index a9459ec6a..422c39eff 100644 --- a/python/sglang/srt/mem_cache/rust_tree_core/adapter.py +++ b/python/sglang/srt/mem_cache/rust_tree_core/adapter.py @@ -327,6 +327,12 @@ class RustUnifiedTreeCore(UnifiedTreeCoreInterface): raise ValueError( "Rust TreeCore does not support component_registry_override" ) + # The Rust core builds its own eviction strategy from the policy name + # alone, so a config would be dropped rather than applied. + if params.eviction_policy_config: + raise ValueError( + "Rust TreeCore does not support --radix-eviction-policy-config" + ) self._page_size = params.page_size self.is_eagle = ( diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index d97046d8c..072a4abfc 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -404,7 +404,9 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): self.is_write_back = False self.has_swa_host_pool = False self.enable_session_radix_cache = params.enable_session_radix_cache - self.eviction_strategy = get_eviction_strategy(params.eviction_policy.lower()) + self.eviction_strategy = get_eviction_strategy( + params.eviction_policy.lower(), params.eviction_policy_config + ) # ``device`` is derived from the construction-time allocator; the # allocator/pool themselves are owned by the cache, not the tree. diff --git a/python/sglang/srt/mem_cache/utils.py b/python/sglang/srt/mem_cache/utils.py index c2cc71a26..310623c91 100644 --- a/python/sglang/srt/mem_cache/utils.py +++ b/python/sglang/srt/mem_cache/utils.py @@ -56,7 +56,7 @@ from sglang.srt.mem_cache.evict_policy import ( SLRUStrategy, ) -_EVICTION_POLICY_FACTORIES: dict[str, Callable[[], EvictionStrategy]] = { +_EVICTION_POLICY_FACTORIES: dict[str, Callable[..., EvictionStrategy]] = { "lru": LRUStrategy, "lfu": LFUStrategy, "fifo": FIFOStrategy, @@ -67,15 +67,19 @@ _EVICTION_POLICY_FACTORIES: dict[str, Callable[[], EvictionStrategy]] = { } -def get_eviction_strategy(eviction_policy: str) -> EvictionStrategy: +def get_eviction_strategy( + eviction_policy: str, config: Optional[dict[str, Any]] = None +) -> EvictionStrategy: + """Build the eviction strategy; ``config`` is passed to it as keyword arguments.""" policy = eviction_policy.lower() try: - return _EVICTION_POLICY_FACTORIES[policy]() + factory = _EVICTION_POLICY_FACTORIES[policy] except KeyError: supported = "', '".join(_EVICTION_POLICY_FACTORIES) raise ValueError( f"Unknown eviction policy: {policy}. Supported policies: '{supported}'." ) from None + return factory(**config) if config else factory() def maybe_init_custom_mem_pool( diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 64a8d2715..e056a7b4e 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -839,12 +839,30 @@ class ServerArgs: "The eviction policy of radix trees. 'lru' stands for Least " "Recently Used, 'lfu' stands for Least Frequently Used, 'slru' " "stands for Segmented Least Recently Used, and 'priority' evicts " - "lower-priority requests first." + "lower-priority requests first. See " + "https://docs.sglang.io/docs/advanced_features/radix_eviction_policy " + "for what each policy optimizes for." ), choices=RADIX_EVICTION_POLICY_CHOICES, ), NS("memory"), ] = "lru" + radix_eviction_policy_config: A[ + Optional[Dict[str, Any]], + Arg( + help=( + "Tuning parameters for --radix-eviction-policy, as a json object " + "passed to the policy as keyword arguments. Only 'slru' takes any " + "today: protected_threshold (int, default 2), e.g. " + "'{\"protected_threshold\": 4}'. An unrecognized key fails at " + "startup, naming the key and the policy. See " + "https://docs.sglang.io/docs/advanced_features/radix_eviction_policy#policy-parameters " + "for the full parameter list." + ), + type_parser=json.loads, + ), + NS("memory"), + ] = None prefill_only_disable_kv_cache: A[ bool, "Skip the physical KV cache allocation for embedding-mode prefill-only workloads. Currently only valid with --is-embedding, --chunked-prefill-size=-1, --disable-radix-cache, an FA prefill backend, and non-FP4 KV cache so the fa_skip_kv_cache path is active (no layer reads or writes the cache). Other prefill-only workloads such as scoring/MIS may benefit from this later once their attention paths stop using paged KV. Scheduler admission accounting is unchanged; per-layer K/V tensors are sized to (page_size, head_num, head_dim) placeholders so GPU memory is not wasted.",