diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 42309d903..42b340a3d 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -263,6 +263,7 @@ class Envs: SGLANG_NCCL_ALL_GATHER_IN_OVERLAP_SCHEDULER_SYNC_BATCH = EnvBool(False) SGLANG_REQ_RUNNING_TIMEOUT = EnvFloat(-1) # in seconds SGLANG_DISAGGREGATION_BOOTSTRAP_ENTRY_CLEANUP_INTERVAL = EnvInt(120) + SGLANG_SWA_EVICTION_INTERVAL_MULTIPLIER = EnvFloat(1.0) # Test: pd-disaggregation SGLANG_TEST_PD_DISAGG_BACKEND = EnvStr("mooncake") diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index ddddbcef1..a6172890f 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -2482,12 +2482,22 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): sliding_window_size = self.tree_cache.sliding_window_size server_args = get_global_server_args() + # Eviction_interval: trade-off between SWA token waste and eviction overhead + page_size = self.tree_cache.page_size + eviction_interval = max( + page_size, + int( + sliding_window_size + * envs.SGLANG_SWA_EVICTION_INTERVAL_MULTIPLIER.get() + ), + ) + eviction_interval = (eviction_interval // page_size) * page_size for idx, req in enumerate(self.reqs): if self.forward_mode.is_decode(): # We set evict_swa condition here with two reasons: # 1. In overlap scheduler, we cannot evict swa when req.decode_batch_idx == 0 since the prev extend batch is still running. - # 2. Evict swa every window_size tokens to reduce the overhead. - if req.decode_batch_idx % sliding_window_size == 1: + # 2. Evict swa every eviction_interval tokens to reduce the overhead. + if req.decode_batch_idx % eviction_interval == 1: self._evict_swa(req, req.seqlen - 1) elif self.forward_mode.is_extend() and self.tree_cache.is_chunk_cache(): pre_len = self.prefix_lens[idx]