From 58cb1a2be6abad4805511e2a6a1e65cdeb56dab5 Mon Sep 17 00:00:00 2001 From: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com> Date: Sun, 11 Oct 2026 12:18:41 +0800 Subject: [PATCH] [Scheduler] Keep waiting-prefix LRU refresh consistent across TP ranks (#43479) --- python/sglang/srt/managers/schedule_policy.py | 15 ++++++++------- .../test_fcfs_waiting_prefix_refresh.py | 19 +++++++++++++++++-- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index 5fa44c7442..54d7ff1166 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -32,7 +32,6 @@ logger = logging.getLogger(__name__) import os import random -import time from collections import Counter from contextlib import contextmanager, nullcontext from dataclasses import dataclass @@ -132,8 +131,8 @@ if PREFILL_TILE_BUDGET_MODE not in {"legacy", "compact"}: # Arbitrary; bounds per-round prefix work on the waiting queue. WAITING_QUEUE_PREFIX_MATCH_MAX = 128 -# Arbitrary; bounds the walk cost of touching waiting prefixes. -WAITING_PREFIX_REFRESH_INTERVAL_S = 0.5 +# Bound the walk cost using scheduling passes, which agree across TP ranks. +WAITING_PREFIX_REFRESH_INTERVAL = 32 def _ceil_div(value: int, divisor: int) -> int: @@ -202,7 +201,7 @@ class SchedulePolicy: self.schedule_low_priority_values_first = schedule_low_priority_values_first self.priority_sign = 1 if schedule_low_priority_values_first else -1 self._shortest_prefill_calls = 0 - self._last_waiting_prefix_refresh = float("-inf") + self._waiting_prefix_refresh_calls = 0 # It is used to find the matching prefix for in-batch prefix caching. self.waiting_queue_radix_tree = RadixCache.create_simulated() @@ -289,10 +288,12 @@ class SchedulePolicy: or not self.tree_cache.supports_prefix_sharing() ): return - now = time.monotonic() - if now - self._last_waiting_prefix_refresh < WAITING_PREFIX_REFRESH_INTERVAL_S: + refresh = ( + self._waiting_prefix_refresh_calls % WAITING_PREFIX_REFRESH_INTERVAL == 0 + ) + self._waiting_prefix_refresh_calls += 1 + if not refresh: return - self._last_waiting_prefix_refresh = now for r in reversed(waiting_queue[:WAITING_QUEUE_PREFIX_MATCH_MAX]): touch_waiting_prefix(r, self.tree_cache) diff --git a/test/registered/unit/mem_cache/test_fcfs_waiting_prefix_refresh.py b/test/registered/unit/mem_cache/test_fcfs_waiting_prefix_refresh.py index 12a6e47c9b..c61a462f47 100644 --- a/test/registered/unit/mem_cache/test_fcfs_waiting_prefix_refresh.py +++ b/test/registered/unit/mem_cache/test_fcfs_waiting_prefix_refresh.py @@ -1,6 +1,7 @@ import time import unittest from array import array +from unittest.mock import patch import torch @@ -33,7 +34,7 @@ class TestFcfsWaitingPrefixRefresh(CustomTestCase): self.addCleanup(reset_context) torch.set_default_device(None) - def _run_and_evict_one_prefix(self, eviction_policy): + def _run_and_evict_one_prefix(self, eviction_policy, second_refresh_time=None): kv_cache = MHATokenToKVPool( size=16, page_size=1, @@ -82,7 +83,15 @@ class TestFcfsWaitingPrefixRefresh(CustomTestCase): ServerArgs(model_path="dummy", radix_eviction_policy=eviction_policy), role="test", ) - SchedulePolicy("fcfs", cache, False, False, False).calc_priority(queue) + policy = SchedulePolicy("fcfs", cache, False, False, False) + # Isolate the scheduler clock from the radix cache's recency timestamps. + with patch("sglang.srt.managers.schedule_policy.time", create=True) as clock: + clock.monotonic.return_value = 100.0 + policy.calc_priority(queue) + if second_refresh_time is not None: + cache.touch_prefix(RadixKey(array("q", NEWER_PREFIX))) + clock.monotonic.return_value = second_refresh_time + policy.calc_priority(queue) cache.evict(EvictParams(num_tokens=len(OLDER_PREFIX))) return [ cache.match_prefix( @@ -94,6 +103,12 @@ class TestFcfsWaitingPrefixRefresh(CustomTestCase): def test_lru_evicts_the_prefix_furthest_from_admission(self): self.assertEqual(self._run_and_evict_one_prefix("lru"), [4, 0]) + def test_rank_local_clock_does_not_change_eviction(self): + before_interval = self._run_and_evict_one_prefix("lru", 100.49) + after_interval = self._run_and_evict_one_prefix("lru", 100.51) + self.assertEqual(before_interval, [0, 4]) + self.assertEqual(after_interval, before_interval) + def test_mru_keeps_plain_order(self): # MRU evicts the most recent first, so a refresh would invert its intent. self.assertEqual(self._run_and_evict_one_prefix("mru"), [4, 0])