From 0d7638b03509967e749c201146bfb04bbdfe380c Mon Sep 17 00:00:00 2001 From: metamergebot Date: Tue, 20 Oct 2026 13:24:04 -0700 Subject: [PATCH] [Fix] scheduler: Exclude accepted storage-tier hits from the waiting prefill load (#45158) Co-authored-by: metamergebot Co-authored-by: Jialin --- python/sglang/srt/managers/scheduler.py | 1 + .../scheduler_components/load_inquirer.py | 18 ++- .../sglang/srt/mem_cache/base_prefix_cache.py | 6 + .../srt/mem_cache/unified_radix_cache.py | 17 ++ .../unit/managers/test_load_inquirer.py | 145 +++++++++++++++++- .../test_storage_prefetch_lifecycle.py | 86 +++++++++++ 6 files changed, 267 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index 30d0506c1b..ddec8b8686 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2563,6 +2563,7 @@ class Scheduler( ), get_total_prefill_busy_us=lambda: self.total_prefill_busy_us, get_decode_moment_totals=lambda: self.decode_moment_totals, + get_waiting_storage_span=self.tree_cache.waiting_storage_span, ) def init_output_streamer(self) -> None: diff --git a/python/sglang/srt/managers/scheduler_components/load_inquirer.py b/python/sglang/srt/managers/scheduler_components/load_inquirer.py index 8808a53793..d475721757 100644 --- a/python/sglang/srt/managers/scheduler_components/load_inquirer.py +++ b/python/sglang/srt/managers/scheduler_components/load_inquirer.py @@ -22,6 +22,7 @@ if TYPE_CHECKING: ) from sglang.srt.managers.tp_worker import BaseTpWorker from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator + from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.spec_info import SpeculativeAlgorithm @@ -54,6 +55,9 @@ class SchedulerLoadInquirer: get_total_prefill_uncached_tokens: Callable get_total_prefill_busy_us: Callable get_decode_moment_totals: Callable + get_waiting_storage_span: ( + Callable[[CacheRequestHandle], tuple[int, int] | None] | None + ) = None def _get_num_pending_tokens(self, chunk_deduct: int = 0) -> int: """Get the total number of tokens pending prefill. @@ -84,9 +88,19 @@ class SchedulerLoadInquirer: num_tokens = 0 for req in self.get_waiting_queue(): if waiting_queue_prefix_matched: - num_tokens += max(0, req.seqlen - req.num_matched_prefix_tokens) + matched = req.num_matched_prefix_tokens + uncached = max(0, req.seqlen - matched) + if self.get_waiting_storage_span is not None: + span = self.get_waiting_storage_span(req.cache_request_handle) + if span is not None: + storage_tokens = max( + 0, min(span[1], req.seqlen) - max(span[0], matched) + ) + uncached -= storage_tokens else: - num_tokens += int(req.seqlen * cache_miss_rate) + # The historical hit rate already includes admitted storage hits. + uncached = int(req.seqlen * cache_miss_rate) + num_tokens += uncached cr = self.get_chunked_req() if cr is not None: num_tokens += max(0, cr.seqlen - cr.prefix_len) diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index e290a0ec35..748594ee2b 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -632,6 +632,12 @@ class BasePrefixCache(ABC, PrefixCacheTrait): def discard_storage_prefetch_accounting(self, handle: CacheRequestHandle) -> None: """Forget storage-hit lifecycle state without emitting a result.""" + def waiting_storage_span( + self, request: CacheRequestHandle + ) -> Optional[tuple[int, int]]: + """Expected storage-hit span in logical tokens for a waiting request.""" + return None + def pop_prefetch_loaded_span( self, handle: CacheRequestHandle ) -> tuple[int, Optional[int]]: diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index afcaa90b31..b9f886839d 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -2325,6 +2325,23 @@ class UnifiedRadixCache(BasePrefixCache): self._handle_prefetch_result(operation) return True + def waiting_storage_span( + self, request: CacheRequestHandle + ) -> Optional[tuple[int, int]]: + if self.buffer_pipeline is None: + return None + info = self.ongoing_prefetch.get(request) + if info is not None: + if info.host_indices is None: + return None + # Scheduler-accepted logical span; worker progress can count pool rows. + op = info.operation + return op.storage_start, op.storage_start + op.storage_hit_count + staged = self.buffer_pipeline.staged_prefetches.get(request) + if staged is not None: + return staged.matched_len, staged.matched_len + staged.num_tokens + return None + @rank_consensus(same_params=True, same_results=True) def check_prefetch_progress(self, request: CacheRequestHandle) -> bool: if ( diff --git a/test/registered/unit/managers/test_load_inquirer.py b/test/registered/unit/managers/test_load_inquirer.py index 51363a11bf..c54ebf1d60 100644 --- a/test/registered/unit/managers/test_load_inquirer.py +++ b/test/registered/unit/managers/test_load_inquirer.py @@ -1,5 +1,7 @@ import unittest +from dataclasses import replace from types import SimpleNamespace +from unittest.mock import Mock from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.managers.schedule_policy import ( @@ -10,6 +12,7 @@ from sglang.srt.managers.schedule_policy import ( from sglang.srt.managers.scheduler_components.load_inquirer import ( SchedulerLoadInquirer, ) +from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=10, suite="base-a-test-cpu") @@ -41,15 +44,43 @@ class TestSchedulePolicyWaitingQueueMatching(unittest.TestCase): class TestSchedulerLoadInquirer(unittest.TestCase): - def make_inquirer(self, waiting_queue_prefix_matched): - waiting_req = SimpleNamespace(seqlen=100, num_matched_prefix_tokens=20) + def make_inquirer( + self, + waiting_queue_prefix_matched=True, + storage_span=None, + disaggregation_mode=DisaggregationMode.NULL, + ): + waiting_req = SimpleNamespace( + seqlen=100, + num_matched_prefix_tokens=20, + cache_request_handle=CacheRequestHandle("waiting", 0), + ) chunked_req = SimpleNamespace(seqlen=50, prefix_len=10) - return SimpleNamespace( - disaggregation_mode=DisaggregationMode.NULL, + return SchedulerLoadInquirer( + disaggregation_mode=disaggregation_mode, + server_args=None, + max_total_num_tokens=1000, + max_running_requests=10, + pool_stats_observer=None, + tp_worker=None, + token_to_kv_pool_allocator=None, + spec_algorithm=None, + get_running_batch=None, get_waiting_queue=lambda: [waiting_req], waiting_queue_prefix_matched=lambda: waiting_queue_prefix_matched, get_chunked_req=lambda: chunked_req, get_recent_cache_hit_rate=lambda: 0.75, + get_stats=None, + get_disagg_prefill_bootstrap_queue=None, + get_disagg_prefill_inflight_queue=None, + get_disagg_decode_prealloc_queue=None, + get_disagg_decode_transfer_queue=None, + get_spec_total_num_accept_tokens=None, + get_spec_total_num_forward_ct=None, + get_total_prefill_uncached_tokens=None, + get_total_prefill_busy_us=None, + get_decode_moment_totals=None, + get_waiting_storage_span=Mock(return_value=storage_span), ) def test_waiting_tokens_are_estimated_when_prefix_matching_is_skipped(self): @@ -68,6 +99,112 @@ class TestSchedulerLoadInquirer(unittest.TestCase): 120, ) + def test_storage_exclusion_is_clipped_to_uncached_prompt(self): + for span, expected in ( + (None, 120), + ((0, 10), 120), + ((0, 40), 100), + ((20, 80), 60), + ((60, 120), 80), + ((20, 120), 40), + ((20, 20), 120), + ((100, 120), 120), + ): + with self.subTest(span=span): + inquirer = self.make_inquirer(storage_span=span) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), expected) + inquirer.get_waiting_storage_span.assert_called_once_with( + CacheRequestHandle("waiting", 0) + ) + + def test_storage_exclusion_requires_per_request_matching(self): + for prefix_matched, expected in ((True, 60), (False, 65)): + with self.subTest(prefix_matched=prefix_matched): + inquirer = self.make_inquirer( + waiting_queue_prefix_matched=prefix_matched, + storage_span=(20, 80), + ) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), expected) + if not prefix_matched: + inquirer.get_waiting_storage_span.assert_not_called() + + def test_storage_exclusion_does_not_change_decode(self): + for prefix_matched in (False, True): + with self.subTest(prefix_matched=prefix_matched): + inquirer = self.make_inquirer( + waiting_queue_prefix_matched=prefix_matched, + storage_span=(20, 100), + disaggregation_mode=DisaggregationMode.DECODE, + ) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 0) + inquirer.get_waiting_storage_span.assert_not_called() + + def test_storage_exclusion_preserves_the_historical_hit_rate_estimate(self): + for span in (None, (20, 50), (20, 80), (80, 100), (0, 120)): + with self.subTest(span=span): + inquirer = self.make_inquirer( + waiting_queue_prefix_matched=False, + storage_span=span, + ) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 65) + inquirer.get_waiting_storage_span.assert_not_called() + + def test_historical_estimate_already_includes_storage_hits(self): + requests = [ + SimpleNamespace(seqlen=100, cache_request_handle=CacheRequestHandle(rid, 0)) + for rid in ("storage_hit", "cold") + ] + get_storage_span = Mock( + side_effect=lambda handle: (0, 80) if handle.rid == "storage_hit" else None + ) + inquirer = replace( + self.make_inquirer(waiting_queue_prefix_matched=False), + get_waiting_queue=lambda: requests, + get_chunked_req=lambda: None, + get_recent_cache_hit_rate=lambda: 0.4, + get_waiting_storage_span=get_storage_span, + ) + + # One 80-token storage hit and one cold prompt need 20 + 100 tokens. + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 120) + get_storage_span.assert_not_called() + + def test_storage_exclusion_without_a_provider_keeps_the_original_estimate(self): + inquirer = replace( + self.make_inquirer(), + get_waiting_storage_span=None, + ) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 120) + + def test_scheduler_wires_storage_provider(self): + from sglang.srt.managers.scheduler import Scheduler + + req = self.make_inquirer().get_waiting_queue()[0] + get_storage_span = Mock(return_value=(20, 80)) + scheduler = SimpleNamespace( + disaggregation_mode=DisaggregationMode.PREFILL, + server_args=None, + max_total_num_tokens=1000, + kv_shard_widening=1, + max_running_requests=10, + pool_stats_observer=None, + tp_worker=None, + token_to_kv_pool_allocator=None, + spec_algorithm=None, + tree_cache=SimpleNamespace(waiting_storage_span=get_storage_span), + policy=SimpleNamespace(waiting_queue_prefix_matched=lambda queue: True), + metrics_reporter=SimpleNamespace(recent_cache_hit_rate=0.75), + waiting_queue=[req], + chunked_req=None, + ) + + Scheduler.init_load_inquirer(scheduler) + + inquirer = scheduler.load_inquirer + self.assertIs(inquirer.get_waiting_storage_span, get_storage_span) + self.assertEqual(inquirer.get_num_waiting_uncached_tokens(), 20) + get_storage_span.assert_called_once_with(req.cache_request_handle) + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py index 1ddfe4b5df..175e27a130 100644 --- a/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py +++ b/test/registered/unit/mem_cache/test_storage_prefetch_lifecycle.py @@ -13,6 +13,7 @@ import torch from sglang.srt.managers.schedule_batch import split_cached_prefix_by_tier from sglang.srt.mem_cache.base_prefix_cache import ( + BasePrefixCache, CacheRequestHandle, InitLoadBackParams, ) @@ -225,6 +226,79 @@ def _two_rank_prefetch_deadline(rank, rendezvous): torch.distributed.destroy_process_group() +class TestWaitingStorageSpan(unittest.TestCase): + def test_base_cache_has_no_storage_span(self): + self.assertIsNone(BasePrefixCache.waiting_storage_span(None, _REQ)) + + def test_query_result_waits_for_scheduler_acceptance(self): + cache, pipeline, _ = _staged_fixture() + pipeline.staged_prefetches.clear() + operation = SimpleNamespace( + storage_start=2, + storage_hit_count=6, + host_indices=torch.arange(6), + ) + cache.ongoing_prefetch[_REQ] = _OngoingPrefetch( + 0, RadixKey(range(6)), None, operation, None, {} + ) + + self.assertIsNone(cache.waiting_storage_span(_REQ)) + cache.ongoing_prefetch[_REQ] = cache.ongoing_prefetch[_REQ]._replace( + host_indices=operation.host_indices + ) + self.assertEqual(cache.waiting_storage_span(_REQ), (2, 8)) + + def test_accepted_span_uses_logical_length_not_worker_progress(self): + for host_length in (0, 2, 6): + with self.subTest(host_length=host_length): + cache, pipeline, _ = _staged_fixture() + pipeline.staged_prefetches.clear() + operation = SimpleNamespace( + storage_start=2, + storage_hit_count=6, + host_indices=None, + completed_tokens=0, + is_terminated=Mock(return_value=True), + ) + cache.ongoing_prefetch[_REQ] = _OngoingPrefetch( + 0, + RadixKey(range(6)), + torch.arange(host_length), + operation, + None, + {}, + ) + + self.assertEqual(cache.waiting_storage_span(_REQ), (2, 8)) + operation.completed_tokens = 4 + self.assertEqual(cache.waiting_storage_span(_REQ), (2, 8)) + operation.is_terminated.assert_not_called() + del cache.ongoing_prefetch[_REQ] + self.assertIsNone(cache.waiting_storage_span(_REQ)) + + def test_staged_span_disappears_when_the_hold_is_released(self): + cache, pipeline, _ = _staged_fixture() + self.assertEqual(cache.waiting_storage_span(_REQ), (2, 8)) + self.assertIsNone(cache.waiting_storage_span(CacheRequestHandle("r", 1))) + + pipeline.release_staged_hold(_REQ) + + self.assertIsNone(cache.waiting_storage_span(_REQ)) + + def test_resident_mode_keeps_existing_accounting(self): + cache, _, _ = _staged_fixture() + cache.buffer_pipeline = None + self.assertIsNone(cache.waiting_storage_span(_REQ)) + operation = SimpleNamespace( + storage_start=2, storage_hit_count=6, completed_tokens=0 + ) + cache.ongoing_prefetch[_REQ] = _OngoingPrefetch( + 0, RadixKey(range(6)), torch.arange(6), operation, None, {} + ) + # An accepted resident fetch may await admission-time completion handling. + self.assertIsNone(cache.waiting_storage_span(_REQ)) + + class TestStoragePrefetchTimeout(unittest.TestCase): def test_metric_episodes_do_not_restart_the_timeout(self): """Closing a wait-metric episode on readiness must not restart the @@ -440,6 +514,9 @@ class TestStagedPrefetchLifecycle(unittest.TestCase): self.assertTrue(operation.assume_stored) operation.hash_value = ["h0", "h1", "h2", "h3"] operation.storage_hit_count = 8 + self.assertIsNone( + cache.waiting_storage_span(req.cache_request_handle) + ) matched_len, hit_tokens = 2, 8 for trim in trims: matched_len += trim @@ -485,6 +562,11 @@ class TestStagedPrefetchLifecycle(unittest.TestCase): ) cache.cache_controller.prefetch_tokens_occupied = hit_tokens cache.storage_existence_cache = Mock() + if hit_tokens: + self.assertEqual( + cache.waiting_storage_span(req.cache_request_handle), + (matched_len, 10), + ) pipeline.stage_completed_prefetch( req.cache_request_handle, hit_tokens, operation.hash_value ) @@ -492,6 +574,10 @@ class TestStagedPrefetchLifecycle(unittest.TestCase): self.assertEqual(held.key_tokens, tokens) self.assertEqual(held.matched_len, matched_len) self.assertEqual(held.num_tokens, hit_tokens) + self.assertEqual( + cache.waiting_storage_span(req.cache_request_handle), + (matched_len, 10), + ) self.assertEqual( len(RadixKey(held.key_tokens, is_bigram=bigram)), 10 )