[Fix] scheduler: Exclude accepted storage-tier hits from the waiting prefill load (#45158)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-20 13:24:04 -07:00
committed by GitHub
co-authored by metamergebot Jialin
parent dc62c3effd
commit 0d7638b035
6 changed files with 267 additions and 6 deletions
+1
View File
@@ -2563,6 +2563,7 @@ class Scheduler(
),
get_total_prefill_busy_us=lambda: self.total_prefill_busy_us,
get_decode_moment_totals=lambda: self.decode_moment_totals,
get_waiting_storage_span=self.tree_cache.waiting_storage_span,
)
def init_output_streamer(self) -> None:
@@ -22,6 +22,7 @@ if TYPE_CHECKING:
)
from sglang.srt.managers.tp_worker import BaseTpWorker
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
@@ -54,6 +55,9 @@ class SchedulerLoadInquirer:
get_total_prefill_uncached_tokens: Callable
get_total_prefill_busy_us: Callable
get_decode_moment_totals: Callable
get_waiting_storage_span: (
Callable[[CacheRequestHandle], tuple[int, int] | None] | None
) = None
def _get_num_pending_tokens(self, chunk_deduct: int = 0) -> int:
"""Get the total number of tokens pending prefill.
@@ -84,9 +88,19 @@ class SchedulerLoadInquirer:
num_tokens = 0
for req in self.get_waiting_queue():
if waiting_queue_prefix_matched:
num_tokens += max(0, req.seqlen - req.num_matched_prefix_tokens)
matched = req.num_matched_prefix_tokens
uncached = max(0, req.seqlen - matched)
if self.get_waiting_storage_span is not None:
span = self.get_waiting_storage_span(req.cache_request_handle)
if span is not None:
storage_tokens = max(
0, min(span[1], req.seqlen) - max(span[0], matched)
)
uncached -= storage_tokens
else:
num_tokens += int(req.seqlen * cache_miss_rate)
# The historical hit rate already includes admitted storage hits.
uncached = int(req.seqlen * cache_miss_rate)
num_tokens += uncached
cr = self.get_chunked_req()
if cr is not None:
num_tokens += max(0, cr.seqlen - cr.prefix_len)
@@ -632,6 +632,12 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
def discard_storage_prefetch_accounting(self, handle: CacheRequestHandle) -> None:
"""Forget storage-hit lifecycle state without emitting a result."""
def waiting_storage_span(
self, request: CacheRequestHandle
) -> Optional[tuple[int, int]]:
"""Expected storage-hit span in logical tokens for a waiting request."""
return None
def pop_prefetch_loaded_span(
self, handle: CacheRequestHandle
) -> tuple[int, Optional[int]]:
@@ -2325,6 +2325,23 @@ class UnifiedRadixCache(BasePrefixCache):
self._handle_prefetch_result(operation)
return True
def waiting_storage_span(
self, request: CacheRequestHandle
) -> Optional[tuple[int, int]]:
if self.buffer_pipeline is None:
return None
info = self.ongoing_prefetch.get(request)
if info is not None:
if info.host_indices is None:
return None
# Scheduler-accepted logical span; worker progress can count pool rows.
op = info.operation
return op.storage_start, op.storage_start + op.storage_hit_count
staged = self.buffer_pipeline.staged_prefetches.get(request)
if staged is not None:
return staged.matched_len, staged.matched_len + staged.num_tokens
return None
@rank_consensus(same_params=True, same_results=True)
def check_prefetch_progress(self, request: CacheRequestHandle) -> bool:
if (