[Fix] scheduler: Exclude accepted storage-tier hits from the waiting prefill load (#45158)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: Jialin <Jialin@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
Jialin
parent
dc62c3effd
commit
0d7638b035
@@ -2563,6 +2563,7 @@ class Scheduler(
|
||||
),
|
||||
get_total_prefill_busy_us=lambda: self.total_prefill_busy_us,
|
||||
get_decode_moment_totals=lambda: self.decode_moment_totals,
|
||||
get_waiting_storage_span=self.tree_cache.waiting_storage_span,
|
||||
)
|
||||
|
||||
def init_output_streamer(self) -> None:
|
||||
|
||||
@@ -22,6 +22,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from sglang.srt.managers.tp_worker import BaseTpWorker
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import CacheRequestHandle
|
||||
from sglang.srt.server_args import ServerArgs
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
@@ -54,6 +55,9 @@ class SchedulerLoadInquirer:
|
||||
get_total_prefill_uncached_tokens: Callable
|
||||
get_total_prefill_busy_us: Callable
|
||||
get_decode_moment_totals: Callable
|
||||
get_waiting_storage_span: (
|
||||
Callable[[CacheRequestHandle], tuple[int, int] | None] | None
|
||||
) = None
|
||||
|
||||
def _get_num_pending_tokens(self, chunk_deduct: int = 0) -> int:
|
||||
"""Get the total number of tokens pending prefill.
|
||||
@@ -84,9 +88,19 @@ class SchedulerLoadInquirer:
|
||||
num_tokens = 0
|
||||
for req in self.get_waiting_queue():
|
||||
if waiting_queue_prefix_matched:
|
||||
num_tokens += max(0, req.seqlen - req.num_matched_prefix_tokens)
|
||||
matched = req.num_matched_prefix_tokens
|
||||
uncached = max(0, req.seqlen - matched)
|
||||
if self.get_waiting_storage_span is not None:
|
||||
span = self.get_waiting_storage_span(req.cache_request_handle)
|
||||
if span is not None:
|
||||
storage_tokens = max(
|
||||
0, min(span[1], req.seqlen) - max(span[0], matched)
|
||||
)
|
||||
uncached -= storage_tokens
|
||||
else:
|
||||
num_tokens += int(req.seqlen * cache_miss_rate)
|
||||
# The historical hit rate already includes admitted storage hits.
|
||||
uncached = int(req.seqlen * cache_miss_rate)
|
||||
num_tokens += uncached
|
||||
cr = self.get_chunked_req()
|
||||
if cr is not None:
|
||||
num_tokens += max(0, cr.seqlen - cr.prefix_len)
|
||||
|
||||
@@ -632,6 +632,12 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
def discard_storage_prefetch_accounting(self, handle: CacheRequestHandle) -> None:
|
||||
"""Forget storage-hit lifecycle state without emitting a result."""
|
||||
|
||||
def waiting_storage_span(
|
||||
self, request: CacheRequestHandle
|
||||
) -> Optional[tuple[int, int]]:
|
||||
"""Expected storage-hit span in logical tokens for a waiting request."""
|
||||
return None
|
||||
|
||||
def pop_prefetch_loaded_span(
|
||||
self, handle: CacheRequestHandle
|
||||
) -> tuple[int, Optional[int]]:
|
||||
|
||||
@@ -2325,6 +2325,23 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
self._handle_prefetch_result(operation)
|
||||
return True
|
||||
|
||||
def waiting_storage_span(
|
||||
self, request: CacheRequestHandle
|
||||
) -> Optional[tuple[int, int]]:
|
||||
if self.buffer_pipeline is None:
|
||||
return None
|
||||
info = self.ongoing_prefetch.get(request)
|
||||
if info is not None:
|
||||
if info.host_indices is None:
|
||||
return None
|
||||
# Scheduler-accepted logical span; worker progress can count pool rows.
|
||||
op = info.operation
|
||||
return op.storage_start, op.storage_start + op.storage_hit_count
|
||||
staged = self.buffer_pipeline.staged_prefetches.get(request)
|
||||
if staged is not None:
|
||||
return staged.matched_len, staged.matched_len + staged.num_tokens
|
||||
return None
|
||||
|
||||
@rank_consensus(same_params=True, same_results=True)
|
||||
def check_prefetch_progress(self, request: CacheRequestHandle) -> bool:
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user