[Scheduler] Skip the decode for a request whose last token is already in flight (#42720)
Co-authored-by: hnyls2002 <lsyincs@gmail.com> Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
co-authored by
hnyls2002
Liangsheng Yin
parent
1fc6b8962c
commit
e14ef4b658
@@ -1376,9 +1376,9 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
last_batch: Optional[ScheduleBatch],
|
||||
running_batch: ScheduleBatch,
|
||||
) -> None:
|
||||
chunked_req_to_exclude = set()
|
||||
reqs_to_exclude = set()
|
||||
if (req := self.chunked_req) is not None:
|
||||
chunked_req_to_exclude.add(req)
|
||||
reqs_to_exclude.add(req)
|
||||
self.checkpoint_disagg_prefill(req)
|
||||
|
||||
if not self.check_bootstrap(req):
|
||||
@@ -1412,10 +1412,10 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if last_batch.chunked_req:
|
||||
# In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req.
|
||||
# We need to discard it.
|
||||
chunked_req_to_exclude.add(last_batch.chunked_req)
|
||||
reqs_to_exclude.add(last_batch.chunked_req)
|
||||
|
||||
last_bs = last_batch.batch_size()
|
||||
last_batch.filter_batch(chunked_req_to_exclude=list(chunked_req_to_exclude))
|
||||
last_batch.filter_batch(reqs_to_exclude=list(reqs_to_exclude))
|
||||
if last_batch.batch_size() < last_bs:
|
||||
running_batch.batch_is_full = False
|
||||
|
||||
|
||||
@@ -1566,6 +1566,10 @@ class Req(ReqDllmMixin):
|
||||
# Whether request reached finished condition
|
||||
return self.finished_reason is not None
|
||||
|
||||
def next_output_finishes_by_length(self) -> bool:
|
||||
# Whether committing one more output token reaches max_new_tokens
|
||||
return len(self.output_ids) + 1 >= self.sampling_params.max_new_tokens
|
||||
|
||||
@property
|
||||
def extend_len(self) -> int:
|
||||
return self.extend_end - self.prefix_len
|
||||
@@ -3627,20 +3631,19 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
|
||||
def filter_batch(
|
||||
self,
|
||||
chunked_req_to_exclude: Optional[Union[Req, List[Req]]] = None,
|
||||
reqs_to_exclude: Optional[Union[Req, List[Req]]] = None,
|
||||
keep_indices: Optional[List[int]] = None,
|
||||
):
|
||||
strip_beam_tail(self)
|
||||
if keep_indices is None:
|
||||
if isinstance(chunked_req_to_exclude, Req):
|
||||
chunked_req_to_exclude = [chunked_req_to_exclude]
|
||||
elif chunked_req_to_exclude is None:
|
||||
chunked_req_to_exclude = []
|
||||
if isinstance(reqs_to_exclude, Req):
|
||||
reqs_to_exclude = [reqs_to_exclude]
|
||||
elif reqs_to_exclude is None:
|
||||
reqs_to_exclude = []
|
||||
keep_indices = [
|
||||
i
|
||||
for i in range(len(self.reqs))
|
||||
if not self.reqs[i].finished()
|
||||
and self.reqs[i] not in chunked_req_to_exclude
|
||||
if not self.reqs[i].finished() and self.reqs[i] not in reqs_to_exclude
|
||||
]
|
||||
|
||||
if keep_indices is None or len(keep_indices) == 0:
|
||||
|
||||
@@ -1287,6 +1287,7 @@ class Scheduler(
|
||||
tuple[ScheduleBatch, GenerationBatchResult | EmbeddingBatchResult]
|
||||
] = deque()
|
||||
self.enable_continuous_input_polling = False
|
||||
self.enable_skip_finishing_decode = False
|
||||
self.forward_ct = 0
|
||||
self.return_health_check_ipcs: Deque[Optional[str]] = deque()
|
||||
self.flush_wrapper = SchedulerFlushWrapper(
|
||||
@@ -1937,6 +1938,7 @@ class Scheduler(
|
||||
@DynamicGradMode()
|
||||
def event_loop_overlap(self):
|
||||
"""A scheduler loop that overlaps the CPU processing and GPU computation."""
|
||||
self.enable_skip_finishing_decode = True
|
||||
self.result_queue: Deque[
|
||||
Tuple[ScheduleBatch, Union[GenerationBatchResult, EmbeddingBatchResult]]
|
||||
] = deque()
|
||||
@@ -3664,17 +3666,17 @@ class Scheduler(
|
||||
self.dllm_manager.filter_finished_reqs()
|
||||
|
||||
# Merge the prefill batch into the running batch
|
||||
chunked_req_to_exclude = set()
|
||||
reqs_to_exclude = set()
|
||||
|
||||
if self.dllm_config is not None and self.dllm_manager.any_staging_reqs():
|
||||
chunked_req_to_exclude.update(self.dllm_manager.staging_queue)
|
||||
reqs_to_exclude.update(self.dllm_manager.staging_queue)
|
||||
for req in self.dllm_manager.staging_queue:
|
||||
self.finish_dllm_forward(req)
|
||||
|
||||
if self.chunked_req is not None:
|
||||
# Move the chunked request out of the batch so that we can merge
|
||||
# only finished requests to running_batch.
|
||||
chunked_req_to_exclude.add(self.chunked_req)
|
||||
reqs_to_exclude.add(self.chunked_req)
|
||||
|
||||
# A parked chunk (add_chunked_req hybrid-SWA early-return) leaves
|
||||
# extend_end at prefix_len: it computed no new KV, so nothing to stash.
|
||||
@@ -3702,14 +3704,14 @@ class Scheduler(
|
||||
if last_batch.chunked_req is not None:
|
||||
# In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req.
|
||||
# We need to discard it.
|
||||
chunked_req_to_exclude.add(last_batch.chunked_req)
|
||||
reqs_to_exclude.add(last_batch.chunked_req)
|
||||
|
||||
if self.dllm_config is not None and last_batch.reqs:
|
||||
chunked_req_to_exclude.update(last_batch.reqs)
|
||||
reqs_to_exclude.update(last_batch.reqs)
|
||||
|
||||
# Filter batch
|
||||
last_bs = last_batch.batch_size()
|
||||
last_batch.filter_batch(chunked_req_to_exclude=list(chunked_req_to_exclude))
|
||||
last_batch.filter_batch(reqs_to_exclude=list(reqs_to_exclude))
|
||||
if last_batch.batch_size() < last_bs:
|
||||
running_batch.batch_is_full = False
|
||||
|
||||
@@ -3760,8 +3762,12 @@ class Scheduler(
|
||||
else:
|
||||
# Run decode (skip for prefill-only batches)
|
||||
if not running_batch.is_empty() and not running_batch.is_prefill_only:
|
||||
running_batch = self.update_running_batch(running_batch)
|
||||
ret = running_batch if not running_batch.is_empty() else None
|
||||
decode_batch = self.update_running_batch(running_batch)
|
||||
ret = (
|
||||
decode_batch
|
||||
if decode_batch is not None and not decode_batch.is_empty()
|
||||
else None
|
||||
)
|
||||
else:
|
||||
ret = None
|
||||
|
||||
@@ -4145,7 +4151,10 @@ class Scheduler(
|
||||
and all(r.beam_group is None for r in running_batch.reqs)
|
||||
):
|
||||
# TODO (lianmin): support return_logprob + mixed chunked prefill
|
||||
running_batch.filter_batch()
|
||||
finishing_reqs = self._reqs_finishing_inflight(running_batch)
|
||||
running_batch.filter_batch(reqs_to_exclude=finishing_reqs)
|
||||
if finishing_reqs:
|
||||
running_batch.batch_is_full = False
|
||||
if not running_batch.is_empty():
|
||||
running_batch.prepare_for_decode()
|
||||
new_batch.mix_with_running(running_batch)
|
||||
@@ -4197,17 +4206,40 @@ class Scheduler(
|
||||
new_lora_set
|
||||
)
|
||||
|
||||
def _reqs_finishing_inflight(self, batch: ScheduleBatch) -> List[Req]:
|
||||
"""Requests whose queued result commits their last output by length."""
|
||||
if not self.enable_skip_finishing_decode or not self.result_queue:
|
||||
return []
|
||||
# At scheduling time the overlap loop has processed every result but the last.
|
||||
assert len(self.result_queue) == 1
|
||||
queued_reqs = set(self.result_queue[0][0].reqs)
|
||||
return [
|
||||
req
|
||||
for req in batch.reqs
|
||||
if req in queued_reqs
|
||||
and req.beam_group is None
|
||||
and req.next_output_finishes_by_length()
|
||||
]
|
||||
|
||||
def update_running_batch(self, batch: ScheduleBatch) -> Optional[ScheduleBatch]:
|
||||
"""Update the current running decoding batch."""
|
||||
"""Update the running decoding batch in place; None skips decode this step."""
|
||||
initial_bs = batch.batch_size()
|
||||
|
||||
batch.filter_batch()
|
||||
finishing_reqs = self._reqs_finishing_inflight(batch)
|
||||
batch.filter_batch(reqs_to_exclude=finishing_reqs)
|
||||
if batch.is_empty():
|
||||
batch.batch_is_full = False
|
||||
return batch
|
||||
|
||||
kv_full_retract_flag = not batch.check_decode_mem()
|
||||
if kv_full_retract_flag and finishing_reqs:
|
||||
# The queued result frees the finishing requests' KV at the end of
|
||||
# this step; decode the rest next step instead of retracting them.
|
||||
batch.batch_is_full = False
|
||||
return None
|
||||
|
||||
# Check if decode out of memory
|
||||
if (kv_full_retract_flag := not batch.check_decode_mem()) or (
|
||||
if kv_full_retract_flag or (
|
||||
TEST_RETRACT and self.forward_ct % TEST_RETRACT_INTERVAL == 0
|
||||
):
|
||||
if self.decode_offload_manager is not None:
|
||||
@@ -5807,6 +5839,7 @@ def dispatch_event_loop(scheduler: Scheduler):
|
||||
def _dispatch_event_loop_once(scheduler: Scheduler):
|
||||
# A PD role switch can select a different loop on the same scheduler.
|
||||
scheduler.enable_continuous_input_polling = False
|
||||
scheduler.enable_skip_finishing_decode = False
|
||||
disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode
|
||||
if disaggregation_mode == DisaggregationMode.NULL:
|
||||
if scheduler.enable_pdmux:
|
||||
|
||||
@@ -24,10 +24,10 @@ from sglang.test.server_fixtures.streaming_session_fixture import (
|
||||
class StreamingSessionKitMixin:
|
||||
"""Streaming-session KV-inheritance + retract/abort-recovery suite."""
|
||||
|
||||
# Allowed inherited-cache offsets vs the previous turn's total. Non-overlap
|
||||
# spec decode can be off by 1: the bonus token's KV is only computed by the
|
||||
# next forward, which sync skips at finish (overlap drains it, so it's 0).
|
||||
kv_inherit_offsets = (0,)
|
||||
# Allowed inherited-cache offsets vs the previous turn's total. The last
|
||||
# output token's KV is only computed by a later forward: a turn that ends
|
||||
# by length skips that forward, so the next turn prefills the token (-1).
|
||||
kv_inherit_offsets = (0, -1)
|
||||
|
||||
def test_kv_cache_inheritance(self, gen_len=12):
|
||||
"""Each turn's cached_tokens must equal previous turn's prompt+completion
|
||||
|
||||
@@ -2,9 +2,12 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import functools
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
import requests
|
||||
|
||||
from sglang.test.kl_test_utils import (
|
||||
_extract_output_logprobs,
|
||||
_flush_cache,
|
||||
@@ -50,13 +53,25 @@ def default_prefill_cache_assert(result: dict, prefix_len: int, label: str):
|
||||
|
||||
|
||||
def default_decode_cache_assert(
|
||||
result: dict, history_len: int, output_len: int, label: str
|
||||
result: dict, history_len: int, output_len: int, label: str, page_size: int = 1
|
||||
):
|
||||
"""Standard radix cache: cached_tokens == history_len + output_len."""
|
||||
"""Standard radix cache: cached_tokens == history_len + output_len.
|
||||
|
||||
A previous turn that finished by length never computed its last output
|
||||
token's KV, so the cache may hold one token less, cut to a page boundary.
|
||||
"""
|
||||
expected = history_len + output_len
|
||||
allowed = {expected, expected - 1, (expected - 1) // page_size * page_size}
|
||||
actual = result["meta_info"]["cached_tokens"]
|
||||
assert actual == expected, (
|
||||
f"{label}: expected cached_tokens={expected}, got {actual}"
|
||||
assert actual in allowed, (
|
||||
f"{label}: expected cached_tokens in {sorted(allowed)}, got {actual}"
|
||||
)
|
||||
|
||||
|
||||
def _default_decode_cache_assert_for(base_url: str) -> Callable:
|
||||
server_info = requests.get(base_url + "/server_info", timeout=30).json()
|
||||
return functools.partial(
|
||||
default_decode_cache_assert, page_size=server_info["page_size"]
|
||||
)
|
||||
|
||||
|
||||
@@ -374,7 +389,7 @@ def test_input_output_logprobs_match_prefill_cache_hit_helper(
|
||||
# Additional turns: decode cache hits (interleaved if order is set)
|
||||
if turn_suffixes:
|
||||
if assert_decode_cached_tokens is None:
|
||||
assert_decode_cached_tokens = default_decode_cache_assert
|
||||
assert_decode_cached_tokens = _default_decode_cache_assert_for(base_url)
|
||||
|
||||
for t, suffixes in enumerate(turn_suffixes):
|
||||
current_input = [
|
||||
@@ -462,7 +477,7 @@ def test_input_output_logprobs_match_decode_cache_hit_helper(
|
||||
"turn_suffixes must have at least 1 entry (for turn 2)"
|
||||
)
|
||||
if assert_decode_cached_tokens is None:
|
||||
assert_decode_cached_tokens = default_decode_cache_assert
|
||||
assert_decode_cached_tokens = _default_decode_cache_assert_for(base_url)
|
||||
|
||||
n = len(first_turn_input_ids)
|
||||
num_turns = 1 + len(turn_suffixes)
|
||||
|
||||
Reference in New Issue
Block a user