[Scheduler] Skip the decode for a request whose last token is already in flight (#42720)

Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
zijiexia
2026-10-08 23:00:46 -07:00
committed by GitHub
co-authored by hnyls2002 Liangsheng Yin
parent 1fc6b8962c
commit e14ef4b658
10 changed files with 209 additions and 56 deletions
+4 -4
View File
@@ -1376,9 +1376,9 @@ class SchedulerDisaggregationPrefillMixin:
last_batch: Optional[ScheduleBatch],
running_batch: ScheduleBatch,
) -> None:
chunked_req_to_exclude = set()
reqs_to_exclude = set()
if (req := self.chunked_req) is not None:
chunked_req_to_exclude.add(req)
reqs_to_exclude.add(req)
self.checkpoint_disagg_prefill(req)
if not self.check_bootstrap(req):
@@ -1412,10 +1412,10 @@ class SchedulerDisaggregationPrefillMixin:
if last_batch.chunked_req:
# In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req.
# We need to discard it.
chunked_req_to_exclude.add(last_batch.chunked_req)
reqs_to_exclude.add(last_batch.chunked_req)
last_bs = last_batch.batch_size()
last_batch.filter_batch(chunked_req_to_exclude=list(chunked_req_to_exclude))
last_batch.filter_batch(reqs_to_exclude=list(reqs_to_exclude))
if last_batch.batch_size() < last_bs:
running_batch.batch_is_full = False
+10 -7
View File
@@ -1566,6 +1566,10 @@ class Req(ReqDllmMixin):
# Whether request reached finished condition
return self.finished_reason is not None
def next_output_finishes_by_length(self) -> bool:
# Whether committing one more output token reaches max_new_tokens
return len(self.output_ids) + 1 >= self.sampling_params.max_new_tokens
@property
def extend_len(self) -> int:
return self.extend_end - self.prefix_len
@@ -3627,20 +3631,19 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
def filter_batch(
self,
chunked_req_to_exclude: Optional[Union[Req, List[Req]]] = None,
reqs_to_exclude: Optional[Union[Req, List[Req]]] = None,
keep_indices: Optional[List[int]] = None,
):
strip_beam_tail(self)
if keep_indices is None:
if isinstance(chunked_req_to_exclude, Req):
chunked_req_to_exclude = [chunked_req_to_exclude]
elif chunked_req_to_exclude is None:
chunked_req_to_exclude = []
if isinstance(reqs_to_exclude, Req):
reqs_to_exclude = [reqs_to_exclude]
elif reqs_to_exclude is None:
reqs_to_exclude = []
keep_indices = [
i
for i in range(len(self.reqs))
if not self.reqs[i].finished()
and self.reqs[i] not in chunked_req_to_exclude
if not self.reqs[i].finished() and self.reqs[i] not in reqs_to_exclude
]
if keep_indices is None or len(keep_indices) == 0:
+45 -12
View File
@@ -1287,6 +1287,7 @@ class Scheduler(
tuple[ScheduleBatch, GenerationBatchResult | EmbeddingBatchResult]
] = deque()
self.enable_continuous_input_polling = False
self.enable_skip_finishing_decode = False
self.forward_ct = 0
self.return_health_check_ipcs: Deque[Optional[str]] = deque()
self.flush_wrapper = SchedulerFlushWrapper(
@@ -1937,6 +1938,7 @@ class Scheduler(
@DynamicGradMode()
def event_loop_overlap(self):
"""A scheduler loop that overlaps the CPU processing and GPU computation."""
self.enable_skip_finishing_decode = True
self.result_queue: Deque[
Tuple[ScheduleBatch, Union[GenerationBatchResult, EmbeddingBatchResult]]
] = deque()
@@ -3664,17 +3666,17 @@ class Scheduler(
self.dllm_manager.filter_finished_reqs()
# Merge the prefill batch into the running batch
chunked_req_to_exclude = set()
reqs_to_exclude = set()
if self.dllm_config is not None and self.dllm_manager.any_staging_reqs():
chunked_req_to_exclude.update(self.dllm_manager.staging_queue)
reqs_to_exclude.update(self.dllm_manager.staging_queue)
for req in self.dllm_manager.staging_queue:
self.finish_dllm_forward(req)
if self.chunked_req is not None:
# Move the chunked request out of the batch so that we can merge
# only finished requests to running_batch.
chunked_req_to_exclude.add(self.chunked_req)
reqs_to_exclude.add(self.chunked_req)
# A parked chunk (add_chunked_req hybrid-SWA early-return) leaves
# extend_end at prefix_len: it computed no new KV, so nothing to stash.
@@ -3702,14 +3704,14 @@ class Scheduler(
if last_batch.chunked_req is not None:
# In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req.
# We need to discard it.
chunked_req_to_exclude.add(last_batch.chunked_req)
reqs_to_exclude.add(last_batch.chunked_req)
if self.dllm_config is not None and last_batch.reqs:
chunked_req_to_exclude.update(last_batch.reqs)
reqs_to_exclude.update(last_batch.reqs)
# Filter batch
last_bs = last_batch.batch_size()
last_batch.filter_batch(chunked_req_to_exclude=list(chunked_req_to_exclude))
last_batch.filter_batch(reqs_to_exclude=list(reqs_to_exclude))
if last_batch.batch_size() < last_bs:
running_batch.batch_is_full = False
@@ -3760,8 +3762,12 @@ class Scheduler(
else:
# Run decode (skip for prefill-only batches)
if not running_batch.is_empty() and not running_batch.is_prefill_only:
running_batch = self.update_running_batch(running_batch)
ret = running_batch if not running_batch.is_empty() else None
decode_batch = self.update_running_batch(running_batch)
ret = (
decode_batch
if decode_batch is not None and not decode_batch.is_empty()
else None
)
else:
ret = None
@@ -4145,7 +4151,10 @@ class Scheduler(
and all(r.beam_group is None for r in running_batch.reqs)
):
# TODO (lianmin): support return_logprob + mixed chunked prefill
running_batch.filter_batch()
finishing_reqs = self._reqs_finishing_inflight(running_batch)
running_batch.filter_batch(reqs_to_exclude=finishing_reqs)
if finishing_reqs:
running_batch.batch_is_full = False
if not running_batch.is_empty():
running_batch.prepare_for_decode()
new_batch.mix_with_running(running_batch)
@@ -4197,17 +4206,40 @@ class Scheduler(
new_lora_set
)
def _reqs_finishing_inflight(self, batch: ScheduleBatch) -> List[Req]:
"""Requests whose queued result commits their last output by length."""
if not self.enable_skip_finishing_decode or not self.result_queue:
return []
# At scheduling time the overlap loop has processed every result but the last.
assert len(self.result_queue) == 1
queued_reqs = set(self.result_queue[0][0].reqs)
return [
req
for req in batch.reqs
if req in queued_reqs
and req.beam_group is None
and req.next_output_finishes_by_length()
]
def update_running_batch(self, batch: ScheduleBatch) -> Optional[ScheduleBatch]:
"""Update the current running decoding batch."""
"""Update the running decoding batch in place; None skips decode this step."""
initial_bs = batch.batch_size()
batch.filter_batch()
finishing_reqs = self._reqs_finishing_inflight(batch)
batch.filter_batch(reqs_to_exclude=finishing_reqs)
if batch.is_empty():
batch.batch_is_full = False
return batch
kv_full_retract_flag = not batch.check_decode_mem()
if kv_full_retract_flag and finishing_reqs:
# The queued result frees the finishing requests' KV at the end of
# this step; decode the rest next step instead of retracting them.
batch.batch_is_full = False
return None
# Check if decode out of memory
if (kv_full_retract_flag := not batch.check_decode_mem()) or (
if kv_full_retract_flag or (
TEST_RETRACT and self.forward_ct % TEST_RETRACT_INTERVAL == 0
):
if self.decode_offload_manager is not None:
@@ -5807,6 +5839,7 @@ def dispatch_event_loop(scheduler: Scheduler):
def _dispatch_event_loop_once(scheduler: Scheduler):
# A PD role switch can select a different loop on the same scheduler.
scheduler.enable_continuous_input_polling = False
scheduler.enable_skip_finishing_decode = False
disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode
if disaggregation_mode == DisaggregationMode.NULL:
if scheduler.enable_pdmux:
@@ -24,10 +24,10 @@ from sglang.test.server_fixtures.streaming_session_fixture import (
class StreamingSessionKitMixin:
"""Streaming-session KV-inheritance + retract/abort-recovery suite."""
# Allowed inherited-cache offsets vs the previous turn's total. Non-overlap
# spec decode can be off by 1: the bonus token's KV is only computed by the
# next forward, which sync skips at finish (overlap drains it, so it's 0).
kv_inherit_offsets = (0,)
# Allowed inherited-cache offsets vs the previous turn's total. The last
# output token's KV is only computed by a later forward: a turn that ends
# by length skips that forward, so the next turn prefills the token (-1).
kv_inherit_offsets = (0, -1)
def test_kv_cache_inheritance(self, gen_len=12):
"""Each turn's cached_tokens must equal previous turn's prompt+completion
+21 -6
View File
@@ -2,9 +2,12 @@
from __future__ import annotations
import functools
import time
from typing import Callable
import requests
from sglang.test.kl_test_utils import (
_extract_output_logprobs,
_flush_cache,
@@ -50,13 +53,25 @@ def default_prefill_cache_assert(result: dict, prefix_len: int, label: str):
def default_decode_cache_assert(
result: dict, history_len: int, output_len: int, label: str
result: dict, history_len: int, output_len: int, label: str, page_size: int = 1
):
"""Standard radix cache: cached_tokens == history_len + output_len."""
"""Standard radix cache: cached_tokens == history_len + output_len.
A previous turn that finished by length never computed its last output
token's KV, so the cache may hold one token less, cut to a page boundary.
"""
expected = history_len + output_len
allowed = {expected, expected - 1, (expected - 1) // page_size * page_size}
actual = result["meta_info"]["cached_tokens"]
assert actual == expected, (
f"{label}: expected cached_tokens={expected}, got {actual}"
assert actual in allowed, (
f"{label}: expected cached_tokens in {sorted(allowed)}, got {actual}"
)
def _default_decode_cache_assert_for(base_url: str) -> Callable:
server_info = requests.get(base_url + "/server_info", timeout=30).json()
return functools.partial(
default_decode_cache_assert, page_size=server_info["page_size"]
)
@@ -374,7 +389,7 @@ def test_input_output_logprobs_match_prefill_cache_hit_helper(
# Additional turns: decode cache hits (interleaved if order is set)
if turn_suffixes:
if assert_decode_cached_tokens is None:
assert_decode_cached_tokens = default_decode_cache_assert
assert_decode_cached_tokens = _default_decode_cache_assert_for(base_url)
for t, suffixes in enumerate(turn_suffixes):
current_input = [
@@ -462,7 +477,7 @@ def test_input_output_logprobs_match_decode_cache_hit_helper(
"turn_suffixes must have at least 1 entry (for turn 2)"
)
if assert_decode_cached_tokens is None:
assert_decode_cached_tokens = default_decode_cache_assert
assert_decode_cached_tokens = _default_decode_cache_assert_for(base_url)
n = len(first_turn_input_ids)
num_turns = 1 + len(turn_suffixes)