diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index 0f7f8e6ecb..6c9ad51ae3 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -1376,9 +1376,9 @@ class SchedulerDisaggregationPrefillMixin: last_batch: Optional[ScheduleBatch], running_batch: ScheduleBatch, ) -> None: - chunked_req_to_exclude = set() + reqs_to_exclude = set() if (req := self.chunked_req) is not None: - chunked_req_to_exclude.add(req) + reqs_to_exclude.add(req) self.checkpoint_disagg_prefill(req) if not self.check_bootstrap(req): @@ -1412,10 +1412,10 @@ class SchedulerDisaggregationPrefillMixin: if last_batch.chunked_req: # In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req. # We need to discard it. - chunked_req_to_exclude.add(last_batch.chunked_req) + reqs_to_exclude.add(last_batch.chunked_req) last_bs = last_batch.batch_size() - last_batch.filter_batch(chunked_req_to_exclude=list(chunked_req_to_exclude)) + last_batch.filter_batch(reqs_to_exclude=list(reqs_to_exclude)) if last_batch.batch_size() < last_bs: running_batch.batch_is_full = False diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 3656fa9c6c..8346c93c3d 100755 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1566,6 +1566,10 @@ class Req(ReqDllmMixin): # Whether request reached finished condition return self.finished_reason is not None + def next_output_finishes_by_length(self) -> bool: + # Whether committing one more output token reaches max_new_tokens + return len(self.output_ids) + 1 >= self.sampling_params.max_new_tokens + @property def extend_len(self) -> int: return self.extend_end - self.prefix_len @@ -3627,20 +3631,19 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin): def filter_batch( self, - chunked_req_to_exclude: Optional[Union[Req, List[Req]]] = None, + reqs_to_exclude: Optional[Union[Req, List[Req]]] = None, keep_indices: Optional[List[int]] = None, ): strip_beam_tail(self) if keep_indices is None: - if isinstance(chunked_req_to_exclude, Req): - chunked_req_to_exclude = [chunked_req_to_exclude] - elif chunked_req_to_exclude is None: - chunked_req_to_exclude = [] + if isinstance(reqs_to_exclude, Req): + reqs_to_exclude = [reqs_to_exclude] + elif reqs_to_exclude is None: + reqs_to_exclude = [] keep_indices = [ i for i in range(len(self.reqs)) - if not self.reqs[i].finished() - and self.reqs[i] not in chunked_req_to_exclude + if not self.reqs[i].finished() and self.reqs[i] not in reqs_to_exclude ] if keep_indices is None or len(keep_indices) == 0: diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index dad4a86c27..4622e32faf 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -1287,6 +1287,7 @@ class Scheduler( tuple[ScheduleBatch, GenerationBatchResult | EmbeddingBatchResult] ] = deque() self.enable_continuous_input_polling = False + self.enable_skip_finishing_decode = False self.forward_ct = 0 self.return_health_check_ipcs: Deque[Optional[str]] = deque() self.flush_wrapper = SchedulerFlushWrapper( @@ -1937,6 +1938,7 @@ class Scheduler( @DynamicGradMode() def event_loop_overlap(self): """A scheduler loop that overlaps the CPU processing and GPU computation.""" + self.enable_skip_finishing_decode = True self.result_queue: Deque[ Tuple[ScheduleBatch, Union[GenerationBatchResult, EmbeddingBatchResult]] ] = deque() @@ -3664,17 +3666,17 @@ class Scheduler( self.dllm_manager.filter_finished_reqs() # Merge the prefill batch into the running batch - chunked_req_to_exclude = set() + reqs_to_exclude = set() if self.dllm_config is not None and self.dllm_manager.any_staging_reqs(): - chunked_req_to_exclude.update(self.dllm_manager.staging_queue) + reqs_to_exclude.update(self.dllm_manager.staging_queue) for req in self.dllm_manager.staging_queue: self.finish_dllm_forward(req) if self.chunked_req is not None: # Move the chunked request out of the batch so that we can merge # only finished requests to running_batch. - chunked_req_to_exclude.add(self.chunked_req) + reqs_to_exclude.add(self.chunked_req) # A parked chunk (add_chunked_req hybrid-SWA early-return) leaves # extend_end at prefix_len: it computed no new KV, so nothing to stash. @@ -3702,14 +3704,14 @@ class Scheduler( if last_batch.chunked_req is not None: # In the context pipeline parallelism, after the last chunk, the current microbatch still track outdated chunked_req. # We need to discard it. - chunked_req_to_exclude.add(last_batch.chunked_req) + reqs_to_exclude.add(last_batch.chunked_req) if self.dllm_config is not None and last_batch.reqs: - chunked_req_to_exclude.update(last_batch.reqs) + reqs_to_exclude.update(last_batch.reqs) # Filter batch last_bs = last_batch.batch_size() - last_batch.filter_batch(chunked_req_to_exclude=list(chunked_req_to_exclude)) + last_batch.filter_batch(reqs_to_exclude=list(reqs_to_exclude)) if last_batch.batch_size() < last_bs: running_batch.batch_is_full = False @@ -3760,8 +3762,12 @@ class Scheduler( else: # Run decode (skip for prefill-only batches) if not running_batch.is_empty() and not running_batch.is_prefill_only: - running_batch = self.update_running_batch(running_batch) - ret = running_batch if not running_batch.is_empty() else None + decode_batch = self.update_running_batch(running_batch) + ret = ( + decode_batch + if decode_batch is not None and not decode_batch.is_empty() + else None + ) else: ret = None @@ -4145,7 +4151,10 @@ class Scheduler( and all(r.beam_group is None for r in running_batch.reqs) ): # TODO (lianmin): support return_logprob + mixed chunked prefill - running_batch.filter_batch() + finishing_reqs = self._reqs_finishing_inflight(running_batch) + running_batch.filter_batch(reqs_to_exclude=finishing_reqs) + if finishing_reqs: + running_batch.batch_is_full = False if not running_batch.is_empty(): running_batch.prepare_for_decode() new_batch.mix_with_running(running_batch) @@ -4197,17 +4206,40 @@ class Scheduler( new_lora_set ) + def _reqs_finishing_inflight(self, batch: ScheduleBatch) -> List[Req]: + """Requests whose queued result commits their last output by length.""" + if not self.enable_skip_finishing_decode or not self.result_queue: + return [] + # At scheduling time the overlap loop has processed every result but the last. + assert len(self.result_queue) == 1 + queued_reqs = set(self.result_queue[0][0].reqs) + return [ + req + for req in batch.reqs + if req in queued_reqs + and req.beam_group is None + and req.next_output_finishes_by_length() + ] + def update_running_batch(self, batch: ScheduleBatch) -> Optional[ScheduleBatch]: - """Update the current running decoding batch.""" + """Update the running decoding batch in place; None skips decode this step.""" initial_bs = batch.batch_size() - batch.filter_batch() + finishing_reqs = self._reqs_finishing_inflight(batch) + batch.filter_batch(reqs_to_exclude=finishing_reqs) if batch.is_empty(): batch.batch_is_full = False return batch + kv_full_retract_flag = not batch.check_decode_mem() + if kv_full_retract_flag and finishing_reqs: + # The queued result frees the finishing requests' KV at the end of + # this step; decode the rest next step instead of retracting them. + batch.batch_is_full = False + return None + # Check if decode out of memory - if (kv_full_retract_flag := not batch.check_decode_mem()) or ( + if kv_full_retract_flag or ( TEST_RETRACT and self.forward_ct % TEST_RETRACT_INTERVAL == 0 ): if self.decode_offload_manager is not None: @@ -5807,6 +5839,7 @@ def dispatch_event_loop(scheduler: Scheduler): def _dispatch_event_loop_once(scheduler: Scheduler): # A PD role switch can select a different loop on the same scheduler. scheduler.enable_continuous_input_polling = False + scheduler.enable_skip_finishing_decode = False disaggregation_mode: DisaggregationMode = scheduler.disaggregation_mode if disaggregation_mode == DisaggregationMode.NULL: if scheduler.enable_pdmux: diff --git a/python/sglang/test/kits/streaming_session_kit.py b/python/sglang/test/kits/streaming_session_kit.py index 356fb7a1ce..c066f9a4c9 100644 --- a/python/sglang/test/kits/streaming_session_kit.py +++ b/python/sglang/test/kits/streaming_session_kit.py @@ -24,10 +24,10 @@ from sglang.test.server_fixtures.streaming_session_fixture import ( class StreamingSessionKitMixin: """Streaming-session KV-inheritance + retract/abort-recovery suite.""" - # Allowed inherited-cache offsets vs the previous turn's total. Non-overlap - # spec decode can be off by 1: the bonus token's KV is only computed by the - # next forward, which sync skips at finish (overlap drains it, so it's 0). - kv_inherit_offsets = (0,) + # Allowed inherited-cache offsets vs the previous turn's total. The last + # output token's KV is only computed by a later forward: a turn that ends + # by length skips that forward, so the next turn prefills the token (-1). + kv_inherit_offsets = (0, -1) def test_kv_cache_inheritance(self, gen_len=12): """Each turn's cached_tokens must equal previous turn's prompt+completion diff --git a/python/sglang/test/kl_multiturn_utils.py b/python/sglang/test/kl_multiturn_utils.py index a24f18bb8c..82d2f863c8 100644 --- a/python/sglang/test/kl_multiturn_utils.py +++ b/python/sglang/test/kl_multiturn_utils.py @@ -2,9 +2,12 @@ from __future__ import annotations +import functools import time from typing import Callable +import requests + from sglang.test.kl_test_utils import ( _extract_output_logprobs, _flush_cache, @@ -50,13 +53,25 @@ def default_prefill_cache_assert(result: dict, prefix_len: int, label: str): def default_decode_cache_assert( - result: dict, history_len: int, output_len: int, label: str + result: dict, history_len: int, output_len: int, label: str, page_size: int = 1 ): - """Standard radix cache: cached_tokens == history_len + output_len.""" + """Standard radix cache: cached_tokens == history_len + output_len. + + A previous turn that finished by length never computed its last output + token's KV, so the cache may hold one token less, cut to a page boundary. + """ expected = history_len + output_len + allowed = {expected, expected - 1, (expected - 1) // page_size * page_size} actual = result["meta_info"]["cached_tokens"] - assert actual == expected, ( - f"{label}: expected cached_tokens={expected}, got {actual}" + assert actual in allowed, ( + f"{label}: expected cached_tokens in {sorted(allowed)}, got {actual}" + ) + + +def _default_decode_cache_assert_for(base_url: str) -> Callable: + server_info = requests.get(base_url + "/server_info", timeout=30).json() + return functools.partial( + default_decode_cache_assert, page_size=server_info["page_size"] ) @@ -374,7 +389,7 @@ def test_input_output_logprobs_match_prefill_cache_hit_helper( # Additional turns: decode cache hits (interleaved if order is set) if turn_suffixes: if assert_decode_cached_tokens is None: - assert_decode_cached_tokens = default_decode_cache_assert + assert_decode_cached_tokens = _default_decode_cache_assert_for(base_url) for t, suffixes in enumerate(turn_suffixes): current_input = [ @@ -462,7 +477,7 @@ def test_input_output_logprobs_match_decode_cache_hit_helper( "turn_suffixes must have at least 1 entry (for turn 2)" ) if assert_decode_cached_tokens is None: - assert_decode_cached_tokens = default_decode_cache_assert + assert_decode_cached_tokens = _default_decode_cache_assert_for(base_url) n = len(first_turn_input_ids) num_turns = 1 + len(turn_suffixes) diff --git a/test/registered/scheduler/test_overlap_skip_finishing_decode.py b/test/registered/scheduler/test_overlap_skip_finishing_decode.py new file mode 100644 index 0000000000..48750490fe --- /dev/null +++ b/test/registered/scheduler/test_overlap_skip_finishing_decode.py @@ -0,0 +1,92 @@ +"""The overlap scheduler skips the decode of requests whose queued result +already carries their max_new_tokens-th token.""" + +import re +import unittest + +import requests + +from sglang.srt.environ import envs +from sglang.srt.utils import kill_process_tree +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import ( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + DEFAULT_URL_FOR_TEST, + CustomTestCase, + popen_launch_server, +) + +register_cuda_ci(est_time=60, stage="base-b", runner_config="1-gpu-small") + +BATCH_SIZE = 8 +MAX_NEW_TOKENS = 4 + + +def _forward_pass_count(base_url: str, mode_prefix: str) -> float: + metrics = requests.get(base_url + "/metrics", timeout=30).text + matches = re.findall( + r'^sglang:cuda_graph_passes_total\{[^}]*mode="' + mode_prefix + r'_[^"]*"' + r"[^}]*\}\s+([0-9.eE+-]+)$", + metrics, + re.MULTILINE, + ) + return sum(map(float, matches), 0.0) + + +class TestOverlapSkipFinishingDecode(CustomTestCase): + @classmethod + def setUpClass(cls): + cls.base_url = DEFAULT_URL_FOR_TEST + with envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY.override(1): + cls.process = popen_launch_server( + DEFAULT_SMALL_MODEL_NAME_FOR_TEST, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=["--enable-metrics"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def _generate(self, batch_size: int) -> list: + # Pre-tokenized input reaches the scheduler as one batched request. + resp = requests.post( + self.base_url + "/generate", + json={ + "input_ids": [[1000 + i] * 16 for i in range(batch_size)], + "sampling_params": { + "temperature": 0, + "max_new_tokens": MAX_NEW_TOKENS, + "ignore_eos": True, + }, + }, + timeout=60, + ) + resp.raise_for_status() + return resp.json() + + def test_batch_decodes_once_per_non_prefill_token(self): + self._generate(1) + prefill_before = _forward_pass_count(self.base_url, "prefill") + decode_before = _forward_pass_count(self.base_url, "decode") + + outputs = self._generate(BATCH_SIZE) + + self.assertEqual(len(outputs), BATCH_SIZE) + for output in outputs: + self.assertEqual(output["meta_info"]["completion_tokens"], MAX_NEW_TOKENS) + # One prefill emits the first token; each later token needs one decode, + # and no decode runs for the token already in flight. + self.assertEqual( + _forward_pass_count(self.base_url, "prefill") - prefill_before, 1 + ) + self.assertEqual( + _forward_pass_count(self.base_url, "decode") - decode_before, + MAX_NEW_TOKENS - 1, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/scripted_runtime/test_scripted_runtime_core.py b/test/registered/scripted_runtime/test_scripted_runtime_core.py index d6829552ea..556d0328f8 100644 --- a/test/registered/scripted_runtime/test_scripted_runtime_core.py +++ b/test/registered/scripted_runtime/test_scripted_runtime_core.py @@ -11,6 +11,7 @@ from sglang.test.scripted_runtime_chunked_helpers import ( advance_to_nth_chunk, base_engine_kwargs, exhaust_row_pool, + run_until, run_until_finished, warmup_radix, ) @@ -301,9 +302,11 @@ class TestScriptedRuntimeCore(ScriptedTestCase): @staticmethod def _script_ignore_eos_runs_full_length(t: ScriptedContext): r = t.start_req(prompt_len=_SHORT_PROMPT_LEN, max_new_tokens=6, ignore_eos=True) - yield from run_until_finished(r) + # Hold the req while it runs: once its last token is processed it may + # leave every scheduler set in the same step. + yield from advance_to_decode_step(r, 1) req = r.req - assert req is not None, "finished req vanished before its output could be read" + yield from run_until_finished(r) assert len(req.output_ids) == 6, ( f"ignore_eos must decode the full length; got {list(req.output_ids)!r}" ) @@ -643,13 +646,14 @@ class TestScriptedRuntimeCore(ScriptedTestCase): yield from warmup_radix(t, [1] * (2 * _CHUNK_SIZE)) r = t.start_req(prompt_len=2 * _CHUNK_SIZE + 1, max_new_tokens=1) + # Hold the req while it is visible: once its last token is processed it + # may leave every scheduler set in the same step. + yield from run_until(r, lambda h: h.req is not None) + req = r.req yield from run_until_finished(r) - assert r.req is not None, ( - "finished req vanished before cached_tokens could be read" - ) - assert r.req.cached_tokens > 0, ( + assert req.cached_tokens > 0, ( f"req with the warmed prefix should hit the radix cache; " - f"got cached_tokens={r.req.cached_tokens}" + f"got cached_tokens={req.cached_tokens}" ) def test_exhaust_kv_creates_pressure_and_release_restores(self): diff --git a/test/registered/sessions/test_streaming_session_extra.py b/test/registered/sessions/test_streaming_session_extra.py index 96be186e5b..8adea371c7 100644 --- a/test/registered/sessions/test_streaming_session_extra.py +++ b/test/registered/sessions/test_streaming_session_extra.py @@ -52,9 +52,8 @@ _EAGLE3_SPEC_ARGS = [ class TestStreamingSessionEagle(StreamingSessionServerBase, StreamingSessionKitMixin): - """EAGLE3 spec v2, overlap disabled; inherited count jitters {0, -1} — see kit's note.""" + """EAGLE3 spec v2, overlap disabled.""" - kv_inherit_offsets = (0, -1) model = DEFAULT_TARGET_MODEL_EAGLE3 extra_args = [ "--disable-overlap-schedule", @@ -83,10 +82,8 @@ class TestStreamingSessionEagleRetractLargePage( StreamingSessionServerBase, StreamingSessionKitMixin ): """EAGLE3 spec v2 (overlap disabled) + retract + page=256: max-pressure on - `_free_tail` (spec tail + retract alloc-commit gap + page alignment). - Inherited count jitters {0, -1} — see kit's note.""" + `_free_tail` (spec tail + retract alloc-commit gap + page alignment).""" - kv_inherit_offsets = (0, -1) model = DEFAULT_TARGET_MODEL_EAGLE3 extra_args = [ "--disable-overlap-schedule", diff --git a/test/registered/spec/eagle/test_adaptive_speculative.py b/test/registered/spec/eagle/test_adaptive_speculative.py index 7b123d0f31..331ca6a74b 100644 --- a/test/registered/spec/eagle/test_adaptive_speculative.py +++ b/test/registered/spec/eagle/test_adaptive_speculative.py @@ -61,6 +61,7 @@ class TestAdaptiveSpeculativeServer(CustomTestCase): f, ) cls.adaptive_config_path = f.name + cls.server_log = tempfile.NamedTemporaryFile("w", suffix=".log", delete=False) try: cls.process = popen_launch_server( @@ -86,9 +87,12 @@ class TestAdaptiveSpeculativeServer(CustomTestCase): "--mem-fraction-static", "0.7", ], + return_stdout_stderr=(cls.server_log, cls.server_log), ) except Exception: os.unlink(cls.adaptive_config_path) + cls.server_log.close() + os.unlink(cls.server_log.name) raise @classmethod @@ -97,6 +101,8 @@ class TestAdaptiveSpeculativeServer(CustomTestCase): kill_process_tree(cls.process.pid) if os.path.exists(cls.adaptive_config_path): os.unlink(cls.adaptive_config_path) + cls.server_log.close() + os.unlink(cls.server_log.name) def _get_internal_state(self) -> dict: response = requests.get(self.base_url + "/server_info", timeout=30) @@ -147,25 +153,27 @@ class TestAdaptiveSpeculativeServer(CustomTestCase): return state return state - def _drive_downshift(self) -> dict: - """Send low-acceptance prompts until steps downshift to 1.""" - state = self._get_internal_state() + def _drive_downshift(self) -> bool: + """Send low-acceptance prompts until the server logs a downshift to 1. + + With ema_alpha=1 the steps follow the last verify, so a low-acceptance + request can end at either step; the log records every switch. + """ + log_start = os.path.getsize(self.server_log.name) for _ in range(MAX_DOWNSHIFT_ATTEMPTS): self._generate(LOW_ACCEPT_PROMPT) - state = self._get_internal_state() - if state["speculative_num_steps"] == 1: - return state - return state + with open(self.server_log.name) as f: + f.seek(log_start) + if "steps 3 -> 1" in f.read(): + return True + return False def test_gsm8k_after_adaptive_switches(self): """Exercise up/down/up adaptive switches, then verify GSM8K accuracy.""" state = self._drive_upshift() self.assertEqual(state["speculative_num_steps"], 3, f"Never upshifted: {state}") - state = self._drive_downshift() - self.assertEqual( - state["speculative_num_steps"], 1, f"Never downshifted: {state}" - ) + self.assertTrue(self._drive_downshift(), "Never downshifted") self._drive_upshift() diff --git a/test/registered/unit/managers/test_scheduler_pause_generation.py b/test/registered/unit/managers/test_scheduler_pause_generation.py index 86785a94ab..c6ec667e37 100644 --- a/test/registered/unit/managers/test_scheduler_pause_generation.py +++ b/test/registered/unit/managers/test_scheduler_pause_generation.py @@ -71,6 +71,7 @@ class TestSchedulerPauseGeneration(CustomTestCase): scheduler = Scheduler.__new__(Scheduler) scheduler._engine_paused = False scheduler.enable_overlap = False + scheduler.enable_skip_finishing_decode = False scheduler.last_batch = None scheduler.cur_batch_for_debug = None scheduler.chunked_req = None