From dab108b9f386b5b991ee070c7e4784f35df03f93 Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sun, 11 Oct 2026 00:18:27 -0700 Subject: [PATCH] [Scheduler] Hold back decode rows on a mixed step when they do not fit the KV pool (#43290) --- python/sglang/srt/managers/scheduler.py | 5 +- .../chunked_prefill/test_scripted_swa_1gpu.py | 86 ++++++++++++++++++- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index e8d7ed755e..205e09af88 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -4184,7 +4184,10 @@ class Scheduler( running_batch.filter_batch(reqs_to_exclude=finishing_reqs) if finishing_reqs: running_batch.batch_is_full = False - if not running_batch.is_empty(): + # The prefill admitted above may leave no KV room for the decode + # rows. They then sit out this step and decode on the next decode + # step, which fits them; a mixed step does not retract them. + if not running_batch.is_empty() and running_batch.check_decode_mem(): running_batch.prepare_for_decode() new_batch.mix_with_running(running_batch) new_batch.decoding_reqs = running_batch.reqs diff --git a/test/registered/chunked_prefill/test_scripted_swa_1gpu.py b/test/registered/chunked_prefill/test_scripted_swa_1gpu.py index 7937f4a157..3bcd9d30fd 100644 --- a/test/registered/chunked_prefill/test_scripted_swa_1gpu.py +++ b/test/registered/chunked_prefill/test_scripted_swa_1gpu.py @@ -5,7 +5,7 @@ from sglang.test.scripted_runtime.context import ScriptedContext from sglang.test.scripted_runtime.test_case import ScriptedTestCase from sglang.test.scripted_runtime_chunked_helpers import base_engine_kwargs -register_cuda_ci(est_time=130, stage="extra-a", runner_config="1-gpu-large") +register_cuda_ci(est_time=330, stage="extra-a", runner_config="1-gpu-large") _SWA_MODEL = "openai/gpt-oss-20b" @@ -26,6 +26,10 @@ _STEPS_PER_CANDIDATE = 120 _DECODER_WARMUP_STEPS = 60 _DRAIN_STEPS = 400 +_MIXED_DECODER_MAX_NEW = 256 +_MIXED_N_CANDIDATES = 4 +_MIXED_MAX_STEPS = 4000 + class TestScriptedSwaChunkedReqEarlyReturn(ScriptedTestCase): ENGINE_KWARGS = base_engine_kwargs( @@ -103,5 +107,85 @@ class TestScriptedSwaChunkedReqEarlyReturn(ScriptedTestCase): ) +class TestScriptedSwaMixedChunkDecodeFit(ScriptedTestCase): + # A mixed step whose admitted prefill leaves no SWA room for the running + # rows must hold them back for a decode step, not run out of the pool in + # alloc_for_decode. + ENGINE_KWARGS = base_engine_kwargs( + model_path=_SWA_MODEL, + chunked_prefill_size=_CHUNK_SIZE, + max_total_tokens=_MAX_TOTAL_TOKENS, + swa_full_tokens_ratio=_SWA_FULL_TOKENS_RATIO, + page_size=1, + mem_fraction_static=0.70, + enable_mixed_chunk=True, + ) + + def test_mixed_step_holds_back_decode_rows_that_do_not_fit(self): + self.server.execute_script( + self._script_mixed_step_holds_back_decode_rows_that_do_not_fit + ) + + @staticmethod + def _script_mixed_step_holds_back_decode_rows_that_do_not_fit( + t: ScriptedContext, + ): + s = t.scheduler + + decoders = [ + t.start_req( + prompt_len=_DECODER_PROMPT, + max_new_tokens=_MIXED_DECODER_MAX_NEW, + ignore_eos=True, + prompt_token=10 + i, + ) + for i in range(_N_DECODERS) + ] + for _ in range(_DECODER_WARMUP_STEPS): + if len(s.running_batch.reqs) >= _DECODER_WARMUP_RUNNING: + break + yield + + candidates = [ + t.start_req( + prompt_len=_CHUNKED_PROMPT, + max_new_tokens=_CHUNKED_MAX_NEW, + ignore_eos=True, + prompt_token=100 + i, + ) + for i in range(_MIXED_N_CANDIDATES) + ] + handles = decoders + candidates + + decode_held_back = False + decoder_rids = {h.rid for h in decoders} + for _ in range(_MIXED_MAX_STEPS): + batch = s.last_batch + if ( + batch is not None + and batch.forward_mode is not None + and batch.forward_mode.is_extend() + and not batch.forward_mode.is_mixed() + and any( + r.rid in decoder_rids and not r.finished() + for r in s.running_batch.reqs + ) + ): + decode_held_back = True + if all(h.finished for h in handles): + break + yield + else: + raise AssertionError( + f"requests still running after {_MIXED_MAX_STEPS} steps: " + f"{[h.rid for h in handles if not h.finished]}" + ) + + assert decode_held_back, ( + "no prefill ran while running decode rows sat out the step; the " + "test never put the SWA pool under pressure on a mixed step" + ) + + if __name__ == "__main__": unittest.main()