From 33d2ef17566685f596b82512e8264456716527f7 Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Sun, 11 Oct 2026 09:05:48 +0800 Subject: [PATCH] [Fix] Reserve remaining tokens for ignore_eos requests during prefill admission (#43564) --- python/sglang/srt/managers/schedule_policy.py | 4 +- .../unit/managers/test_prefill_adder.py | 4 +- .../mem_cache/test_prefill_memory_budget.py | 54 ++++++++++++++++++- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index b76432d122..5fa44c7442 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -751,12 +751,14 @@ class PrefillAdder: ) def _get_running_request_total_token_offset(self, req: Req) -> int: + # Fixed-length requests cannot benefit from the early-EOS estimate. + ratio = 1.0 if req.sampling_params.ignore_eos else self.new_token_ratio return ( min( (req.sampling_params.max_new_tokens - len(req.output_ids)), CLIP_MAX_NEW_TOKENS, ) - * self.new_token_ratio + * ratio ) @property diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index 37d704dac9..a8d11f9e58 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -128,7 +128,9 @@ class TestPrefillAdder(CustomTestCase): req.extend_end = None req.full_untruncated_fill_ids = [] req.output_ids = [0] * output_len - req.sampling_params = SimpleNamespace(max_new_tokens=max_new_tokens) + req.sampling_params = SimpleNamespace( + max_new_tokens=max_new_tokens, ignore_eos=False + ) req.time_stats = SimpleNamespace(wait_queue_entry_time=wait_time) req.retracted_stain = False req.host_hit_length = 0 diff --git a/test/registered/unit/mem_cache/test_prefill_memory_budget.py b/test/registered/unit/mem_cache/test_prefill_memory_budget.py index af36a77a51..eadb5f7894 100644 --- a/test/registered/unit/mem_cache/test_prefill_memory_budget.py +++ b/test/registered/unit/mem_cache/test_prefill_memory_budget.py @@ -8,7 +8,7 @@ from unittest.mock import MagicMock import torch from sglang.srt.managers.schedule_batch import Req -from sglang.srt.managers.schedule_policy import PrefillAdder +from sglang.srt.managers.schedule_policy import AddReqResult, PrefillAdder from sglang.srt.managers.scheduler import Scheduler from sglang.srt.mem_cache.allocator.hisparse import ( DeepSeekV4HiSparseTokenToKVPoolAllocator, @@ -17,15 +17,18 @@ from sglang.srt.mem_cache.allocator.swa import ( PureSWATokenToKVPoolAllocator, SWATokenToKVPoolAllocator, ) +from sglang.srt.mem_cache.allocator.token import TokenToKVPoolAllocator from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedMambaSWATokenToKVPoolAllocator, ) from sglang.srt.mem_cache.common import evict_from_tree_cache from sglang.srt.mem_cache.prefill_budget import SWAPrefillBudget +from sglang.srt.mem_cache.radix_cache import RadixCache from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools from sglang.srt.runtime_context import get_parallel from sglang.srt.sampling.sampling_params import SamplingParams from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, published_topology register_cpu_ci(est_time=5, suite="base-a-test-cpu") @@ -226,6 +229,55 @@ class TestSharedPrefillAdmission(unittest.TestCase): self.assertEqual(adder.can_run_list, [req]) +class TestRunningDecodeReservation(CustomTestCase): + def test_cached_admission_reserves_fixed_length_decode(self): + # 900 tokens remain free. A new 600-token prompt with 100 output tokens + # fits only if the running request can stop before its remaining 300. + for ignore_eos, decoded, should_admit in ( + (True, 100, False), + (True, 350, True), + (False, 100, True), + ): + with ( + self.subTest(ignore_eos=ignore_eos, decoded=decoded), + published_topology(), + ): + allocator = TokenToKVPoolAllocator( + 2048, torch.bfloat16, "cpu", None, False + ) + allocator.alloc(1148) + cache = RadixCache.create_simulated(mock_allocator=allocator) + running = Req( + "running", + None, + array("q", [1]), + SamplingParams(max_new_tokens=400, ignore_eos=ignore_eos), + ) + running.output_ids.extend([1] * decoded) + waiting = Req( + "waiting", + None, + array("q", [2] * 600), + SamplingParams(max_new_tokens=100, ignore_eos=True), + ) + waiting.init_next_round_input(cache) + adder = PrefillAdder( + page_size=1, + tree_cache=cache, + token_to_kv_pool_allocator=allocator, + running_batch=SimpleNamespace(reqs=[running]), + new_token_ratio=0.1, + rem_input_tokens=8192, + rem_chunk_tokens=8192, + ) + result = adder.add_one_req( + waiting, has_chunked_req=False, truncation_align_size=None + ) + self.assertEqual(bool(adder.can_run_list), should_admit) + if not should_admit: + self.assertEqual(result, AddReqResult.NO_TOKEN) + + class TestFixedPrefillMemoryBudget(unittest.TestCase): def _allocator(self, cls=SWATokenToKVPoolAllocator): allocator = object.__new__(cls)