From 1461152cb0857ea2d8d95d3e87c491e4438c54ab Mon Sep 17 00:00:00 2001 From: Liangsheng Yin Date: Sun, 18 Oct 2026 15:43:08 -0700 Subject: [PATCH] [Fix] deepseek-v4.1: Leave room for encoder replay rows when sizing a prefill chunk (#45040) --- python/sglang/srt/managers/schedule_policy.py | 23 ++++++++++-- .../srt/model_executor/encoder_swa_replay.py | 10 ++++- .../unit/managers/test_prefill_adder.py | 37 +++++++++++++++++++ 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index 8b31d88d0f..7eb3adcd8f 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -74,6 +74,7 @@ from sglang.srt.mem_cache.unified_cache.components import ( ComponentType, ) from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache +from sglang.srt.model_executor.encoder_swa_replay import replay_len if TYPE_CHECKING: from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator @@ -787,6 +788,7 @@ class PrefillAdder: extend_input_len: int, total_tokens: int, swa_host_hit_length: int, + chunk_limit: Optional[int], ) -> tuple[bool, Optional[int]]: return self.memory_budget.check_prefill( extend_input_len=extend_input_len, @@ -794,7 +796,7 @@ class PrefillAdder: max_new_tokens=self._swa_new_tokens(req), input_tokens=len(req.full_untruncated_fill_ids), swa_host_hit_length=swa_host_hit_length, - chunk_limit=self.rem_chunk_tokens, + chunk_limit=chunk_limit, auxiliary_swa_host_hit_length=req.auxiliary_swa_host_hit_length, ) @@ -908,11 +910,16 @@ class PrefillAdder: self.reprocessed_log_hit_tokens += prefix_len self.reprocessed_log_input_tokens += raw_extend_input_len - def _account_prefill_cache_admission(self, req: Req, prefix_len: int) -> None: + def _replay_rows(self, req: Req, prefix_len: int) -> int: + """Cached rows encoder replay folds in front of this request's extend.""" if get_exec().features.enable_encoder_swa_bounded_replay and ( req.kv.req_pool_idx is None or req.is_retracted ): - replay_tokens = min(prefix_len, 128) + return replay_len(prefix_len) + return 0 + + def _account_prefill_cache_admission(self, req: Req, prefix_len: int) -> None: + if replay_tokens := self._replay_rows(req, prefix_len): self.log_replay_tokens += replay_tokens self.rem_input_tokens -= replay_tokens if self.rem_chunk_tokens is not None: @@ -1537,11 +1544,21 @@ class PrefillAdder: # exact-chunk-fill, so a request whose ceiled length would spill is # not needlessly split into a second chunk. chunk_fit_tokens = extend_len if self.exact_chunk_fill else input_tokens + chunk_limit = self.rem_chunk_tokens + if chunk_limit is not None and ( + replay_rows := self._replay_rows(req, prefix_len) + ): + # The step also runs the replay rows, so they share the chunk: an + # extend sized to the whole chunk would pass the MegaMoE token cap. + chunk_limit -= replay_rows + if chunk_limit <= 0: + return AddReqResult.OTHER can_admit, chunk_tokens_limit = self._check_prefill_budget( req, extend_input_len=extend_len, total_tokens=total_tokens, swa_host_hit_length=swa_host_hit_length, + chunk_limit=chunk_limit, ) if not can_admit: return AddReqResult.NO_TOKEN diff --git a/python/sglang/srt/model_executor/encoder_swa_replay.py b/python/sglang/srt/model_executor/encoder_swa_replay.py index cad9522d57..593185c308 100644 --- a/python/sglang/srt/model_executor/encoder_swa_replay.py +++ b/python/sglang/srt/model_executor/encoder_swa_replay.py @@ -28,11 +28,17 @@ def fold_encoder_swa_replay(worker, batch) -> Optional[FoldedExtend]: return _fold_batch(batch=batch, runner=runner, rows=rows) +def replay_len(prefix_len: int) -> int: + """Cached tokens replayed in front of a hit's extend: the last SWA window. + The prefill budget, the MLP-sync row count and the fold all read this.""" + return min(prefix_len, 128) + + def folded_replay_rows(batch) -> int: if not batch.forward_mode.is_extend_without_speculative(): return 0 return sum( - min(end, 128) + replay_len(end) for reset, end in zip(batch.encoder_swa_reset, batch.prefix_lens) if reset ) @@ -114,7 +120,7 @@ def _reset_windows_and_collect_hits(*, batch, window): raise ValueError( "encoder SWA replay requires an even cached-prefix boundary" ) - rows.append((i, max(0, end - 128), end)) + rows.append((i, end - replay_len(end), end)) return rows diff --git a/test/registered/unit/managers/test_prefill_adder.py b/test/registered/unit/managers/test_prefill_adder.py index 0d90203997..4a0f9e1afd 100644 --- a/test/registered/unit/managers/test_prefill_adder.py +++ b/test/registered/unit/managers/test_prefill_adder.py @@ -374,6 +374,43 @@ class TestPrefillAdder(CustomTestCase): ) self.assertEqual(req.extend_len, expected) + def test_encoder_replay_rows_share_the_chunk(self): + # A hit's replay rows run in the same step as its extend, so both must + # fit the chunk: the MegaMoE token cap is sized from it. + override = get_context().override_server_args( + enable_encoder_swa_bounded_replay=True + ) + override.install() + self.addCleanup(override.restore) + self.mock_tree_cache.supports_mamba.return_value = False + self.mock_tree_cache.supports_prefix_sharing.return_value = False + self.mock_token_allocator.available_size.return_value = 32768 + for prefix_len, rem_chunk, extend_len in ( + (0, 1024, 1024), + (64, 1024, 960), + (4096, 1024, 896), + (4096, 128, None), + ): + with self.subTest(prefix_len=prefix_len, rem_chunk=rem_chunk): + adder = self.create_adder( + self.create_running_batch(), + page_size=64, + rem_chunk_tokens=rem_chunk, + ) + req = self.create_shared_req(f"hit-{prefix_len}-{rem_chunk}") + req.prefix_len = prefix_len + req.full_untruncated_fill_ids = list(range(prefix_len + 4096)) + result = adder.add_one_req( + req, has_chunked_req=False, truncation_align_size=None + ) + if extend_len is None: + self.assertEqual(result, AddReqResult.OTHER) + self.assertEqual(adder.can_run_list, []) + continue + self.assertEqual(req.extend_len, extend_len) + self.assertEqual(adder.log_replay_tokens, min(prefix_len, 128)) + self.assertEqual(adder.rem_chunk_tokens, 0) + def test_shared_admission_reserves_all_pending_requests(self): adder = self.create_shared_adder() first, second = (