[Fix] deepseek-v4.1: Leave room for encoder replay rows when sizing a prefill chunk (#45040)

This commit is contained in:
Liangsheng Yin
2026-10-18 15:43:08 -07:00
committed by GitHub
parent fe89a7c890
commit 1461152cb0
3 changed files with 65 additions and 5 deletions
+20 -3
View File
@@ -74,6 +74,7 @@ from sglang.srt.mem_cache.unified_cache.components import (
ComponentType,
)
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
from sglang.srt.model_executor.encoder_swa_replay import replay_len
if TYPE_CHECKING:
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
@@ -787,6 +788,7 @@ class PrefillAdder:
extend_input_len: int,
total_tokens: int,
swa_host_hit_length: int,
chunk_limit: Optional[int],
) -> tuple[bool, Optional[int]]:
return self.memory_budget.check_prefill(
extend_input_len=extend_input_len,
@@ -794,7 +796,7 @@ class PrefillAdder:
max_new_tokens=self._swa_new_tokens(req),
input_tokens=len(req.full_untruncated_fill_ids),
swa_host_hit_length=swa_host_hit_length,
chunk_limit=self.rem_chunk_tokens,
chunk_limit=chunk_limit,
auxiliary_swa_host_hit_length=req.auxiliary_swa_host_hit_length,
)
@@ -908,11 +910,16 @@ class PrefillAdder:
self.reprocessed_log_hit_tokens += prefix_len
self.reprocessed_log_input_tokens += raw_extend_input_len
def _account_prefill_cache_admission(self, req: Req, prefix_len: int) -> None:
def _replay_rows(self, req: Req, prefix_len: int) -> int:
"""Cached rows encoder replay folds in front of this request's extend."""
if get_exec().features.enable_encoder_swa_bounded_replay and (
req.kv.req_pool_idx is None or req.is_retracted
):
replay_tokens = min(prefix_len, 128)
return replay_len(prefix_len)
return 0
def _account_prefill_cache_admission(self, req: Req, prefix_len: int) -> None:
if replay_tokens := self._replay_rows(req, prefix_len):
self.log_replay_tokens += replay_tokens
self.rem_input_tokens -= replay_tokens
if self.rem_chunk_tokens is not None:
@@ -1537,11 +1544,21 @@ class PrefillAdder:
# exact-chunk-fill, so a request whose ceiled length would spill is
# not needlessly split into a second chunk.
chunk_fit_tokens = extend_len if self.exact_chunk_fill else input_tokens
chunk_limit = self.rem_chunk_tokens
if chunk_limit is not None and (
replay_rows := self._replay_rows(req, prefix_len)
):
# The step also runs the replay rows, so they share the chunk: an
# extend sized to the whole chunk would pass the MegaMoE token cap.
chunk_limit -= replay_rows
if chunk_limit <= 0:
return AddReqResult.OTHER
can_admit, chunk_tokens_limit = self._check_prefill_budget(
req,
extend_input_len=extend_len,
total_tokens=total_tokens,
swa_host_hit_length=swa_host_hit_length,
chunk_limit=chunk_limit,
)
if not can_admit:
return AddReqResult.NO_TOKEN
@@ -28,11 +28,17 @@ def fold_encoder_swa_replay(worker, batch) -> Optional[FoldedExtend]:
return _fold_batch(batch=batch, runner=runner, rows=rows)
def replay_len(prefix_len: int) -> int:
"""Cached tokens replayed in front of a hit's extend: the last SWA window.
The prefill budget, the MLP-sync row count and the fold all read this."""
return min(prefix_len, 128)
def folded_replay_rows(batch) -> int:
if not batch.forward_mode.is_extend_without_speculative():
return 0
return sum(
min(end, 128)
replay_len(end)
for reset, end in zip(batch.encoder_swa_reset, batch.prefix_lens)
if reset
)
@@ -114,7 +120,7 @@ def _reset_windows_and_collect_hits(*, batch, window):
raise ValueError(
"encoder SWA replay requires an even cached-prefix boundary"
)
rows.append((i, max(0, end - 128), end))
rows.append((i, end - replay_len(end), end))
return rows