[Fix] Reserve remaining tokens for ignore_eos requests during prefill admission (#43564)

This commit is contained in:
Xiaoyu Zhang
2026-10-11 09:05:48 +08:00
committed by GitHub
parent 5730345717
commit 33d2ef1756
3 changed files with 59 additions and 3 deletions
@@ -751,12 +751,14 @@ class PrefillAdder:
)
def _get_running_request_total_token_offset(self, req: Req) -> int:
# Fixed-length requests cannot benefit from the early-EOS estimate.
ratio = 1.0 if req.sampling_params.ignore_eos else self.new_token_ratio
return (
min(
(req.sampling_params.max_new_tokens - len(req.output_ids)),
CLIP_MAX_NEW_TOKENS,
)
* self.new_token_ratio
* ratio
)
@property