[Fix] Reserve remaining tokens for ignore_eos requests during prefill admission (#43564)
This commit is contained in:
@@ -751,12 +751,14 @@ class PrefillAdder:
|
||||
)
|
||||
|
||||
def _get_running_request_total_token_offset(self, req: Req) -> int:
|
||||
# Fixed-length requests cannot benefit from the early-EOS estimate.
|
||||
ratio = 1.0 if req.sampling_params.ignore_eos else self.new_token_ratio
|
||||
return (
|
||||
min(
|
||||
(req.sampling_params.max_new_tokens - len(req.output_ids)),
|
||||
CLIP_MAX_NEW_TOKENS,
|
||||
)
|
||||
* self.new_token_ratio
|
||||
* ratio
|
||||
)
|
||||
|
||||
@property
|
||||
|
||||
Reference in New Issue
Block a user