Fix stop boundaries for grammar-constrained speculative decoding (#31738)
This commit is contained in:
@@ -1489,11 +1489,6 @@ class Req(ReqDllmMixin):
|
||||
self.finished_len = self.sampling_params.max_new_tokens
|
||||
return
|
||||
|
||||
if self.grammar is not None:
|
||||
if self.grammar.is_terminated():
|
||||
self.finished_reason = FINISH_MATCHED_TOKEN(matched=self.output_ids[-1])
|
||||
return
|
||||
|
||||
new_accepted_tokens = self.output_ids[-new_accepted_len:]
|
||||
|
||||
# Sanitize out-of-range / NaN token ids before any decode.
|
||||
@@ -1509,6 +1504,10 @@ class Req(ReqDllmMixin):
|
||||
if self._check_token_based_finish(new_accepted_tokens):
|
||||
return
|
||||
|
||||
if self.grammar is not None and self.grammar.is_terminated():
|
||||
self.finished_reason = FINISH_MATCHED_TOKEN(matched=self.output_ids[-1])
|
||||
return
|
||||
|
||||
def reset_for_retract(self):
|
||||
# Increment retraction count before resetting other state. We should not reset this
|
||||
# since we are tracking the total number of retractions for each request.
|
||||
|
||||
Reference in New Issue
Block a user