Fix stop boundaries for grammar-constrained speculative decoding (#31738)

This commit is contained in:
Yang Liu
2026-07-20 17:55:27 -07:00
committed by GitHub
parent 4682ded472
commit 9462c303a5
2 changed files with 73 additions and 5 deletions
+4 -5
View File
@@ -1489,11 +1489,6 @@ class Req(ReqDllmMixin):
self.finished_len = self.sampling_params.max_new_tokens
return
if self.grammar is not None:
if self.grammar.is_terminated():
self.finished_reason = FINISH_MATCHED_TOKEN(matched=self.output_ids[-1])
return
new_accepted_tokens = self.output_ids[-new_accepted_len:]
# Sanitize out-of-range / NaN token ids before any decode.
@@ -1509,6 +1504,10 @@ class Req(ReqDllmMixin):
if self._check_token_based_finish(new_accepted_tokens):
return
if self.grammar is not None and self.grammar.is_terminated():
self.finished_reason = FINISH_MATCHED_TOKEN(matched=self.output_ids[-1])
return
def reset_for_retract(self):
# Increment retraction count before resetting other state. We should not reset this
# since we are tracking the total number of retractions for each request.