minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 13 Packages Projects Releases Wiki Activity

fix: stop-string check misses early matches during speculative decoding (#23802)

Browse Source 
Co-authored-by: xythink <xythink@users.noreply.github.com>
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
Co-authored-by: Liangsheng Yin <hnyls2002@gmail.com>
This commit is contained in:
Yi Xie
2026-06-08 23:25:58 -07:00
committed by GitHub
co-authored by xythink hnyls2002 Liangsheng Yin
parent 991689fd0d
commit d145a6127a
2 changed files with 70 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/managers/schedule_batch.py
+10 -5
View File
@@ -1192,7 +1192,7 @@ class Req(ReqDllmMixin):
return self.surr_and_decode_ids, self.read_offset - self.surr_offset
def tail_str(self) -> str:
def tail_str(self, new_accepted_len: int = 1) -> str:
# Check stop strings and stop regex patterns together
if (
len(self.sampling_params.stop_strs) == 0
@@ -1205,7 +1205,12 @@ class Req(ReqDllmMixin):
self.sampling_params.stop_regex_max_len + 1,
)
tail_len = min(max_len_tail_str, len(self.output_ids))
# Spec decode accepts multiple tokens per step; widen the window to cover
# the whole accepted chunk so a stop string landing mid-chunk (with more
# tokens accepted after it) is not pushed out of view.
tail_len = min(
max_len_tail_str + max(new_accepted_len - 1, 0), len(self.output_ids)
)
return self.tokenizer.decode(self.output_ids[-tail_len:])
def check_match_stop_str_prefix(self) -> bool:
@@ -1261,12 +1266,12 @@ class Req(ReqDllmMixin):
return False
def _check_str_based_finish(self):
def _check_str_based_finish(self, new_accepted_len: int = 1):
if (
len(self.sampling_params.stop_strs) > 0
or len(self.sampling_params.stop_regex_strs) > 0
):
tail_str = self.tail_str()
tail_str = self.tail_str(new_accepted_len)
# Check stop strings
if len(self.sampling_params.stop_strs) > 0:
@@ -1331,7 +1336,7 @@ class Req(ReqDllmMixin):
if self._check_vocab_boundary_finish(new_accepted_tokens):
return
if self._check_str_based_finish():
if self._check_str_based_finish(new_accepted_len):
return
def reset_for_retract(self):