[Scheduler] Use committed length in spec mamba track boundary check (#43029)
Co-authored-by: Ke Bao <ispobaoke@gmail.com> Co-authored-by: Shuwen Wang <47200617+alphabetc1@users.noreply.github.com>
This commit is contained in:
co-authored by
Ke Bao
Shuwen Wang
parent
f9cee8d2b2
commit
0acd93c35c
@@ -1608,7 +1608,15 @@ class SchedulerBatchResultProcessor:
|
||||
return True, committed_len
|
||||
elif result.num_correct_drafts_per_req_cpu is not None:
|
||||
cur = req.seqlen - 1
|
||||
prev = cur - result.num_correct_drafts_per_req_cpu[i] - 1
|
||||
num_non_draft = result.num_non_draft_tokens_per_req
|
||||
retained = result.grammar_retained_tokens
|
||||
if retained is not None and retained[i] is not None:
|
||||
# NOTE(kpham-sgl): grammar-truncated run, minus non-draft
|
||||
# tokens ahead of the tracked verify window (UNO's root).
|
||||
num_committed = max(len(retained[i]) - (num_non_draft - 1), 0)
|
||||
else:
|
||||
num_committed = result.num_correct_drafts_per_req_cpu[i] + 1
|
||||
prev = cur - num_committed
|
||||
if cur // interval != prev // interval:
|
||||
return True, cur // interval * interval
|
||||
|
||||
|
||||
Reference in New Issue
Block a user