Overlap grammar (constrained decoding) with speculative decode verify (#31488)
Co-authored-by: Jason Park <jasonjk@fb.com>
This commit is contained in:
co-authored by
Jason Park
parent
8905cbd42f
commit
e7e8aaa73c
@@ -43,9 +43,19 @@ class _FakeSpecAlgorithm:
|
||||
return False
|
||||
|
||||
|
||||
class _FakeForwardMode:
|
||||
def is_decode(self) -> bool:
|
||||
return True
|
||||
|
||||
def is_extend(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class _FakeBatch:
|
||||
def __init__(self, reqs):
|
||||
self.reqs = reqs
|
||||
self.has_grammar = any(req.grammar is not None for req in reqs)
|
||||
self.forward_mode = _FakeForwardMode()
|
||||
self.spec_algorithm = _FakeSpecAlgorithm()
|
||||
|
||||
|
||||
@@ -95,6 +105,8 @@ def _make_result(num_draft_tokens, accept_lens, flat_tokens):
|
||||
num_correct_drafts_per_req_cpu=None,
|
||||
block_accept_lens=None,
|
||||
cap_lens=None,
|
||||
copy_done=None,
|
||||
grammar_advanced=False,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user