[Spec] Consolidate the grammar sync decision into ScheduleBatch.grammar_needs_sync (#32353)
This commit is contained in:
@@ -2042,6 +2042,11 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
def is_dllm(self):
|
||||
return self.dllm_config is not None
|
||||
|
||||
def grammar_needs_sync(self) -> bool:
|
||||
"""Whether grammar forces this batch onto the synchronous path, i.e. the
|
||||
previous batch's result is resolved before this forward."""
|
||||
return self.has_grammar and not self.spec_algorithm.supports_grammar_overlap()
|
||||
|
||||
def prepare_encoder_info_extend(
|
||||
self, input_ids: List[array[int]], seq_lens: List[int]
|
||||
):
|
||||
|
||||
@@ -1672,14 +1672,12 @@ class Scheduler(
|
||||
and last_batch_is_extend
|
||||
)
|
||||
|
||||
# Spec algorithms that don't advance the grammar FSM inside verify() (see
|
||||
# supports_grammar_overlap) still need overlap forced off for grammar decode
|
||||
# batches, so the FSM is advanced before the next batch's bitmask.
|
||||
# Sync so the FSM advance lands before the next batch's bitmask. Permanent
|
||||
# path for host-draft algorithms, not a pending migration.
|
||||
need_grammar_sync = (
|
||||
batch
|
||||
and not batch.spec_algorithm.is_none()
|
||||
and not batch.spec_algorithm.supports_grammar_overlap()
|
||||
and batch.has_grammar
|
||||
and batch.grammar_needs_sync()
|
||||
and batch.forward_mode.is_decode()
|
||||
and len(self.result_queue) > 0
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user