FrozenKVMTPVerifyInput: add _draft_preprocess_idle call for when all requests in the verify batch finish in the same iteration (#26859)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Harmya Bhatt <harmyacs@gmail.com>
This commit is contained in:
Adam Kelch
2026-06-04 21:47:32 -07:00
committed by GitHub
co-authored by github-actions[bot] Harmya Bhatt
parent aed0808e18
commit 6cbc035dc9
2 changed files with 152 additions and 0 deletions
@@ -352,6 +352,15 @@ class FrozenKVMTPWorker(TpModelWorker):
) -> None:
capture_for_decode(logits_output, draft_input, self.topk)
def _draft_preprocess_idle(self, batch: ScheduleBatch) -> None:
batch.spec_info = FrozenKVMTPDraftInput.create_idle_input(
device=self.device,
hidden_size=self._recurrent_hidden_size,
dtype=self.model_config.dtype,
topk=self.topk,
capture_hidden_mode=CaptureHiddenMode.LAST,
)
def _run_assistant_seed_step(
self,
batch: ScheduleBatch,
@@ -457,6 +466,13 @@ class FrozenKVMTPWorker(TpModelWorker):
# `FrozenKVMTPDraftInput` for next iter.
batch.spec_info = draft_extend_input
self.forward_draft_extend_after_decode(batch)
else:
# All reqs finished and dp_attention isn't forcing extend.
# Install an idle FrozenKVMTPDraftInput so next iter's scheduler
# ops (merge_batch / filter_batch) see well-typed empty
# tensors instead of None.
self._draft_preprocess_idle(batch)
set_time_batch(batch.reqs, "set_spec_draft_extend_end_time", trace_only=True)
return GenerationBatchResult(