[Spec] Publish the final multi-layer EAGLE shared-read event (#36752)

Co-authored-by: Qiaolin-Yu <liin1211@outlook.com>
This commit is contained in:
paulzhang-tm
2026-09-02 15:57:50 -07:00
committed by GitHub
co-authored by Qiaolin-Yu
parent 19c30dff56
commit 3fa6b86504
3 changed files with 185 additions and 3 deletions
@@ -812,6 +812,7 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
out = runner.replay(
self.bs, self.seq_lens_sum, self._replay_spec_info, self.seq_lens_cpu
)
self._publish_shared_read_done(step)
raw_bs = self.raw_bs
raw_num_tokens = self.raw_num_tokens
num_logit_rows = raw_bs if self.prune_draft_extend_logits else raw_num_tokens
@@ -829,6 +830,14 @@ class MultiLayerEagleMultiStepDraftExtendCudaGraphRunner:
out.topk_index[:raw_bs],
)
def _publish_shared_read_done(self, step: int) -> None:
if step != self.speculative_num_steps - 1:
return
runner = self.runners[step]
read_done = runner.device_module.Event()
read_done.record()
runner.model_runner.shared_read_done_event = read_done
def clone_draft_probs(self) -> torch.Tensor:
"""Materialize the in-graph-written proposal q [raw_bs, num_steps, vocab]
after replay; the clone must land before a later replay rewrites the
@@ -989,4 +998,6 @@ class OneGraphMultiLayerEagleMultiStepDraftExtendCudaGraphRunner(
out.topk_p[:raw_bs],
out.topk_index[:raw_bs],
)
return self._cached[step]
result = self._cached[step]
self._publish_shared_read_done(step)
return result
@@ -962,8 +962,7 @@ class MultiLayerEagleWorkerV2(BaseSpecWorker):
@property
def last_shared_read_runner(self):
# Multi-layer eagle has no draft forward, only draft extend.
return self._draft_worker.draft_runner
return self._draft_worker.draft_runner_list[-1]
@property
def spec_v2_attn_backends(self) -> tuple: