[PD][Spec] Give a PD decode's first EAGLE draft step a proposal distribution (#41783)

Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Sasha Sidorov
2026-10-12 18:49:52 +08:00
committed by GitHub
co-authored by Shangming Cai
parent 8afe107c24
commit 7953751bfa
2 changed files with 148 additions and 0 deletions
@@ -22,6 +22,30 @@ def _requires_dsa_seed_for_cuda_graph(hf_config, topk: int) -> bool:
return topk == 1 and get_dsa_seed_metadata_dim(hf_config) > 0
def pd_first_draft_proposal(topk_index: torch.Tensor, vocab_size: int) -> torch.Tensor:
"""Proposal distribution for a PD decode's first verify of a request.
Rejection sampling verifies each draft token X against the distribution q
it was drawn from. PD prefill hands decode the drafted tokens but not q,
so decode proposes each received X with probability one. The verify stays
exact: it accepts X with probability p(X) and otherwise resamples from p
with X removed, so the committed token is distributed as p, the verify's
target distribution. A greedy request's p is one-hot, so it accepts X
exactly when X is the target argmax, as it would with q. A sampled one
accepts X with probability p(X) instead of min(1, p(X) / q(X)).
``topk_index`` is (b, n) under top-k 1: n is 1 for single-layer EAGLE and
the number of draft steps for multi-layer EAGLE. Returns (b, n, vocab).
"""
probs = torch.zeros(
(*topk_index.shape, vocab_size),
dtype=torch.float32,
device=topk_index.device,
)
probs.scatter_(-1, topk_index.unsqueeze(-1), 1.0)
return probs
def build_eagle_disagg_draft_input(
batch: ScheduleBatch,
last_tokens_tensor: torch.Tensor,
@@ -97,9 +121,18 @@ def build_eagle_disagg_draft_input(
spec.speculative_eagle_topk,
)
draft_probs = None
if spec.speculative_use_rejection_sampling:
# Single-layer verifies one (b, vocab) proposal; multi-layer verifies
# the received (b, num_steps, vocab) chain directly.
draft_probs = pd_first_draft_proposal(topk_index, batch.model_config.vocab_size)
if not spec.enable_multi_layer_eagle:
draft_probs = draft_probs[:, 0]
spec_info = EagleDraftInput(
topk_p=topk_p,
topk_index=topk_index,
draft_probs=draft_probs,
hidden_states=hidden_states,
bonus_tokens=last_tokens_tensor,
dsa_topk_indices=dsa_topk_indices,