[PD][Spec] Give a PD decode's first EAGLE draft step a proposal distribution (#41783)
Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
co-authored by
Shangming Cai
parent
8afe107c24
commit
7953751bfa
@@ -22,6 +22,30 @@ def _requires_dsa_seed_for_cuda_graph(hf_config, topk: int) -> bool:
|
||||
return topk == 1 and get_dsa_seed_metadata_dim(hf_config) > 0
|
||||
|
||||
|
||||
def pd_first_draft_proposal(topk_index: torch.Tensor, vocab_size: int) -> torch.Tensor:
|
||||
"""Proposal distribution for a PD decode's first verify of a request.
|
||||
|
||||
Rejection sampling verifies each draft token X against the distribution q
|
||||
it was drawn from. PD prefill hands decode the drafted tokens but not q,
|
||||
so decode proposes each received X with probability one. The verify stays
|
||||
exact: it accepts X with probability p(X) and otherwise resamples from p
|
||||
with X removed, so the committed token is distributed as p, the verify's
|
||||
target distribution. A greedy request's p is one-hot, so it accepts X
|
||||
exactly when X is the target argmax, as it would with q. A sampled one
|
||||
accepts X with probability p(X) instead of min(1, p(X) / q(X)).
|
||||
|
||||
``topk_index`` is (b, n) under top-k 1: n is 1 for single-layer EAGLE and
|
||||
the number of draft steps for multi-layer EAGLE. Returns (b, n, vocab).
|
||||
"""
|
||||
probs = torch.zeros(
|
||||
(*topk_index.shape, vocab_size),
|
||||
dtype=torch.float32,
|
||||
device=topk_index.device,
|
||||
)
|
||||
probs.scatter_(-1, topk_index.unsqueeze(-1), 1.0)
|
||||
return probs
|
||||
|
||||
|
||||
def build_eagle_disagg_draft_input(
|
||||
batch: ScheduleBatch,
|
||||
last_tokens_tensor: torch.Tensor,
|
||||
@@ -97,9 +121,18 @@ def build_eagle_disagg_draft_input(
|
||||
spec.speculative_eagle_topk,
|
||||
)
|
||||
|
||||
draft_probs = None
|
||||
if spec.speculative_use_rejection_sampling:
|
||||
# Single-layer verifies one (b, vocab) proposal; multi-layer verifies
|
||||
# the received (b, num_steps, vocab) chain directly.
|
||||
draft_probs = pd_first_draft_proposal(topk_index, batch.model_config.vocab_size)
|
||||
if not spec.enable_multi_layer_eagle:
|
||||
draft_probs = draft_probs[:, 0]
|
||||
|
||||
spec_info = EagleDraftInput(
|
||||
topk_p=topk_p,
|
||||
topk_index=topk_index,
|
||||
draft_probs=draft_probs,
|
||||
hidden_states=hidden_states,
|
||||
bonus_tokens=last_tokens_tensor,
|
||||
dsa_topk_indices=dsa_topk_indices,
|
||||
|
||||
Reference in New Issue
Block a user