[PD][Spec] Give a PD decode's first EAGLE draft step a proposal distribution (#41783)

Co-authored-by: Shangming Cai <csmthu@gmail.com>
This commit is contained in:
Sasha Sidorov
2026-10-12 18:49:52 +08:00
committed by GitHub
co-authored by Shangming Cai
parent 8afe107c24
commit 7953751bfa
2 changed files with 148 additions and 0 deletions
@@ -22,6 +22,30 @@ def _requires_dsa_seed_for_cuda_graph(hf_config, topk: int) -> bool:
return topk == 1 and get_dsa_seed_metadata_dim(hf_config) > 0
def pd_first_draft_proposal(topk_index: torch.Tensor, vocab_size: int) -> torch.Tensor:
"""Proposal distribution for a PD decode's first verify of a request.
Rejection sampling verifies each draft token X against the distribution q
it was drawn from. PD prefill hands decode the drafted tokens but not q,
so decode proposes each received X with probability one. The verify stays
exact: it accepts X with probability p(X) and otherwise resamples from p
with X removed, so the committed token is distributed as p, the verify's
target distribution. A greedy request's p is one-hot, so it accepts X
exactly when X is the target argmax, as it would with q. A sampled one
accepts X with probability p(X) instead of min(1, p(X) / q(X)).
``topk_index`` is (b, n) under top-k 1: n is 1 for single-layer EAGLE and
the number of draft steps for multi-layer EAGLE. Returns (b, n, vocab).
"""
probs = torch.zeros(
(*topk_index.shape, vocab_size),
dtype=torch.float32,
device=topk_index.device,
)
probs.scatter_(-1, topk_index.unsqueeze(-1), 1.0)
return probs
def build_eagle_disagg_draft_input(
batch: ScheduleBatch,
last_tokens_tensor: torch.Tensor,
@@ -97,9 +121,18 @@ def build_eagle_disagg_draft_input(
spec.speculative_eagle_topk,
)
draft_probs = None
if spec.speculative_use_rejection_sampling:
# Single-layer verifies one (b, vocab) proposal; multi-layer verifies
# the received (b, num_steps, vocab) chain directly.
draft_probs = pd_first_draft_proposal(topk_index, batch.model_config.vocab_size)
if not spec.enable_multi_layer_eagle:
draft_probs = draft_probs[:, 0]
spec_info = EagleDraftInput(
topk_p=topk_p,
topk_index=topk_index,
draft_probs=draft_probs,
hidden_states=hidden_states,
bonus_tokens=last_tokens_tensor,
dsa_topk_indices=dsa_topk_indices,
@@ -0,0 +1,115 @@
"""A PD decode's first EAGLE draft input carries a rejection-sampling proposal.
PD prefill sends the first draft token, its probability and hidden state, but
not the distribution the token was drawn from. With rejection sampling on
(the ROCm default for EAGLE top-k 1), the decode's first draft step stacks
that missing proposal with the steps it drafts itself.
"""
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.speculative import eagle_disaggregation
from sglang.srt.speculative.eagle_disaggregation import (
build_eagle_disagg_draft_input,
pd_first_draft_proposal,
)
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
VOCAB = 11
def _spec(rejection_sampling=True, multi_layer=False):
return SimpleNamespace(
speculative_eagle_topk=1,
speculative_num_steps=3,
enable_multi_layer_eagle=multi_layer,
speculative_use_rejection_sampling=rejection_sampling,
)
def _batch(tokens):
# One entry per request: a token, or the token chain of multi-layer EAGLE.
chains = [list(t) if isinstance(t, tuple) else [t] for t in tokens]
reqs = [
SimpleNamespace(
output_topk_p=[0.25] * len(chain),
output_topk_index=chain,
hidden_states_tensor=torch.full((4,), float(i)),
output_dsa_topk_indices=None,
)
for i, chain in enumerate(chains)
]
return SimpleNamespace(
reqs=reqs,
device="cpu",
enable_overlap=False,
# A model without DSA seed metadata: no MTP index sharing.
model_config=SimpleNamespace(vocab_size=VOCAB, hf_config=SimpleNamespace()),
)
class TestPdEagleDraftProposal(CustomTestCase):
def build(self, spec, tokens=(3, 7)):
with patch.object(eagle_disaggregation, "get_spec", return_value=spec):
return build_eagle_disagg_draft_input(
_batch(tokens), torch.tensor([1, 2]), future_map=None
)
def test_rejection_sampling_gets_one_hot_proposal_at_drafted_token(self):
spec_info = self.build(_spec())
expected = torch.zeros((2, VOCAB))
expected[0, 3] = expected[1, 7] = 1.0
self.assertEqual(spec_info.draft_probs.dtype, torch.float32)
torch.testing.assert_close(spec_info.draft_probs, expected)
torch.testing.assert_close(spec_info.topk_index, torch.tensor([[3], [7]]))
def test_proposal_matches_target_vocab_width(self):
# eagle_sample refuses a proposal whose width differs from the target's.
spec_info = self.build(_spec())
self.assertEqual(spec_info.draft_probs.shape[-1], VOCAB)
def test_no_proposal_without_rejection_sampling(self):
self.assertIsNone(self.build(_spec(rejection_sampling=False)).draft_probs)
self.assertIsNone(
self.build(_spec(rejection_sampling=False, multi_layer=True)).draft_probs
)
def test_multi_layer_gets_one_hot_proposal_per_chain_step(self):
# Multi-layer EAGLE verifies the received chain with no draft forward,
# so it needs a (b, num_steps, vocab) proposal; None fails the verify.
chains = ((3, 5, 9), (7, 0, 10))
spec_info = self.build(_spec(multi_layer=True), tokens=chains)
expected = torch.zeros((2, 3, VOCAB))
for row, chain in enumerate(chains):
for step, token in enumerate(chain):
expected[row, step, token] = 1.0
self.assertEqual(spec_info.draft_probs.dtype, torch.float32)
torch.testing.assert_close(spec_info.draft_probs, expected)
torch.testing.assert_close(spec_info.topk_index, torch.tensor(chains))
def test_one_hot_proposal_keeps_verify_exact(self):
# Chain verify of one token: accept X if coin * q(X) < p(X), otherwise
# resample from (p - q)+. Enumerating X ~ q_draft and both outcomes, the
# committed token's distribution must equal p for any drafting q.
p = torch.tensor([0.5, 0.3, 0.2])
q_draft = torch.tensor([0.1, 0.6, 0.3])
committed = torch.zeros(3)
for x in range(3):
q = pd_first_draft_proposal(torch.tensor([[x]]), 3)[0, 0]
accept = torch.clamp(p[x] / q[x], max=1.0)
residual = torch.clamp(p - q, min=0.0)
residual = residual / residual.sum()
committed[x] += q_draft[x] * accept
committed += q_draft[x] * (1 - accept) * residual
torch.testing.assert_close(committed, p)
if __name__ == "__main__":
unittest.main()