From 7953751bfa83239e1dad6dd3a45864db8ce28021 Mon Sep 17 00:00:00 2001 From: Sasha Sidorov Date: Mon, 12 Oct 2026 03:49:52 -0700 Subject: [PATCH] [PD][Spec] Give a PD decode's first EAGLE draft step a proposal distribution (#41783) Co-authored-by: Shangming Cai --- .../srt/speculative/eagle_disaggregation.py | 33 +++++ .../test_pd_eagle_draft_proposal.py | 115 ++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 test/registered/unit/speculative/test_pd_eagle_draft_proposal.py diff --git a/python/sglang/srt/speculative/eagle_disaggregation.py b/python/sglang/srt/speculative/eagle_disaggregation.py index 37335b21cd..b8541c5fc2 100644 --- a/python/sglang/srt/speculative/eagle_disaggregation.py +++ b/python/sglang/srt/speculative/eagle_disaggregation.py @@ -22,6 +22,30 @@ def _requires_dsa_seed_for_cuda_graph(hf_config, topk: int) -> bool: return topk == 1 and get_dsa_seed_metadata_dim(hf_config) > 0 +def pd_first_draft_proposal(topk_index: torch.Tensor, vocab_size: int) -> torch.Tensor: + """Proposal distribution for a PD decode's first verify of a request. + + Rejection sampling verifies each draft token X against the distribution q + it was drawn from. PD prefill hands decode the drafted tokens but not q, + so decode proposes each received X with probability one. The verify stays + exact: it accepts X with probability p(X) and otherwise resamples from p + with X removed, so the committed token is distributed as p, the verify's + target distribution. A greedy request's p is one-hot, so it accepts X + exactly when X is the target argmax, as it would with q. A sampled one + accepts X with probability p(X) instead of min(1, p(X) / q(X)). + + ``topk_index`` is (b, n) under top-k 1: n is 1 for single-layer EAGLE and + the number of draft steps for multi-layer EAGLE. Returns (b, n, vocab). + """ + probs = torch.zeros( + (*topk_index.shape, vocab_size), + dtype=torch.float32, + device=topk_index.device, + ) + probs.scatter_(-1, topk_index.unsqueeze(-1), 1.0) + return probs + + def build_eagle_disagg_draft_input( batch: ScheduleBatch, last_tokens_tensor: torch.Tensor, @@ -97,9 +121,18 @@ def build_eagle_disagg_draft_input( spec.speculative_eagle_topk, ) + draft_probs = None + if spec.speculative_use_rejection_sampling: + # Single-layer verifies one (b, vocab) proposal; multi-layer verifies + # the received (b, num_steps, vocab) chain directly. + draft_probs = pd_first_draft_proposal(topk_index, batch.model_config.vocab_size) + if not spec.enable_multi_layer_eagle: + draft_probs = draft_probs[:, 0] + spec_info = EagleDraftInput( topk_p=topk_p, topk_index=topk_index, + draft_probs=draft_probs, hidden_states=hidden_states, bonus_tokens=last_tokens_tensor, dsa_topk_indices=dsa_topk_indices, diff --git a/test/registered/unit/speculative/test_pd_eagle_draft_proposal.py b/test/registered/unit/speculative/test_pd_eagle_draft_proposal.py new file mode 100644 index 0000000000..07f1aa343c --- /dev/null +++ b/test/registered/unit/speculative/test_pd_eagle_draft_proposal.py @@ -0,0 +1,115 @@ +"""A PD decode's first EAGLE draft input carries a rejection-sampling proposal. + +PD prefill sends the first draft token, its probability and hidden state, but +not the distribution the token was drawn from. With rejection sampling on +(the ROCm default for EAGLE top-k 1), the decode's first draft step stacks +that missing proposal with the steps it drafts itself. +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.speculative import eagle_disaggregation +from sglang.srt.speculative.eagle_disaggregation import ( + build_eagle_disagg_draft_input, + pd_first_draft_proposal, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + +VOCAB = 11 + + +def _spec(rejection_sampling=True, multi_layer=False): + return SimpleNamespace( + speculative_eagle_topk=1, + speculative_num_steps=3, + enable_multi_layer_eagle=multi_layer, + speculative_use_rejection_sampling=rejection_sampling, + ) + + +def _batch(tokens): + # One entry per request: a token, or the token chain of multi-layer EAGLE. + chains = [list(t) if isinstance(t, tuple) else [t] for t in tokens] + reqs = [ + SimpleNamespace( + output_topk_p=[0.25] * len(chain), + output_topk_index=chain, + hidden_states_tensor=torch.full((4,), float(i)), + output_dsa_topk_indices=None, + ) + for i, chain in enumerate(chains) + ] + return SimpleNamespace( + reqs=reqs, + device="cpu", + enable_overlap=False, + # A model without DSA seed metadata: no MTP index sharing. + model_config=SimpleNamespace(vocab_size=VOCAB, hf_config=SimpleNamespace()), + ) + + +class TestPdEagleDraftProposal(CustomTestCase): + def build(self, spec, tokens=(3, 7)): + with patch.object(eagle_disaggregation, "get_spec", return_value=spec): + return build_eagle_disagg_draft_input( + _batch(tokens), torch.tensor([1, 2]), future_map=None + ) + + def test_rejection_sampling_gets_one_hot_proposal_at_drafted_token(self): + spec_info = self.build(_spec()) + expected = torch.zeros((2, VOCAB)) + expected[0, 3] = expected[1, 7] = 1.0 + self.assertEqual(spec_info.draft_probs.dtype, torch.float32) + torch.testing.assert_close(spec_info.draft_probs, expected) + torch.testing.assert_close(spec_info.topk_index, torch.tensor([[3], [7]])) + + def test_proposal_matches_target_vocab_width(self): + # eagle_sample refuses a proposal whose width differs from the target's. + spec_info = self.build(_spec()) + self.assertEqual(spec_info.draft_probs.shape[-1], VOCAB) + + def test_no_proposal_without_rejection_sampling(self): + self.assertIsNone(self.build(_spec(rejection_sampling=False)).draft_probs) + self.assertIsNone( + self.build(_spec(rejection_sampling=False, multi_layer=True)).draft_probs + ) + + def test_multi_layer_gets_one_hot_proposal_per_chain_step(self): + # Multi-layer EAGLE verifies the received chain with no draft forward, + # so it needs a (b, num_steps, vocab) proposal; None fails the verify. + chains = ((3, 5, 9), (7, 0, 10)) + spec_info = self.build(_spec(multi_layer=True), tokens=chains) + expected = torch.zeros((2, 3, VOCAB)) + for row, chain in enumerate(chains): + for step, token in enumerate(chain): + expected[row, step, token] = 1.0 + self.assertEqual(spec_info.draft_probs.dtype, torch.float32) + torch.testing.assert_close(spec_info.draft_probs, expected) + torch.testing.assert_close(spec_info.topk_index, torch.tensor(chains)) + + def test_one_hot_proposal_keeps_verify_exact(self): + # Chain verify of one token: accept X if coin * q(X) < p(X), otherwise + # resample from (p - q)+. Enumerating X ~ q_draft and both outcomes, the + # committed token's distribution must equal p for any drafting q. + p = torch.tensor([0.5, 0.3, 0.2]) + q_draft = torch.tensor([0.1, 0.6, 0.3]) + committed = torch.zeros(3) + for x in range(3): + q = pd_first_draft_proposal(torch.tensor([[x]]), 3)[0, 0] + accept = torch.clamp(p[x] / q[x], max=1.0) + residual = torch.clamp(p - q, min=0.0) + residual = residual / residual.sum() + committed[x] += q_draft[x] * accept + committed += q_draft[x] * (1 - accept) * residual + torch.testing.assert_close(committed, p) + + +if __name__ == "__main__": + unittest.main()