[Speculative] Support block verification for the DFlash family (#43012)
Co-authored-by: maocheng <mao.cheng@MacBook-Pro-MD663G60VV.local> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
maocheng
Claude Opus 5.5
parent
d1ba943016
commit
e902207a4e
@@ -413,6 +413,14 @@ For stochastic EAGLE, EAGLE3, or NEXTN decoding on NVIDIA CUDA or AMD ROCm, add
|
||||
with `--speculative-eagle-topk 1`. The draft model must produce a full target-vocabulary
|
||||
proposal distribution, as required by the existing rejection sampler.
|
||||
|
||||
The same flag applies to the DFlash family (`DFLASH` and `DSPARK`) on CUDA or ROCm
|
||||
when the draft samples its proposals: DSpark drafts, DFlash checkpoints with a
|
||||
candidate selector, and [LiLiCorr](#lilicorr-reranked-dflash-drafts) drafts with
|
||||
`SGLANG_ENABLE_LILICORR_SAMPLING=1`. These paths already publish their proposal
|
||||
distribution, so `--speculative-use-rejection-sampling` is not needed. A DFlash draft
|
||||
that proposes greedy tokens is a point mass, where block and token-wise verification
|
||||
accept the same prefixes; the server warns that the flag has no effect.
|
||||
|
||||
[Block verification](https://arxiv.org/abs/2403.10444) checks every draft prefix and
|
||||
keeps the longest accepted prefix, including when an earlier prefix was rejected.
|
||||
It uses a corresponding correction distribution to preserve target sampling.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from functools import partial
|
||||
from typing import Optional
|
||||
|
||||
import msgspec
|
||||
@@ -47,6 +48,7 @@ class AcceptSampling:
|
||||
gamma: int,
|
||||
verify_num_draft_tokens: int,
|
||||
cutoff_verify_lens: Optional[torch.Tensor] = None,
|
||||
block_verification: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return accept_sampling(
|
||||
candidates=candidates,
|
||||
@@ -57,6 +59,7 @@ class AcceptSampling:
|
||||
gamma=gamma,
|
||||
verify_num_draft_tokens=verify_num_draft_tokens,
|
||||
cutoff_verify_lens=cutoff_verify_lens,
|
||||
block_verification=block_verification,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -71,6 +74,7 @@ class AcceptSampling:
|
||||
gamma: int,
|
||||
verify_num_draft_tokens: int,
|
||||
cutoff_verify_lens: Optional[torch.Tensor] = None,
|
||||
block_verification: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return accept_sampling_triton(
|
||||
candidates=candidates,
|
||||
@@ -81,6 +85,7 @@ class AcceptSampling:
|
||||
gamma=gamma,
|
||||
verify_num_draft_tokens=verify_num_draft_tokens,
|
||||
cutoff_verify_lens=cutoff_verify_lens,
|
||||
block_verification=block_verification,
|
||||
)
|
||||
|
||||
|
||||
@@ -94,6 +99,7 @@ def _accept_sampling_core(
|
||||
gamma: int,
|
||||
verify_num_draft_tokens: int,
|
||||
cutoff_verify_lens: Optional[torch.Tensor],
|
||||
block_verification: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
bs = candidates.shape[0]
|
||||
device = candidates.device
|
||||
@@ -131,7 +137,11 @@ def _accept_sampling_core(
|
||||
(bs, uniform_width), dtype=torch.float32, device=device
|
||||
)
|
||||
uniform_samples_final = torch.rand((bs,), dtype=torch.float32, device=device)
|
||||
chain_speculative_sampling_triton(
|
||||
sampling_fn = chain_speculative_sampling_triton
|
||||
if block_verification:
|
||||
# Startup validation rejects NPU, whose sampler has no block mode.
|
||||
sampling_fn = partial(sampling_fn, block_verification=True)
|
||||
sampling_fn(
|
||||
predicts=predicts,
|
||||
accept_index=accept_index,
|
||||
accept_token_num=accept_token_num,
|
||||
@@ -167,6 +177,7 @@ def accept_sampling(
|
||||
gamma: int,
|
||||
verify_num_draft_tokens: int,
|
||||
cutoff_verify_lens: Optional[torch.Tensor] = None,
|
||||
block_verification: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
bs = candidates.shape[0]
|
||||
device = candidates.device
|
||||
@@ -179,6 +190,7 @@ def accept_sampling(
|
||||
gamma=gamma,
|
||||
verify_num_draft_tokens=verify_num_draft_tokens,
|
||||
cutoff_verify_lens=cutoff_verify_lens,
|
||||
block_verification=block_verification,
|
||||
)
|
||||
row_ids = torch.arange(bs, dtype=torch.long, device=device)
|
||||
accept_pos = accept_index[row_ids, correct_len.to(torch.long)].to(torch.long)
|
||||
@@ -235,6 +247,7 @@ def accept_sampling_triton(
|
||||
gamma: int,
|
||||
verify_num_draft_tokens: int,
|
||||
cutoff_verify_lens: Optional[torch.Tensor] = None,
|
||||
block_verification: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
correct_len, cap_trim_lens, accept_index, predicts = _accept_sampling_core(
|
||||
candidates=candidates,
|
||||
@@ -245,6 +258,7 @@ def accept_sampling_triton(
|
||||
gamma=gamma,
|
||||
verify_num_draft_tokens=verify_num_draft_tokens,
|
||||
cutoff_verify_lens=cutoff_verify_lens,
|
||||
block_verification=block_verification,
|
||||
)
|
||||
bonus = gather_two_level_bonus_triton(
|
||||
accept_index=accept_index, predicts=predicts, correct_len=correct_len
|
||||
|
||||
@@ -119,8 +119,8 @@ class Spec(msgspec.Struct):
|
||||
] = False
|
||||
speculative_use_block_verification: A[
|
||||
bool,
|
||||
"Use block verification for EAGLE/EAGLE3/NEXTN on CUDA or ROCm "
|
||||
"(requires topk=1).",
|
||||
"Use block verification for EAGLE/EAGLE3/NEXTN (requires topk=1) or for "
|
||||
"sampled DFLASH/DSPARK drafts, on CUDA or ROCm.",
|
||||
] = False
|
||||
speculative_token_map: A[
|
||||
Optional[str],
|
||||
|
||||
@@ -159,16 +159,21 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
|
||||
)
|
||||
|
||||
if cfg.speculative_use_block_verification:
|
||||
if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3"):
|
||||
if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3", "DFLASH", "DSPARK"):
|
||||
raise ValueError(
|
||||
"--speculative-use-block-verification only supports EAGLE / EAGLE3 / NEXTN."
|
||||
"--speculative-use-block-verification only supports EAGLE / EAGLE3 / "
|
||||
"NEXTN / DFLASH / DSPARK."
|
||||
)
|
||||
if cfg.device != "cuda":
|
||||
raise ValueError(
|
||||
"--speculative-use-block-verification only supports CUDA or ROCm."
|
||||
)
|
||||
# Block verification needs sampled proposals and their full distributions.
|
||||
if not cfg.speculative_use_rejection_sampling:
|
||||
# The DFLASH family's sampled drafts always carry them.
|
||||
if (
|
||||
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
|
||||
and not cfg.speculative_use_rejection_sampling
|
||||
):
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"handle_speculative_decoding",
|
||||
|
||||
@@ -467,6 +467,22 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
prefix_gru=prefix_gru,
|
||||
embed_proj=embed_proj,
|
||||
)
|
||||
self._block_verification = get_spec().speculative_use_block_verification
|
||||
if (
|
||||
self._block_verification
|
||||
and (
|
||||
self._is_domino
|
||||
or (self.selector is None and not self._lilicorr_sampling_enabled)
|
||||
)
|
||||
and get_parallel().tp_rank == 0
|
||||
):
|
||||
# A greedy draft is a point mass, where token-wise verification
|
||||
# already accepts every prefix with its target probability.
|
||||
logger.warning(
|
||||
"--speculative-use-block-verification has no effect for this DFLASH "
|
||||
"draft: it proposes greedy tokens, and block verification only "
|
||||
"differs from token-wise verification for sampled proposals."
|
||||
)
|
||||
if get_spec().speculative_num_draft_tokens is None:
|
||||
# Should not happen (ServerArgs should have inferred it), but keep a fallback.
|
||||
self.block_size = int(draft_config.resolve_block_size(default=16))
|
||||
@@ -1484,6 +1500,7 @@ class DFlashWorkerV2(BaseSpecWorker):
|
||||
gamma=gamma,
|
||||
verify_num_draft_tokens=block,
|
||||
cutoff_verify_lens=None,
|
||||
block_verification=self._block_verification,
|
||||
)
|
||||
finally:
|
||||
# Here, not before the next write: candidate_ids may be a view of a
|
||||
|
||||
@@ -118,6 +118,7 @@ class TargetVerifyExecutor:
|
||||
tp_sync: SpecTpSync,
|
||||
verify_epilogue=None,
|
||||
simulate_acc_len: float = 0.0,
|
||||
block_verification: bool = False,
|
||||
) -> None:
|
||||
self.target_worker = target_worker
|
||||
# candidate_max_seq_len_upper_bound only feeds the V4.1 candidate graphs.
|
||||
@@ -137,6 +138,7 @@ class TargetVerifyExecutor:
|
||||
self.verify_epilogue = verify_epilogue
|
||||
self._verify_backend_self_adds_seq_lens_cache: Optional[bool] = None
|
||||
self._simulate_acc_len = float(simulate_acc_len)
|
||||
self._block_verification = block_verification
|
||||
self._simulated_correct_drafts_buf: Optional[torch.Tensor] = None
|
||||
|
||||
def accept_and_finalize(
|
||||
@@ -178,6 +180,7 @@ class TargetVerifyExecutor:
|
||||
verify_num_draft_tokens=self.verify_num_draft_tokens,
|
||||
cutoff_layout=layout,
|
||||
fused_argmax=self._target_is_dsv41,
|
||||
block_verification=self._block_verification,
|
||||
)
|
||||
if simulate:
|
||||
correct_len = self._simulated_correct_len(
|
||||
@@ -870,6 +873,7 @@ def accept_draft_tokens(
|
||||
verify_num_draft_tokens: int,
|
||||
cutoff_layout: Optional[RaggedVerifyLayout] = None,
|
||||
fused_argmax: bool = False,
|
||||
block_verification: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
greedy_mask = draft_block.greedy_mask
|
||||
cutoff_verify_lens = None if cutoff_layout is None else cutoff_layout.verify_lens
|
||||
@@ -899,6 +903,7 @@ def accept_draft_tokens(
|
||||
gamma=gamma,
|
||||
verify_num_draft_tokens=verify_num_draft_tokens,
|
||||
cutoff_verify_lens=cutoff_verify_lens,
|
||||
block_verification=block_verification,
|
||||
)
|
||||
greedy_len, greedy_bonus, greedy_trim = AcceptGreedy.execute(
|
||||
candidates=candidates,
|
||||
@@ -916,6 +921,7 @@ def accept_draft_tokens(
|
||||
gamma=gamma,
|
||||
verify_num_draft_tokens=verify_num_draft_tokens,
|
||||
cutoff_verify_lens=cutoff_verify_lens,
|
||||
block_verification=block_verification,
|
||||
)
|
||||
selected = SelectMixedAccept.execute(
|
||||
greedy_mask=greedy_mask,
|
||||
|
||||
@@ -398,6 +398,7 @@ class DSparkWorkerV2(BaseSpecWorker):
|
||||
tp_sync=self._tp_sync,
|
||||
verify_epilogue=self._verify_epilogue,
|
||||
simulate_acc_len=self._simulate_acc_len,
|
||||
block_verification=get_spec().speculative_use_block_verification,
|
||||
)
|
||||
|
||||
self._forced_budget_frac: Optional[float] = None
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import itertools
|
||||
import os
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -8,6 +9,10 @@ import torch
|
||||
from sglang.kernels.ops.speculative.reject_sampling import (
|
||||
chain_speculative_sampling_triton,
|
||||
)
|
||||
from sglang.srt.speculative.dspark_components.dspark_draft import DraftBlockResult
|
||||
from sglang.srt.speculative.dspark_components.dspark_verify import (
|
||||
accept_draft_tokens,
|
||||
)
|
||||
from sglang.test.ci.ci_register import register_cuda_ci
|
||||
|
||||
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
|
||||
@@ -327,5 +332,54 @@ def test_context_dependent_output_distribution(block, steps):
|
||||
assert abs(observed.item() - expected.item()) < 0.008
|
||||
|
||||
|
||||
@pytest.mark.parametrize("any_greedy", [False, True])
|
||||
def test_dflash_family_accept_uses_block_verification(any_greedy):
|
||||
"""The DFLASH-family accept path must reach the block sampler. Here r_1 = 1/2
|
||||
and r_2 = 1, so block verification always accepts both drafts; token-wise
|
||||
verification rejects the first draft half the time."""
|
||||
batch, vocab, masked = 64, 8, -1e4
|
||||
# Draft proposals: q(0) = 1, then q(1) = q(2) = 1/2.
|
||||
draft_logits = torch.full((batch, 2, vocab), masked, device=DEVICE)
|
||||
draft_logits[:, 0, 0] = 0
|
||||
draft_logits[:, 1, 1:3] = 0
|
||||
# Target: p(0) = p(3) = 1/2, then p(1) = 1, then bonus token 4.
|
||||
target_logits = torch.full((batch, 3, vocab), masked, device=DEVICE)
|
||||
target_logits[:, 0, [0, 3]] = 0
|
||||
target_logits[:, 1, 1] = 0
|
||||
target_logits[:, 2, 4] = 0
|
||||
greedy_mask = torch.zeros(batch, dtype=torch.bool, device=DEVICE)
|
||||
greedy_mask[0] = any_greedy
|
||||
sampled = ~greedy_mask
|
||||
|
||||
def accept(block_verification):
|
||||
return accept_draft_tokens(
|
||||
candidates=torch.tensor([[5, 0, 1]], device=DEVICE).repeat(batch, 1),
|
||||
target_logits=target_logits.view(batch * 3, vocab),
|
||||
draft_block=DraftBlockResult(
|
||||
draft_tokens=torch.tensor([[0, 1]], device=DEVICE).repeat(batch, 1),
|
||||
corrected_logits=draft_logits,
|
||||
greedy_mask=greedy_mask,
|
||||
temperatures=torch.ones(batch, device=DEVICE),
|
||||
),
|
||||
sampling_info=SimpleNamespace(
|
||||
is_all_greedy=False,
|
||||
is_any_greedy=any_greedy,
|
||||
need_top_k_sampling=False,
|
||||
need_top_p_sampling=False,
|
||||
temperatures=torch.ones(batch, 1, device=DEVICE),
|
||||
),
|
||||
draft_input=SimpleNamespace(max_top_k=1, uniform_top_k_value=None),
|
||||
gamma=2,
|
||||
verify_num_draft_tokens=3,
|
||||
block_verification=block_verification,
|
||||
)
|
||||
|
||||
correct_len, bonus, _ = accept(True)
|
||||
assert (correct_len[sampled] == 2).all()
|
||||
assert (bonus[sampled] == 4).all()
|
||||
token_correct_len, _, _ = accept(False)
|
||||
assert (token_correct_len[sampled] == 0).any()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolution_result
|
||||
from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding
|
||||
@@ -48,6 +49,25 @@ class TestBlockVerificationArgs(unittest.TestCase):
|
||||
args.speculative_use_rejection_sampling, use_rejection_sampling
|
||||
)
|
||||
|
||||
def test_dflash_family(self):
|
||||
"""DFLASH/DSPARK sampled drafts publish their own proposal distribution,
|
||||
so block verification must not switch on EAGLE's rejection sampling."""
|
||||
for algorithm in ("DFLASH", "DSPARK"):
|
||||
with self.subTest(algorithm=algorithm):
|
||||
args = self._args(speculative_algorithm=algorithm)
|
||||
# The per-algorithm handlers need a real draft checkpoint.
|
||||
with (
|
||||
mock.patch("sglang.srt.arg_groups.speculative_hook._handle_dflash"),
|
||||
mock.patch("sglang.srt.arg_groups.speculative_hook._handle_dspark"),
|
||||
):
|
||||
handle_speculative_decoding(args)
|
||||
self.assertTrue(
|
||||
resolution_result(args, "speculative_use_block_verification")
|
||||
)
|
||||
self.assertFalse(
|
||||
resolution_result(args, "speculative_use_rejection_sampling")
|
||||
)
|
||||
|
||||
def test_other_verification_modes_unchanged(self):
|
||||
for use_rejection_sampling in (False, True):
|
||||
with self.subTest(use_rejection_sampling=use_rejection_sampling):
|
||||
@@ -69,9 +89,18 @@ class TestBlockVerificationArgs(unittest.TestCase):
|
||||
cases = [
|
||||
({"speculative_algorithm": None}, "only supports EAGLE"),
|
||||
({"speculative_algorithm": "STANDALONE"}, "only supports EAGLE"),
|
||||
({"speculative_algorithm": "DFLASH"}, "only supports EAGLE"),
|
||||
({"speculative_algorithm": "NGRAM"}, "only supports EAGLE"),
|
||||
({"device": "cpu"}, "only supports CUDA or ROCm"),
|
||||
({"device": "npu"}, "only supports CUDA or ROCm"),
|
||||
# DFLASH/DSPARK serve on NPU, whose chain sampler has no block mode.
|
||||
(
|
||||
{"speculative_algorithm": "DFLASH", "device": "npu"},
|
||||
"only supports CUDA or ROCm",
|
||||
),
|
||||
(
|
||||
{"speculative_algorithm": "DSPARK", "device": "npu"},
|
||||
"only supports CUDA or ROCm",
|
||||
),
|
||||
({"speculative_eagle_topk": 2}, "requires --speculative-eagle-topk=1"),
|
||||
({"speculative_accept_threshold_single": 0.5}, "incompatible"),
|
||||
({"speculative_accept_threshold_acc": 0.5}, "incompatible"),
|
||||
|
||||
Reference in New Issue
Block a user