[Speculative] Support block verification for the DFlash family (#43012)

Co-authored-by: maocheng <mao.cheng@MacBook-Pro-MD663G60VV.local>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maocheng23
2026-10-08 13:59:52 -07:00
committed by GitHub
co-authored by maocheng Claude Opus 5.5
parent d1ba943016
commit e902207a4e
9 changed files with 141 additions and 7 deletions
@@ -1,5 +1,6 @@
from __future__ import annotations
from functools import partial
from typing import Optional
import msgspec
@@ -47,6 +48,7 @@ class AcceptSampling:
gamma: int,
verify_num_draft_tokens: int,
cutoff_verify_lens: Optional[torch.Tensor] = None,
block_verification: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return accept_sampling(
candidates=candidates,
@@ -57,6 +59,7 @@ class AcceptSampling:
gamma=gamma,
verify_num_draft_tokens=verify_num_draft_tokens,
cutoff_verify_lens=cutoff_verify_lens,
block_verification=block_verification,
)
@classmethod
@@ -71,6 +74,7 @@ class AcceptSampling:
gamma: int,
verify_num_draft_tokens: int,
cutoff_verify_lens: Optional[torch.Tensor] = None,
block_verification: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
return accept_sampling_triton(
candidates=candidates,
@@ -81,6 +85,7 @@ class AcceptSampling:
gamma=gamma,
verify_num_draft_tokens=verify_num_draft_tokens,
cutoff_verify_lens=cutoff_verify_lens,
block_verification=block_verification,
)
@@ -94,6 +99,7 @@ def _accept_sampling_core(
gamma: int,
verify_num_draft_tokens: int,
cutoff_verify_lens: Optional[torch.Tensor],
block_verification: bool,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
bs = candidates.shape[0]
device = candidates.device
@@ -131,7 +137,11 @@ def _accept_sampling_core(
(bs, uniform_width), dtype=torch.float32, device=device
)
uniform_samples_final = torch.rand((bs,), dtype=torch.float32, device=device)
chain_speculative_sampling_triton(
sampling_fn = chain_speculative_sampling_triton
if block_verification:
# Startup validation rejects NPU, whose sampler has no block mode.
sampling_fn = partial(sampling_fn, block_verification=True)
sampling_fn(
predicts=predicts,
accept_index=accept_index,
accept_token_num=accept_token_num,
@@ -167,6 +177,7 @@ def accept_sampling(
gamma: int,
verify_num_draft_tokens: int,
cutoff_verify_lens: Optional[torch.Tensor] = None,
block_verification: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
bs = candidates.shape[0]
device = candidates.device
@@ -179,6 +190,7 @@ def accept_sampling(
gamma=gamma,
verify_num_draft_tokens=verify_num_draft_tokens,
cutoff_verify_lens=cutoff_verify_lens,
block_verification=block_verification,
)
row_ids = torch.arange(bs, dtype=torch.long, device=device)
accept_pos = accept_index[row_ids, correct_len.to(torch.long)].to(torch.long)
@@ -235,6 +247,7 @@ def accept_sampling_triton(
gamma: int,
verify_num_draft_tokens: int,
cutoff_verify_lens: Optional[torch.Tensor] = None,
block_verification: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
correct_len, cap_trim_lens, accept_index, predicts = _accept_sampling_core(
candidates=candidates,
@@ -245,6 +258,7 @@ def accept_sampling_triton(
gamma=gamma,
verify_num_draft_tokens=verify_num_draft_tokens,
cutoff_verify_lens=cutoff_verify_lens,
block_verification=block_verification,
)
bonus = gather_two_level_bonus_triton(
accept_index=accept_index, predicts=predicts, correct_len=correct_len
+2 -2
View File
@@ -119,8 +119,8 @@ class Spec(msgspec.Struct):
] = False
speculative_use_block_verification: A[
bool,
"Use block verification for EAGLE/EAGLE3/NEXTN on CUDA or ROCm "
"(requires topk=1).",
"Use block verification for EAGLE/EAGLE3/NEXTN (requires topk=1) or for "
"sampled DFLASH/DSPARK drafts, on CUDA or ROCm.",
] = False
speculative_token_map: A[
Optional[str],
@@ -159,16 +159,21 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
)
if cfg.speculative_use_block_verification:
if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3"):
if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3", "DFLASH", "DSPARK"):
raise ValueError(
"--speculative-use-block-verification only supports EAGLE / EAGLE3 / NEXTN."
"--speculative-use-block-verification only supports EAGLE / EAGLE3 / "
"NEXTN / DFLASH / DSPARK."
)
if cfg.device != "cuda":
raise ValueError(
"--speculative-use-block-verification only supports CUDA or ROCm."
)
# Block verification needs sampled proposals and their full distributions.
if not cfg.speculative_use_rejection_sampling:
# The DFLASH family's sampled drafts always carry them.
if (
cfg.speculative_algorithm in ("EAGLE", "EAGLE3")
and not cfg.speculative_use_rejection_sampling
):
declare_resolution(
server_args,
"handle_speculative_decoding",
@@ -467,6 +467,22 @@ class DFlashWorkerV2(BaseSpecWorker):
prefix_gru=prefix_gru,
embed_proj=embed_proj,
)
self._block_verification = get_spec().speculative_use_block_verification
if (
self._block_verification
and (
self._is_domino
or (self.selector is None and not self._lilicorr_sampling_enabled)
)
and get_parallel().tp_rank == 0
):
# A greedy draft is a point mass, where token-wise verification
# already accepts every prefix with its target probability.
logger.warning(
"--speculative-use-block-verification has no effect for this DFLASH "
"draft: it proposes greedy tokens, and block verification only "
"differs from token-wise verification for sampled proposals."
)
if get_spec().speculative_num_draft_tokens is None:
# Should not happen (ServerArgs should have inferred it), but keep a fallback.
self.block_size = int(draft_config.resolve_block_size(default=16))
@@ -1484,6 +1500,7 @@ class DFlashWorkerV2(BaseSpecWorker):
gamma=gamma,
verify_num_draft_tokens=block,
cutoff_verify_lens=None,
block_verification=self._block_verification,
)
finally:
# Here, not before the next write: candidate_ids may be a view of a
@@ -118,6 +118,7 @@ class TargetVerifyExecutor:
tp_sync: SpecTpSync,
verify_epilogue=None,
simulate_acc_len: float = 0.0,
block_verification: bool = False,
) -> None:
self.target_worker = target_worker
# candidate_max_seq_len_upper_bound only feeds the V4.1 candidate graphs.
@@ -137,6 +138,7 @@ class TargetVerifyExecutor:
self.verify_epilogue = verify_epilogue
self._verify_backend_self_adds_seq_lens_cache: Optional[bool] = None
self._simulate_acc_len = float(simulate_acc_len)
self._block_verification = block_verification
self._simulated_correct_drafts_buf: Optional[torch.Tensor] = None
def accept_and_finalize(
@@ -178,6 +180,7 @@ class TargetVerifyExecutor:
verify_num_draft_tokens=self.verify_num_draft_tokens,
cutoff_layout=layout,
fused_argmax=self._target_is_dsv41,
block_verification=self._block_verification,
)
if simulate:
correct_len = self._simulated_correct_len(
@@ -870,6 +873,7 @@ def accept_draft_tokens(
verify_num_draft_tokens: int,
cutoff_layout: Optional[RaggedVerifyLayout] = None,
fused_argmax: bool = False,
block_verification: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
greedy_mask = draft_block.greedy_mask
cutoff_verify_lens = None if cutoff_layout is None else cutoff_layout.verify_lens
@@ -899,6 +903,7 @@ def accept_draft_tokens(
gamma=gamma,
verify_num_draft_tokens=verify_num_draft_tokens,
cutoff_verify_lens=cutoff_verify_lens,
block_verification=block_verification,
)
greedy_len, greedy_bonus, greedy_trim = AcceptGreedy.execute(
candidates=candidates,
@@ -916,6 +921,7 @@ def accept_draft_tokens(
gamma=gamma,
verify_num_draft_tokens=verify_num_draft_tokens,
cutoff_verify_lens=cutoff_verify_lens,
block_verification=block_verification,
)
selected = SelectMixedAccept.execute(
greedy_mask=greedy_mask,
@@ -398,6 +398,7 @@ class DSparkWorkerV2(BaseSpecWorker):
tp_sync=self._tp_sync,
verify_epilogue=self._verify_epilogue,
simulate_acc_len=self._simulate_acc_len,
block_verification=get_spec().speculative_use_block_verification,
)
self._forced_budget_frac: Optional[float] = None