[Speculative] Support block verification for the DFlash family (#43012)

Co-authored-by: maocheng <mao.cheng@MacBook-Pro-MD663G60VV.local>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
maocheng23
2026-10-08 13:59:52 -07:00
committed by GitHub
co-authored by maocheng Claude Opus 5.5
parent d1ba943016
commit e902207a4e
9 changed files with 141 additions and 7 deletions
@@ -1,6 +1,7 @@
import itertools
import os
import sys
from types import SimpleNamespace
import pytest
import torch
@@ -8,6 +9,10 @@ import torch
from sglang.kernels.ops.speculative.reject_sampling import (
chain_speculative_sampling_triton,
)
from sglang.srt.speculative.dspark_components.dspark_draft import DraftBlockResult
from sglang.srt.speculative.dspark_components.dspark_verify import (
accept_draft_tokens,
)
from sglang.test.ci.ci_register import register_cuda_ci
register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large")
@@ -327,5 +332,54 @@ def test_context_dependent_output_distribution(block, steps):
assert abs(observed.item() - expected.item()) < 0.008
@pytest.mark.parametrize("any_greedy", [False, True])
def test_dflash_family_accept_uses_block_verification(any_greedy):
"""The DFLASH-family accept path must reach the block sampler. Here r_1 = 1/2
and r_2 = 1, so block verification always accepts both drafts; token-wise
verification rejects the first draft half the time."""
batch, vocab, masked = 64, 8, -1e4
# Draft proposals: q(0) = 1, then q(1) = q(2) = 1/2.
draft_logits = torch.full((batch, 2, vocab), masked, device=DEVICE)
draft_logits[:, 0, 0] = 0
draft_logits[:, 1, 1:3] = 0
# Target: p(0) = p(3) = 1/2, then p(1) = 1, then bonus token 4.
target_logits = torch.full((batch, 3, vocab), masked, device=DEVICE)
target_logits[:, 0, [0, 3]] = 0
target_logits[:, 1, 1] = 0
target_logits[:, 2, 4] = 0
greedy_mask = torch.zeros(batch, dtype=torch.bool, device=DEVICE)
greedy_mask[0] = any_greedy
sampled = ~greedy_mask
def accept(block_verification):
return accept_draft_tokens(
candidates=torch.tensor([[5, 0, 1]], device=DEVICE).repeat(batch, 1),
target_logits=target_logits.view(batch * 3, vocab),
draft_block=DraftBlockResult(
draft_tokens=torch.tensor([[0, 1]], device=DEVICE).repeat(batch, 1),
corrected_logits=draft_logits,
greedy_mask=greedy_mask,
temperatures=torch.ones(batch, device=DEVICE),
),
sampling_info=SimpleNamespace(
is_all_greedy=False,
is_any_greedy=any_greedy,
need_top_k_sampling=False,
need_top_p_sampling=False,
temperatures=torch.ones(batch, 1, device=DEVICE),
),
draft_input=SimpleNamespace(max_top_k=1, uniform_top_k_value=None),
gamma=2,
verify_num_draft_tokens=3,
block_verification=block_verification,
)
correct_len, bonus, _ = accept(True)
assert (correct_len[sampled] == 2).all()
assert (bonus[sampled] == 4).all()
token_correct_len, _, _ = accept(False)
assert (token_correct_len[sampled] == 0).any()
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))
@@ -1,5 +1,6 @@
import unittest
from types import SimpleNamespace
from unittest import mock
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding
@@ -48,6 +49,25 @@ class TestBlockVerificationArgs(unittest.TestCase):
args.speculative_use_rejection_sampling, use_rejection_sampling
)
def test_dflash_family(self):
"""DFLASH/DSPARK sampled drafts publish their own proposal distribution,
so block verification must not switch on EAGLE's rejection sampling."""
for algorithm in ("DFLASH", "DSPARK"):
with self.subTest(algorithm=algorithm):
args = self._args(speculative_algorithm=algorithm)
# The per-algorithm handlers need a real draft checkpoint.
with (
mock.patch("sglang.srt.arg_groups.speculative_hook._handle_dflash"),
mock.patch("sglang.srt.arg_groups.speculative_hook._handle_dspark"),
):
handle_speculative_decoding(args)
self.assertTrue(
resolution_result(args, "speculative_use_block_verification")
)
self.assertFalse(
resolution_result(args, "speculative_use_rejection_sampling")
)
def test_other_verification_modes_unchanged(self):
for use_rejection_sampling in (False, True):
with self.subTest(use_rejection_sampling=use_rejection_sampling):
@@ -69,9 +89,18 @@ class TestBlockVerificationArgs(unittest.TestCase):
cases = [
({"speculative_algorithm": None}, "only supports EAGLE"),
({"speculative_algorithm": "STANDALONE"}, "only supports EAGLE"),
({"speculative_algorithm": "DFLASH"}, "only supports EAGLE"),
({"speculative_algorithm": "NGRAM"}, "only supports EAGLE"),
({"device": "cpu"}, "only supports CUDA or ROCm"),
({"device": "npu"}, "only supports CUDA or ROCm"),
# DFLASH/DSPARK serve on NPU, whose chain sampler has no block mode.
(
{"speculative_algorithm": "DFLASH", "device": "npu"},
"only supports CUDA or ROCm",
),
(
{"speculative_algorithm": "DSPARK", "device": "npu"},
"only supports CUDA or ROCm",
),
({"speculative_eagle_topk": 2}, "requires --speculative-eagle-topk=1"),
({"speculative_accept_threshold_single": 0.5}, "incompatible"),
({"speculative_accept_threshold_acc": 0.5}, "incompatible"),