diff --git a/docs/docs/advanced_features/speculative_decoding.mdx b/docs/docs/advanced_features/speculative_decoding.mdx index 727e24e67c..a9b1386a52 100644 --- a/docs/docs/advanced_features/speculative_decoding.mdx +++ b/docs/docs/advanced_features/speculative_decoding.mdx @@ -413,6 +413,14 @@ For stochastic EAGLE, EAGLE3, or NEXTN decoding on NVIDIA CUDA or AMD ROCm, add with `--speculative-eagle-topk 1`. The draft model must produce a full target-vocabulary proposal distribution, as required by the existing rejection sampler. +The same flag applies to the DFlash family (`DFLASH` and `DSPARK`) on CUDA or ROCm +when the draft samples its proposals: DSpark drafts, DFlash checkpoints with a +candidate selector, and [LiLiCorr](#lilicorr-reranked-dflash-drafts) drafts with +`SGLANG_ENABLE_LILICORR_SAMPLING=1`. These paths already publish their proposal +distribution, so `--speculative-use-rejection-sampling` is not needed. A DFlash draft +that proposes greedy tokens is a point mass, where block and token-wise verification +accept the same prefixes; the server warns that the flag has no effect. + [Block verification](https://arxiv.org/abs/2403.10444) checks every draft prefix and keeps the longest accepted prefix, including when an earlier prefix was rejected. It uses a corresponding correction distribution to preserve target sampling. diff --git a/python/sglang/kernels/ops/speculative/dspark/dspark_accept.py b/python/sglang/kernels/ops/speculative/dspark/dspark_accept.py index 0a229f0015..47dc383aa6 100644 --- a/python/sglang/kernels/ops/speculative/dspark/dspark_accept.py +++ b/python/sglang/kernels/ops/speculative/dspark/dspark_accept.py @@ -1,5 +1,6 @@ from __future__ import annotations +from functools import partial from typing import Optional import msgspec @@ -47,6 +48,7 @@ class AcceptSampling: gamma: int, verify_num_draft_tokens: int, cutoff_verify_lens: Optional[torch.Tensor] = None, + block_verification: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: return accept_sampling( candidates=candidates, @@ -57,6 +59,7 @@ class AcceptSampling: gamma=gamma, verify_num_draft_tokens=verify_num_draft_tokens, cutoff_verify_lens=cutoff_verify_lens, + block_verification=block_verification, ) @classmethod @@ -71,6 +74,7 @@ class AcceptSampling: gamma: int, verify_num_draft_tokens: int, cutoff_verify_lens: Optional[torch.Tensor] = None, + block_verification: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: return accept_sampling_triton( candidates=candidates, @@ -81,6 +85,7 @@ class AcceptSampling: gamma=gamma, verify_num_draft_tokens=verify_num_draft_tokens, cutoff_verify_lens=cutoff_verify_lens, + block_verification=block_verification, ) @@ -94,6 +99,7 @@ def _accept_sampling_core( gamma: int, verify_num_draft_tokens: int, cutoff_verify_lens: Optional[torch.Tensor], + block_verification: bool, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: bs = candidates.shape[0] device = candidates.device @@ -131,7 +137,11 @@ def _accept_sampling_core( (bs, uniform_width), dtype=torch.float32, device=device ) uniform_samples_final = torch.rand((bs,), dtype=torch.float32, device=device) - chain_speculative_sampling_triton( + sampling_fn = chain_speculative_sampling_triton + if block_verification: + # Startup validation rejects NPU, whose sampler has no block mode. + sampling_fn = partial(sampling_fn, block_verification=True) + sampling_fn( predicts=predicts, accept_index=accept_index, accept_token_num=accept_token_num, @@ -167,6 +177,7 @@ def accept_sampling( gamma: int, verify_num_draft_tokens: int, cutoff_verify_lens: Optional[torch.Tensor] = None, + block_verification: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: bs = candidates.shape[0] device = candidates.device @@ -179,6 +190,7 @@ def accept_sampling( gamma=gamma, verify_num_draft_tokens=verify_num_draft_tokens, cutoff_verify_lens=cutoff_verify_lens, + block_verification=block_verification, ) row_ids = torch.arange(bs, dtype=torch.long, device=device) accept_pos = accept_index[row_ids, correct_len.to(torch.long)].to(torch.long) @@ -235,6 +247,7 @@ def accept_sampling_triton( gamma: int, verify_num_draft_tokens: int, cutoff_verify_lens: Optional[torch.Tensor] = None, + block_verification: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: correct_len, cap_trim_lens, accept_index, predicts = _accept_sampling_core( candidates=candidates, @@ -245,6 +258,7 @@ def accept_sampling_triton( gamma=gamma, verify_num_draft_tokens=verify_num_draft_tokens, cutoff_verify_lens=cutoff_verify_lens, + block_verification=block_verification, ) bonus = gather_two_level_bonus_triton( accept_index=accept_index, predicts=predicts, correct_len=correct_len diff --git a/python/sglang/srt/arg_groups/fields/spec.py b/python/sglang/srt/arg_groups/fields/spec.py index 199e865ef1..c0344394db 100644 --- a/python/sglang/srt/arg_groups/fields/spec.py +++ b/python/sglang/srt/arg_groups/fields/spec.py @@ -119,8 +119,8 @@ class Spec(msgspec.Struct): ] = False speculative_use_block_verification: A[ bool, - "Use block verification for EAGLE/EAGLE3/NEXTN on CUDA or ROCm " - "(requires topk=1).", + "Use block verification for EAGLE/EAGLE3/NEXTN (requires topk=1) or for " + "sampled DFLASH/DSPARK drafts, on CUDA or ROCm.", ] = False speculative_token_map: A[ Optional[str], diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index 39134de148..9c856e57fc 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -159,16 +159,21 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None: ) if cfg.speculative_use_block_verification: - if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3"): + if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3", "DFLASH", "DSPARK"): raise ValueError( - "--speculative-use-block-verification only supports EAGLE / EAGLE3 / NEXTN." + "--speculative-use-block-verification only supports EAGLE / EAGLE3 / " + "NEXTN / DFLASH / DSPARK." ) if cfg.device != "cuda": raise ValueError( "--speculative-use-block-verification only supports CUDA or ROCm." ) # Block verification needs sampled proposals and their full distributions. - if not cfg.speculative_use_rejection_sampling: + # The DFLASH family's sampled drafts always carry them. + if ( + cfg.speculative_algorithm in ("EAGLE", "EAGLE3") + and not cfg.speculative_use_rejection_sampling + ): declare_resolution( server_args, "handle_speculative_decoding", diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py index 70dc79a1bf..22d8ea809c 100644 --- a/python/sglang/srt/speculative/dflash_worker_v2.py +++ b/python/sglang/srt/speculative/dflash_worker_v2.py @@ -467,6 +467,22 @@ class DFlashWorkerV2(BaseSpecWorker): prefix_gru=prefix_gru, embed_proj=embed_proj, ) + self._block_verification = get_spec().speculative_use_block_verification + if ( + self._block_verification + and ( + self._is_domino + or (self.selector is None and not self._lilicorr_sampling_enabled) + ) + and get_parallel().tp_rank == 0 + ): + # A greedy draft is a point mass, where token-wise verification + # already accepts every prefix with its target probability. + logger.warning( + "--speculative-use-block-verification has no effect for this DFLASH " + "draft: it proposes greedy tokens, and block verification only " + "differs from token-wise verification for sampled proposals." + ) if get_spec().speculative_num_draft_tokens is None: # Should not happen (ServerArgs should have inferred it), but keep a fallback. self.block_size = int(draft_config.resolve_block_size(default=16)) @@ -1484,6 +1500,7 @@ class DFlashWorkerV2(BaseSpecWorker): gamma=gamma, verify_num_draft_tokens=block, cutoff_verify_lens=None, + block_verification=self._block_verification, ) finally: # Here, not before the next write: candidate_ids may be a view of a diff --git a/python/sglang/srt/speculative/dspark_components/dspark_verify.py b/python/sglang/srt/speculative/dspark_components/dspark_verify.py index 44439c377b..3eb1495592 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_verify.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_verify.py @@ -118,6 +118,7 @@ class TargetVerifyExecutor: tp_sync: SpecTpSync, verify_epilogue=None, simulate_acc_len: float = 0.0, + block_verification: bool = False, ) -> None: self.target_worker = target_worker # candidate_max_seq_len_upper_bound only feeds the V4.1 candidate graphs. @@ -137,6 +138,7 @@ class TargetVerifyExecutor: self.verify_epilogue = verify_epilogue self._verify_backend_self_adds_seq_lens_cache: Optional[bool] = None self._simulate_acc_len = float(simulate_acc_len) + self._block_verification = block_verification self._simulated_correct_drafts_buf: Optional[torch.Tensor] = None def accept_and_finalize( @@ -178,6 +180,7 @@ class TargetVerifyExecutor: verify_num_draft_tokens=self.verify_num_draft_tokens, cutoff_layout=layout, fused_argmax=self._target_is_dsv41, + block_verification=self._block_verification, ) if simulate: correct_len = self._simulated_correct_len( @@ -870,6 +873,7 @@ def accept_draft_tokens( verify_num_draft_tokens: int, cutoff_layout: Optional[RaggedVerifyLayout] = None, fused_argmax: bool = False, + block_verification: bool = False, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: greedy_mask = draft_block.greedy_mask cutoff_verify_lens = None if cutoff_layout is None else cutoff_layout.verify_lens @@ -899,6 +903,7 @@ def accept_draft_tokens( gamma=gamma, verify_num_draft_tokens=verify_num_draft_tokens, cutoff_verify_lens=cutoff_verify_lens, + block_verification=block_verification, ) greedy_len, greedy_bonus, greedy_trim = AcceptGreedy.execute( candidates=candidates, @@ -916,6 +921,7 @@ def accept_draft_tokens( gamma=gamma, verify_num_draft_tokens=verify_num_draft_tokens, cutoff_verify_lens=cutoff_verify_lens, + block_verification=block_verification, ) selected = SelectMixedAccept.execute( greedy_mask=greedy_mask, diff --git a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py index 3f24ab7c5b..fda170cf15 100644 --- a/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py +++ b/python/sglang/srt/speculative/dspark_components/dspark_worker_v2.py @@ -398,6 +398,7 @@ class DSparkWorkerV2(BaseSpecWorker): tp_sync=self._tp_sync, verify_epilogue=self._verify_epilogue, simulate_acc_len=self._simulate_acc_len, + block_verification=get_spec().speculative_use_block_verification, ) self._forced_budget_frac: Optional[float] = None diff --git a/test/registered/kernels/ops/speculative/test_block_verification.py b/test/registered/kernels/ops/speculative/test_block_verification.py index f9593b8135..2e69343d7b 100644 --- a/test/registered/kernels/ops/speculative/test_block_verification.py +++ b/test/registered/kernels/ops/speculative/test_block_verification.py @@ -1,6 +1,7 @@ import itertools import os import sys +from types import SimpleNamespace import pytest import torch @@ -8,6 +9,10 @@ import torch from sglang.kernels.ops.speculative.reject_sampling import ( chain_speculative_sampling_triton, ) +from sglang.srt.speculative.dspark_components.dspark_draft import DraftBlockResult +from sglang.srt.speculative.dspark_components.dspark_verify import ( + accept_draft_tokens, +) from sglang.test.ci.ci_register import register_cuda_ci register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") @@ -327,5 +332,54 @@ def test_context_dependent_output_distribution(block, steps): assert abs(observed.item() - expected.item()) < 0.008 +@pytest.mark.parametrize("any_greedy", [False, True]) +def test_dflash_family_accept_uses_block_verification(any_greedy): + """The DFLASH-family accept path must reach the block sampler. Here r_1 = 1/2 + and r_2 = 1, so block verification always accepts both drafts; token-wise + verification rejects the first draft half the time.""" + batch, vocab, masked = 64, 8, -1e4 + # Draft proposals: q(0) = 1, then q(1) = q(2) = 1/2. + draft_logits = torch.full((batch, 2, vocab), masked, device=DEVICE) + draft_logits[:, 0, 0] = 0 + draft_logits[:, 1, 1:3] = 0 + # Target: p(0) = p(3) = 1/2, then p(1) = 1, then bonus token 4. + target_logits = torch.full((batch, 3, vocab), masked, device=DEVICE) + target_logits[:, 0, [0, 3]] = 0 + target_logits[:, 1, 1] = 0 + target_logits[:, 2, 4] = 0 + greedy_mask = torch.zeros(batch, dtype=torch.bool, device=DEVICE) + greedy_mask[0] = any_greedy + sampled = ~greedy_mask + + def accept(block_verification): + return accept_draft_tokens( + candidates=torch.tensor([[5, 0, 1]], device=DEVICE).repeat(batch, 1), + target_logits=target_logits.view(batch * 3, vocab), + draft_block=DraftBlockResult( + draft_tokens=torch.tensor([[0, 1]], device=DEVICE).repeat(batch, 1), + corrected_logits=draft_logits, + greedy_mask=greedy_mask, + temperatures=torch.ones(batch, device=DEVICE), + ), + sampling_info=SimpleNamespace( + is_all_greedy=False, + is_any_greedy=any_greedy, + need_top_k_sampling=False, + need_top_p_sampling=False, + temperatures=torch.ones(batch, 1, device=DEVICE), + ), + draft_input=SimpleNamespace(max_top_k=1, uniform_top_k_value=None), + gamma=2, + verify_num_draft_tokens=3, + block_verification=block_verification, + ) + + correct_len, bonus, _ = accept(True) + assert (correct_len[sampled] == 2).all() + assert (bonus[sampled] == 4).all() + token_correct_len, _, _ = accept(False) + assert (token_correct_len[sampled] == 0).any() + + if __name__ == "__main__": sys.exit(pytest.main([__file__])) diff --git a/test/registered/unit/spec/test_block_verification_args.py b/test/registered/unit/spec/test_block_verification_args.py index dd75779919..c3dc876627 100644 --- a/test/registered/unit/spec/test_block_verification_args.py +++ b/test/registered/unit/spec/test_block_verification_args.py @@ -1,5 +1,6 @@ import unittest from types import SimpleNamespace +from unittest import mock from sglang.srt.arg_groups.overrides import resolution_result from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding @@ -48,6 +49,25 @@ class TestBlockVerificationArgs(unittest.TestCase): args.speculative_use_rejection_sampling, use_rejection_sampling ) + def test_dflash_family(self): + """DFLASH/DSPARK sampled drafts publish their own proposal distribution, + so block verification must not switch on EAGLE's rejection sampling.""" + for algorithm in ("DFLASH", "DSPARK"): + with self.subTest(algorithm=algorithm): + args = self._args(speculative_algorithm=algorithm) + # The per-algorithm handlers need a real draft checkpoint. + with ( + mock.patch("sglang.srt.arg_groups.speculative_hook._handle_dflash"), + mock.patch("sglang.srt.arg_groups.speculative_hook._handle_dspark"), + ): + handle_speculative_decoding(args) + self.assertTrue( + resolution_result(args, "speculative_use_block_verification") + ) + self.assertFalse( + resolution_result(args, "speculative_use_rejection_sampling") + ) + def test_other_verification_modes_unchanged(self): for use_rejection_sampling in (False, True): with self.subTest(use_rejection_sampling=use_rejection_sampling): @@ -69,9 +89,18 @@ class TestBlockVerificationArgs(unittest.TestCase): cases = [ ({"speculative_algorithm": None}, "only supports EAGLE"), ({"speculative_algorithm": "STANDALONE"}, "only supports EAGLE"), - ({"speculative_algorithm": "DFLASH"}, "only supports EAGLE"), + ({"speculative_algorithm": "NGRAM"}, "only supports EAGLE"), ({"device": "cpu"}, "only supports CUDA or ROCm"), ({"device": "npu"}, "only supports CUDA or ROCm"), + # DFLASH/DSPARK serve on NPU, whose chain sampler has no block mode. + ( + {"speculative_algorithm": "DFLASH", "device": "npu"}, + "only supports CUDA or ROCm", + ), + ( + {"speculative_algorithm": "DSPARK", "device": "npu"}, + "only supports CUDA or ROCm", + ), ({"speculative_eagle_topk": 2}, "requires --speculative-eagle-topk=1"), ({"speculative_accept_threshold_single": 0.5}, "incompatible"), ({"speculative_accept_threshold_acc": 0.5}, "incompatible"),