diff --git a/benchmark/kernels/bench_block_verification.py b/benchmark/kernels/bench_block_verification.py new file mode 100644 index 0000000000..2c05eeec86 --- /dev/null +++ b/benchmark/kernels/bench_block_verification.py @@ -0,0 +1,124 @@ +"""Compare token and block samplers on precomputed FP32 probability tensors.""" + +import argparse +import importlib.util +import json +import statistics +import sys +from pathlib import Path + +import torch +import triton + +from sglang.kernels.ops.speculative.reject_sampling import ( + chain_speculative_sampling_triton, +) + + +def benchmark(args: argparse.Namespace) -> None: + samplers = [("current", chain_speculative_sampling_triton)] + if args.baseline_file is not None: + spec = importlib.util.spec_from_file_location( + "baseline_reject_sampling", args.baseline_file + ) + baseline = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = baseline + spec.loader.exec_module(baseline) + samplers.insert(0, ("baseline", baseline.chain_speculative_sampling_triton)) + + print( + json.dumps( + { + "gpu": torch.cuda.get_device_name(), + "torch": torch.__version__, + "triton": triton.__version__, + "seed": args.seed, + "steps": args.steps, + "vocab": args.vocab, + "warmup_ms": args.warmup, + "measurement_ms": args.measurement, + } + ), + flush=True, + ) + torch.manual_seed(args.seed) + for batch in args.batches: + slots = args.steps + 1 + target = torch.randn(batch, slots, args.vocab, device="cuda").softmax(-1) + draft = ( + 0.7 * target[:, : args.steps] + + 0.3 + * torch.randn(batch, args.steps, args.vocab, device="cuda").softmax(-1) + ).contiguous() + candidates = torch.zeros(batch, slots, dtype=torch.long, device="cuda") + if args.steps: + candidates[:, 1:] = torch.multinomial(draft.flatten(0, 1), 1).view( + batch, args.steps + ) + indices = torch.arange(batch * slots, device="cuda").view(batch, slots) + predicts = torch.empty(batch * slots, dtype=torch.int32, device="cuda") + accepted = torch.empty_like(indices, dtype=torch.int32) + lengths = torch.empty(batch, dtype=torch.int32, device="cuda") + coins = torch.rand(batch, slots, device="cuda") + final_coins = torch.rand(batch, device="cuda") + + measurements = { + (name, block): [] for name, _ in samplers for block in (False, True) + } + for repeat in range(args.repeats): + order = samplers if repeat % 2 == 0 else samplers[::-1] + for name, sampler in order: + for block in (False, True): + + def run(): + sampler( + predicts, + accepted, + lengths, + candidates, + indices, + None, + None, + coins, + final_coins, + target, + draft, + 1.0, + 1.0, + True, + block_verification=block, + ) + + run() + milliseconds = triton.testing.do_bench( + run, warmup=args.warmup, rep=args.measurement + ) + measurements[name, block].append(milliseconds * 1000) + for (name, block), microseconds in measurements.items(): + print( + json.dumps( + { + "batch": batch, + "implementation": name, + "block": block, + "microseconds": microseconds, + "median_us": statistics.median(microseconds), + } + ), + flush=True, + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--batches", type=int, nargs="+", default=[1, 16, 128]) + parser.add_argument("--steps", type=int, default=5) + parser.add_argument("--vocab", type=int, default=131072) + parser.add_argument("--seed", type=int, default=2026) + parser.add_argument("--warmup", type=int, default=100, help="Warmup milliseconds") + parser.add_argument( + "--measurement", type=int, default=500, help="Measurement milliseconds" + ) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--baseline-file", type=Path) + benchmark(parser.parse_args()) diff --git a/docs/docs/advanced_features/speculative_decoding.mdx b/docs/docs/advanced_features/speculative_decoding.mdx index c3dff04502..727e24e67c 100644 --- a/docs/docs/advanced_features/speculative_decoding.mdx +++ b/docs/docs/advanced_features/speculative_decoding.mdx @@ -14,6 +14,7 @@ SGLang provides several speculative decoding options, including EAGLE-2/EAGLE-3, - [EAGLE-2 Decoding with torch.compile](#eagle-2-decoding-with-torch-compile) - [EAGLE-2 Decoding via Frequency-Ranked Speculative Sampling](#eagle-2-decoding-via-frequency-ranked-speculative-sampling) - [EAGLE-3 Decoding](#eagle-3-decoding) +- [Block verification](#block-verification) - [Multi Token Prediction](#multi-token-prediction) - [UNO decoding](#uno-decoding) - [DFlash Decoding](#dflash-decoding) @@ -405,6 +406,23 @@ print(response.choices[0].message.content) --- +## Block verification + +For stochastic EAGLE, EAGLE3, or NEXTN decoding on NVIDIA CUDA or AMD ROCm, add +`--speculative-use-block-verification` +with `--speculative-eagle-topk 1`. The draft model must produce a full target-vocabulary +proposal distribution, as required by the existing rejection sampler. + +[Block verification](https://arxiv.org/abs/2403.10444) checks every draft prefix and +keeps the longest accepted prefix, including when an earlier prefix was rejected. +It uses a corresponding correction distribution to preserve target sampling. +Use `--speculative-use-rejection-sampling` alone for token-wise verification, which +stops at the first rejected token. +Greedy requests keep their existing verification behavior. + +Block verification adds vocabulary reductions during verification. Benchmark your +model and workload to measure the effect on latency and throughput. + ## Multi Token Prediction We support [MTP (Multi-Token Prediction)](https://arxiv.org/pdf/2404.19737) in SGLang by using speculative decoding. We use `XiaomiMiMo/MiMo-7B-RL` as an example here (for DeepSeek MTP usage, refer to [DeepSeek-V3.2 cookbook ยง4.2.3](/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2#4-2-3-multi-token-prediction-eagle-speculative-decoding)). diff --git a/python/sglang/kernels/ops/speculative/reject_sampling.py b/python/sglang/kernels/ops/speculative/reject_sampling.py index 7d5ad9e9cd..cd495a6173 100644 --- a/python/sglang/kernels/ops/speculative/reject_sampling.py +++ b/python/sglang/kernels/ops/speculative/reject_sampling.py @@ -31,6 +31,8 @@ def speculative_sampling_classic_kernel( NUM_SLOTS: tl.constexpr, VOCAB_SIZE: tl.constexpr, BLOCK_V: tl.constexpr, + BLOCK_VERIFICATION: tl.constexpr = False, + BLOCK_REDUCE: tl.constexpr = 4096, ): pid = tl.program_id(0) cur_prob_row = 0 @@ -44,60 +46,124 @@ def speculative_sampling_classic_kernel( last_accepted_global_idx = root_global_idx num_accept = 0 + residual_scale = 1.0 + norm_sum = -1.0 - # Verification Loop - step = 1 - continue_verifying = 1 - - while (step < NUM_SLOTS) and (continue_verifying == 1): - draft_token = tl.load(cand_ptr_base + step * stride_cand_s) - - offset_prob = ( - (pid * stride_tp_b) - + (cur_prob_row * stride_tp_s) - + (draft_token * stride_tp_v) - ) - offset_draft = ( - (pid * stride_dp_b) - + (cur_prob_row * stride_dp_s) - + (draft_token * stride_dp_v) - ) - - p = tl.load(TargetProbs + offset_prob) - q = tl.load(DraftProbs + offset_draft) - - coin = tl.load(uni_ptr_base + (step - 1) * stride_uni_s) - - # X was sampled from q, so q(X) has to be a positive probability. - # Anything else means this row is not the distribution X came from, and - # `coin * q < p` would then accept unconditionally -- -inf < p for an - # -inf q, 0 < p for a zero one, and the range guard the residual passes - # use lets zero through. Reject instead: the residual path resamples - # from the target, which is the safe direction to fail in. - q_is_prob = (q > 0.0) & (q <= 1.0) - - if q_is_prob & (coin * q < p): - num_accept += 1 - cur_prob_row = step - tl.store(Predicts + last_accepted_global_idx, draft_token) - - curr_global_idx = tl.load(idx_ptr_base + step * stride_idx_s) - tl.store( - AcceptIndex + pid * stride_idx_b + num_accept * stride_idx_s, - curr_global_idx, + if BLOCK_VERIFICATION: + # Algorithm 2: https://arxiv.org/abs/2403.10444 + # Keep the short recurrence in registers; only the longest accepted prefix matters. + prefix_prob = 1.0 + prefix_probs = () + for step in tl.static_range(1, NUM_SLOTS): + draft_token = tl.load(cand_ptr_base + step * stride_cand_s) + p = tl.load( + TargetProbs + + pid * stride_tp_b + + (step - 1) * stride_tp_s + + draft_token * stride_tp_v ) - last_accepted_global_idx = curr_global_idx + q = tl.load( + DraftProbs + + pid * stride_dp_b + + (step - 1) * stride_dp_s + + draft_token * stride_dp_v + ) + q_is_prob = (q > 0.0) & (q <= 1.0) + prefix_prob = tl.where(q_is_prob, tl.minimum(prefix_prob * p / q, 1.0), 0.0) + prefix_probs += (prefix_prob,) - step += 1 - else: - continue_verifying = 0 + for step in tl.static_range(NUM_SLOTS - 1, 0, -1): + if num_accept == 0: + prefix_prob = prefix_probs[step - 1] + coin = tl.load(uni_ptr_base + (step - 1) * stride_uni_s) + # For normalized distributions h_i <= r_i; reject before scanning. + if coin < prefix_prob: + accept_prob = prefix_prob + residual_mass = -1.0 + if ( + (step < NUM_SLOTS - 1) + & (prefix_prob > 0.0) + & (prefix_prob < 1.0) + ): + residual_mass = 0.0 + for v_start in range(0, VOCAB_SIZE, BLOCK_REDUCE): + v_offsets = v_start + tl.arange(0, BLOCK_REDUCE) + mask = v_offsets < VOCAB_SIZE + p_next = tl.load( + TargetProbs + + pid * stride_tp_b + + step * stride_tp_s + + v_offsets * stride_tp_v, + mask=mask, + other=0.0, + ) + q_next = tl.load( + DraftProbs + + pid * stride_dp_b + + step * stride_dp_s + + v_offsets * stride_dp_v, + mask=mask, + other=0.0, + ) + residual_mass += tl.sum( + tl.maximum(prefix_prob * p_next - q_next, 0.0) + ) + accept_prob = residual_mass / ( + residual_mass + 1.0 - prefix_prob + ) + if coin < accept_prob: + num_accept = step + residual_scale = prefix_prob + norm_sum = residual_mass + + cur_prob_row = num_accept + for step in range(1, num_accept + 1): + draft_token = tl.load(cand_ptr_base + step * stride_cand_s) + tl.store(Predicts + last_accepted_global_idx, draft_token) + last_accepted_global_idx = tl.load(idx_ptr_base + step * stride_idx_s) + tl.store( + AcceptIndex + pid * stride_idx_b + step * stride_idx_s, + last_accepted_global_idx, + ) + else: + step = 1 + continue_verifying = 1 + # At each token-verification loop entry, cur_prob_row == num_accept == step - 1. + while (step < NUM_SLOTS) and (continue_verifying == 1): + draft_token = tl.load(cand_ptr_base + step * stride_cand_s) + p = tl.load( + TargetProbs + + pid * stride_tp_b + + cur_prob_row * stride_tp_s + + draft_token * stride_tp_v + ) + q = tl.load( + DraftProbs + + pid * stride_dp_b + + cur_prob_row * stride_dp_s + + draft_token * stride_dp_v + ) + coin = tl.load(uni_ptr_base + (step - 1) * stride_uni_s) + # A proposal token must have positive probability under its draft distribution. + q_is_prob = (q > 0.0) & (q <= 1.0) + if q_is_prob & (coin * q < p): + num_accept += 1 + cur_prob_row = step + tl.store(Predicts + last_accepted_global_idx, draft_token) + last_accepted_global_idx = tl.load(idx_ptr_base + step * stride_idx_s) + tl.store( + AcceptIndex + pid * stride_idx_b + num_accept * stride_idx_s, + last_accepted_global_idx, + ) + step += 1 + else: + continue_verifying = 0 tl.store(AcceptTokenNum + pid, num_accept) # Final Sampling - all_drafts_accepted = continue_verifying + all_drafts_accepted = num_accept == NUM_SLOTS - 1 coin_final = tl.load(UniformSamplesFinal + pid) - norm_sum = 0.0 tp_base_ptr = TargetProbs + (pid * stride_tp_b) + (cur_prob_row * stride_tp_s) # DraftProbs has only num_steps rows (TargetProbs has num_steps + 1). When @@ -106,27 +172,30 @@ def speculative_sampling_classic_kernel( # dereferences this pointer; on rejection cur_prob_row <= num_steps - 1. dp_base_ptr_safe = DraftProbs + (pid * stride_dp_b) + (cur_prob_row * stride_dp_s) - # Pass 1: Sum - for v_start in range(0, VOCAB_SIZE, BLOCK_V): - v_offsets = v_start + tl.arange(0, BLOCK_V) - mask = v_offsets < VOCAB_SIZE + # Reuse the selected prefix mass as the correction normalizer. + if norm_sum < 0.0: + norm_sum = 0.0 + # Pass 1: Sum + for v_start in range(0, VOCAB_SIZE, BLOCK_V): + v_offsets = v_start + tl.arange(0, BLOCK_V) + mask = v_offsets < VOCAB_SIZE - p_ptr = tp_base_ptr + v_offsets * stride_tp_v - p_val = tl.load(p_ptr, mask=mask, other=0.0) + p_ptr = tp_base_ptr + v_offsets * stride_tp_v + p_val = tl.load(p_ptr, mask=mask, other=0.0) - if all_drafts_accepted: - val = p_val - else: - q_ptr = dp_base_ptr_safe + v_offsets * stride_dp_v - q_val = tl.load(q_ptr, mask=mask, other=0.0) - # Treat any non-probability q (NaN, +-inf, negative) as 0: the - # residual falls back to p. A comparison against NaN is false, so - # the range test rejects it along with the infinities. - q_val = tl.where((q_val >= 0.0) & (q_val <= 1.0), q_val, 0.0) - diff = p_val - q_val - val = tl.where(diff > 0.0, diff, 0.0) + if all_drafts_accepted: + val = p_val + else: + q_ptr = dp_base_ptr_safe + v_offsets * stride_dp_v + q_val = tl.load(q_ptr, mask=mask, other=0.0) + # Treat any non-probability q (NaN, +-inf, negative) as 0: the + # residual falls back to p. A comparison against NaN is false, so + # the range test rejects it along with the infinities. + q_val = tl.where((q_val >= 0.0) & (q_val <= 1.0), q_val, 0.0) + diff = residual_scale * p_val - q_val + val = tl.where(diff > 0.0, diff, 0.0) - norm_sum += tl.sum(val) + norm_sum += tl.sum(val) # Pass 2: CDF. Degenerate residual (norm_sum == 0, i.e. p == q everywhere on # rejection) leaves the cumsum at 0 <= target_u, so final_token falls back to @@ -151,7 +220,7 @@ def speculative_sampling_classic_kernel( q_val = tl.load(q_ptr, mask=mask, other=0.0) # Same guard as pass 1. q_val = tl.where((q_val >= 0.0) & (q_val <= 1.0), q_val, 0.0) - diff = p_val - q_val + diff = residual_scale * p_val - q_val val = tl.where(diff > 0.0, diff, 0.0) block_cumsum = tl.cumsum(val, axis=0) @@ -185,9 +254,14 @@ def chain_speculative_sampling_triton( threshold_single, threshold_acc, deterministic, # not used + *, + block_verification: bool = False, ): batch_size, num_slots = candidates.shape vocab_size = target_probs.shape[-1] + # Keep CDF tiles small, and cap multi-tile reductions to avoid register spills. + block_reduce = min(triton.next_power_of_2(vocab_size), 32768) + num_warps = 8 if block_verification and block_reduce > 4096 else 4 grid = (batch_size,) speculative_sampling_classic_kernel[grid]( @@ -215,4 +289,7 @@ def chain_speculative_sampling_triton( NUM_SLOTS=num_slots, VOCAB_SIZE=vocab_size, BLOCK_V=4096, + BLOCK_VERIFICATION=block_verification, + BLOCK_REDUCE=block_reduce, + num_warps=num_warps, ) diff --git a/python/sglang/srt/arg_groups/fields/spec.py b/python/sglang/srt/arg_groups/fields/spec.py index 71154d8188..fc35e8bf77 100644 --- a/python/sglang/srt/arg_groups/fields/spec.py +++ b/python/sglang/srt/arg_groups/fields/spec.py @@ -116,6 +116,11 @@ class Spec(msgspec.Struct): speculative_use_rejection_sampling: A[ bool, "Use rejection sampling for speculative decoding (requires topk=1)." ] = False + speculative_use_block_verification: A[ + bool, + "Use block verification for EAGLE/EAGLE3/NEXTN on CUDA or ROCm " + "(requires topk=1).", + ] = False speculative_token_map: A[ Optional[str], "The path of the draft model's small vocab table.", diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index 18a84bd4cb..39134de148 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -158,6 +158,23 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None: ), ) + if cfg.speculative_use_block_verification: + if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3"): + raise ValueError( + "--speculative-use-block-verification only supports EAGLE / EAGLE3 / NEXTN." + ) + if cfg.device != "cuda": + raise ValueError( + "--speculative-use-block-verification only supports CUDA or ROCm." + ) + # Block verification needs sampled proposals and their full distributions. + if not cfg.speculative_use_rejection_sampling: + declare_resolution( + server_args, + "handle_speculative_decoding", + speculative_use_rejection_sampling=True, + ) + # Validate --speculative-draft-window-size / --speculative-draft-sink-size once, # regardless of algorithm. Consumed by DFLASH (compact draft KV cache), Llama # EAGLE-3 (drafter attention SWA), and the built-in MTP/NEXTN + EAGLE draft-decode diff --git a/python/sglang/srt/speculative/eagle_utils.py b/python/sglang/srt/speculative/eagle_utils.py index c1bacff0e1..153ca448ea 100644 --- a/python/sglang/srt/speculative/eagle_utils.py +++ b/python/sglang/srt/speculative/eagle_utils.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging import math from enum import IntEnum +from functools import partial from typing import TYPE_CHECKING, List, Optional, Tuple import torch @@ -898,6 +899,8 @@ def eagle_sample( # branch not taken, and HIP only reaches here with rejection sampling on. if use_rejection_sampling: sampling_fn = chain_speculative_sampling_triton + if get_spec().speculative_use_block_verification: + sampling_fn = partial(sampling_fn, block_verification=True) else: if _is_cuda: from sglang.kernels.ops.speculative.sampling import ( diff --git a/test/registered/kernels/ops/speculative/test_block_verification.py b/test/registered/kernels/ops/speculative/test_block_verification.py new file mode 100644 index 0000000000..f9593b8135 --- /dev/null +++ b/test/registered/kernels/ops/speculative/test_block_verification.py @@ -0,0 +1,331 @@ +import itertools +import os +import sys + +import pytest +import torch + +from sglang.kernels.ops.speculative.reject_sampling import ( + chain_speculative_sampling_triton, +) +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=30, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +DEVICE = "cpu" if os.environ.get("TRITON_INTERPRET") == "1" else "cuda" + + +def _verify(target, draft, candidates, coins, final_coins, *, block=True): + batch, slots = candidates.shape + # Exercise the global-index contract independently of the probability rows. + indices = ( + torch.arange(batch * slots, device=target.device).flip(0).view(batch, slots) + ) + predicts = torch.full((batch * slots,), -1, dtype=torch.int32, device=target.device) + accepted = torch.full((batch, slots), -1, dtype=torch.int32, device=target.device) + lengths = torch.empty(batch, dtype=torch.int32, device=target.device) + chain_speculative_sampling_triton( + predicts, + accepted, + lengths, + candidates, + indices, + None, + None, + coins, + final_coins, + target, + draft, + 1.0, + 1.0, + True, + block_verification=block, + ) + positions = torch.arange(slots, device=target.device)[None, :] + valid = positions <= lengths[:, None] + torch.testing.assert_close(accepted[valid].long(), indices[valid]) + assert (accepted[~valid] == -1).all() + return lengths, predicts[indices] + + +def _reference(target, draft, candidates, coins, final_coins, *, block): + target, draft = target.double(), draft.double() + batch, slots, _ = target.shape + lengths = torch.zeros(batch, dtype=torch.long) + prefix_prob = torch.ones(batch) + residual_scale = torch.ones(batch) + active = torch.ones(batch, dtype=torch.bool) + rows = torch.arange(batch) + for step in range(1, slots): + token = candidates[:, step] + ratio = target[rows, step - 1, token] / draft[rows, step - 1, token] + if block: + prefix_prob = (prefix_prob * ratio).clamp(max=1) + if step == slots - 1: + probability = prefix_prob + else: + mass = ( + (prefix_prob[:, None] * target[:, step] - draft[:, step]) + .clamp(min=0) + .sum(-1) + ) + probability = torch.where( + prefix_prob == 1, 1, mass / (mass + 1 - prefix_prob) + ) + accept = coins[:, step - 1] < probability + residual_scale = torch.where(accept, prefix_prob, residual_scale) + else: + active &= coins[:, step - 1] < ratio + accept = active + lengths = torch.where(accept, step, lengths) + weights = target[rows, lengths].clone() + rejected = lengths < slots - 1 + weights[rejected] = ( + residual_scale[rejected, None] * weights[rejected] + - draft[rows[rejected], lengths[rejected]] + ).clamp(min=0) + cdf = weights.cumsum(-1) / weights.sum(-1, keepdim=True) + final = (cdf <= final_coins[:, None]).sum(-1) + output = torch.full((batch, slots), -1, dtype=torch.int32) + for row in range(batch): + length = lengths[row] + output[row, :length] = candidates[row, 1 : length + 1] + output[row, length] = final[row] + return lengths, output, cdf + + +@pytest.mark.parametrize("block", [False, True]) +@pytest.mark.parametrize( + "steps,vocab", [(0, 3), (1, 7), (3, 17), (5, 4099), (3, 131073)] +) +def test_matches_reference(steps, vocab, block): + torch.manual_seed(42) + batch = 7 + target = torch.rand(batch, steps + 1, vocab).softmax(-1) + draft = torch.rand(batch, steps, vocab).softmax(-1) + candidates = torch.zeros(batch, steps + 1, dtype=torch.long) + if steps: + candidates[:, 1:] = torch.multinomial(draft.flatten(0, 1), 1).view(batch, steps) + coins = torch.rand(batch, steps + 1) + final_coins = torch.rand(batch) + expected_lengths, expected, cdf = _reference( + target, draft, candidates, coins, final_coins, block=block + ) + + # Non-contiguous probability, candidate and coin tensors must honor strides. + tensors = [ + torch.stack([x, x], dim=-1).to(DEVICE)[..., 0] + for x in (target, draft, candidates, coins) + ] + lengths, output = _verify(*tensors, final_coins.to(DEVICE), block=block) + torch.testing.assert_close(lengths.cpu().long(), expected_lengths) + output = output.cpu() + accepted = torch.arange(steps + 1)[None, :] < expected_lengths[:, None] + torch.testing.assert_close(output[accepted], expected[accepted]) + final = output[torch.arange(batch), expected_lengths].long() + assert ((final >= 0) & (final < vocab)).all() + upper = cdf[torch.arange(batch), final] + lower = torch.where( + final > 0, cdf[torch.arange(batch), (final - 1).clamp(min=0)], 0 + ) + # FP32 reductions may cross a nearby CDF boundary; compare probability, not token IDs. + assert (upper > lower).all() + assert (final_coins >= lower - 1e-6).all() + assert (final_coins <= upper + 1e-6).all() + + +def test_accepts_longer_prefix_after_rejection(): + target = torch.tensor([[[1 / 3, 2 / 3]] * 3], device=DEVICE) + draft = torch.tensor([[[2 / 3, 1 / 3]] * 2], device=DEVICE) + candidates = torch.tensor([[0, 0, 1]], device=DEVICE) + coins = torch.full((1, 3), 0.9, device=DEVICE) + final_coins = torch.tensor([0.5], device=DEVICE) + lengths, output = _verify(target, draft, candidates, coins, final_coins) + assert lengths.item() == 2 + assert output.tolist() == [[0, 1, 1]] + token_lengths, _ = _verify( + target, draft, candidates, coins, final_coins, block=False + ) + assert token_lengths.item() == 0 + + +def test_scales_residual_at_last_accepted_prefix(): + target = torch.tensor( + [[[0.25, 0.5, 0.25], [0.6, 0.3, 0.1], [1.0, 0.0, 0.0]]], device=DEVICE + ) + draft = torch.tensor([[[0.5, 0.25, 0.25], [0.1, 0.1, 0.8]]], device=DEVICE) + lengths, output = _verify( + target, + draft, + torch.tensor([[0, 0, 2]], device=DEVICE), + torch.tensor([[0.2, 0.9, 0.0]], device=DEVICE), + torch.tensor([0.75], device=DEVICE), + ) + assert lengths.item() == 1 + # Scaled residual is [0.8, 0.2, 0]; unscaled residual would sample token 1. + assert output[0, :2].tolist() == [0, 0] + + +def test_selects_longest_accepted_prefix(): + steps = 5 + batch = steps + 1 + target = torch.tensor([0.25, 0.625, 0.125]).repeat(batch, steps + 1, 1) + draft = torch.tensor([0.25, 0.125, 0.625]).repeat(batch, steps, 1) + draft[:, 0] = torch.tensor([0.5, 0.125, 0.375]) + candidates = torch.zeros(batch, steps + 1, dtype=torch.long) + expected_lengths = torch.arange(batch) + # Every shorter prefix also passes, so descending selection must stop at tau. + coins = torch.where( + torch.arange(steps + 1)[None, :] < expected_lengths[:, None], 0.0, 0.9 + ) + final_coins = torch.full((batch,), 0.75) + reference_lengths, expected, _ = _reference( + target, draft, candidates, coins, final_coins, block=True + ) + torch.testing.assert_close(reference_lengths, expected_lengths) + lengths, output = _verify( + *(x.to(DEVICE) for x in (target, draft, candidates, coins, final_coins)) + ) + torch.testing.assert_close(lengths.cpu().long(), expected_lengths) + valid = torch.arange(steps + 1)[None, :] <= expected_lengths[:, None] + torch.testing.assert_close(output.cpu()[valid], expected[valid]) + + +@pytest.mark.parametrize( + "target_next,draft_next,threshold", + [([0.5, 0.5], [0.125, 0.875], 0.5), ([1.0, 0.0], [0.0, 1.0], 0.75)], +) +def test_prefix_acceptance_boundary(target_next, draft_next, threshold): + target = torch.tensor([[[0.375, 0.625], target_next, [0.5, 0.5]]]).repeat(3, 1, 1) + draft = torch.tensor([[[0.5, 0.5], draft_next]]).repeat(3, 1, 1) + candidates = torch.tensor([[0, 0, 1]]).repeat(3, 1) + coins = torch.full((3, 3), 0.99) + # Exercise strict comparison at h_1, including the h_1 == r_1 shortcut boundary. + threshold = torch.tensor(threshold) + coins[:, 0] = torch.stack( + [ + torch.nextafter(threshold, torch.tensor(0.0)), + threshold, + torch.nextafter(threshold, torch.tensor(1.0)), + ] + ) + final_coins = torch.full((3,), 0.5) + expected_lengths, expected, _ = _reference( + target, draft, candidates, coins, final_coins, block=True + ) + assert expected_lengths.tolist() == [1, 0, 0] + lengths, output = _verify( + *(x.to(DEVICE) for x in (target, draft, candidates, coins, final_coins)) + ) + torch.testing.assert_close(lengths.cpu().long(), expected_lengths) + valid = torch.arange(3)[None, :] <= expected_lengths[:, None] + torch.testing.assert_close(output.cpu()[valid], expected[valid]) + + +@pytest.mark.parametrize("identical", [False, True]) +def test_zero_and_unit_prefix_probabilities(identical): + target = torch.tensor([[[1.0, 0.0]] * 4], device=DEVICE) + draft = target[:, :3] if identical else 1 - target[:, :3] + candidates = torch.full((1, 4), 0 if identical else 1, device=DEVICE) + lengths, output = _verify( + target, + draft, + candidates, + torch.zeros(1, 4, device=DEVICE), + torch.zeros(1, device=DEVICE), + ) + assert lengths.item() == (3 if identical else 0) + assert (output[0, : lengths.item() + 1] == 0).all() + + +@pytest.mark.parametrize("block,expected_length", [(False, 10 / 9), (True, 11 / 9)]) +def test_paper_example_distribution(block, expected_length): + torch.manual_seed(123) + batch = 65536 + target_row = torch.tensor([1 / 3, 2 / 3], device=DEVICE) + draft_row = 1 - target_row + target = target_row.expand(batch, 3, 2) + draft = draft_row.expand(batch, 2, 2) + candidates = torch.cat( + [ + torch.zeros(batch, 1, dtype=torch.long, device=DEVICE), + torch.multinomial(draft_row, batch * 2, replacement=True).view(batch, 2), + ], + dim=1, + ) + lengths, output = _verify( + target, + draft, + candidates, + torch.rand(batch, 3, device=DEVICE), + torch.rand(batch, device=DEVICE), + block=block, + ) + assert abs(lengths.float().mean().item() - expected_length) < 0.015 + # Complete each variable-length output from the target before comparing joints. + completion = torch.multinomial(target_row, batch * 3, replacement=True).view( + batch, 3 + ) + output = torch.where( + torch.arange(3, device=DEVICE)[None, :] <= lengths[:, None], output, completion + ) + for sequence in itertools.product(range(2), repeat=3): + observed = ( + (output == torch.tensor(sequence, device=DEVICE)).all(-1).float().mean() + ) + expected = target_row[list(sequence)].prod() + assert abs(observed.item() - expected.item()) < 0.008 + + +@pytest.mark.parametrize("block", [False, True]) +@pytest.mark.parametrize("steps", [2, 5]) +def test_context_dependent_output_distribution(block, steps): + torch.manual_seed(456) + batch = 65536 + target_initial = torch.tensor([0.4, 0.6], device=DEVICE) + draft_initial = torch.tensor([0.7, 0.3], device=DEVICE) + target_transition = torch.tensor([[0.25, 0.75], [0.65, 0.35]], device=DEVICE) + draft_transition = torch.tensor([[0.7, 0.3], [0.15, 0.85]], device=DEVICE) + tokens = [torch.multinomial(draft_initial, batch, replacement=True)] + for _ in range(1, steps): + tokens.append(torch.multinomial(draft_transition[tokens[-1]], 1).squeeze(1)) + candidates = torch.stack([torch.zeros_like(tokens[0]), *tokens], dim=1) + target = torch.stack( + [ + target_initial.expand(batch, -1), + *[target_transition[token] for token in tokens], + ], + dim=1, + ) + draft = torch.stack( + [ + draft_initial.expand(batch, -1), + *[draft_transition[token] for token in tokens[:-1]], + ], + dim=1, + ) + lengths, output = _verify( + target, + draft, + candidates, + torch.rand(batch, steps + 1, device=DEVICE), + torch.rand(batch, device=DEVICE), + block=block, + ) + for step in range(1, steps + 1): + completion = torch.multinomial( + target_transition[output[:, step - 1].long()], 1 + ).squeeze(1) + output[:, step] = torch.where(step <= lengths, output[:, step], completion) + for sequence in itertools.product(range(2), repeat=steps + 1): + observed = ( + (output == torch.tensor(sequence, device=DEVICE)).all(-1).float().mean() + ) + expected = target_initial[sequence[0]] + for previous, token in zip(sequence, sequence[1:]): + expected = expected * target_transition[previous, token] + assert abs(observed.item() - expected.item()) < 0.008 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/test/registered/unit/spec/test_block_verification_args.py b/test/registered/unit/spec/test_block_verification_args.py new file mode 100644 index 0000000000..dd75779919 --- /dev/null +++ b/test/registered/unit/spec/test_block_verification_args.py @@ -0,0 +1,87 @@ +import unittest +from types import SimpleNamespace + +from sglang.srt.arg_groups.overrides import resolution_result +from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding +from sglang.srt.server_args import ServerArgs +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="base-a-test-cpu") + + +class TestBlockVerificationArgs(unittest.TestCase): + def _args(self, **overrides): + args = ServerArgs( + model_path="dummy", + device="cuda", + speculative_algorithm="EAGLE", + speculative_eagle_topk=1, + speculative_num_steps=3, + speculative_num_draft_tokens=4, + speculative_use_block_verification=True, + ) + args._model_config = SimpleNamespace( + hf_config=SimpleNamespace(architectures=["LlamaForCausalLM"]) + ) + for key, value in overrides.items(): + setattr(args, key, value) + return args + + def test_supported_algorithms(self): + for algorithm in ("EAGLE", "EAGLE3", "NEXTN"): + for use_rejection_sampling in (False, True): + with self.subTest( + algorithm=algorithm, use_rejection_sampling=use_rejection_sampling + ): + args = self._args( + speculative_algorithm=algorithm, + speculative_use_rejection_sampling=use_rejection_sampling, + ) + handle_speculative_decoding(args) + self.assertTrue( + resolution_result(args, "speculative_use_block_verification") + ) + self.assertTrue( + resolution_result(args, "speculative_use_rejection_sampling") + ) + self.assertEqual( + args.speculative_use_rejection_sampling, use_rejection_sampling + ) + + def test_other_verification_modes_unchanged(self): + for use_rejection_sampling in (False, True): + with self.subTest(use_rejection_sampling=use_rejection_sampling): + args = self._args( + speculative_algorithm="EAGLE3", + speculative_use_block_verification=False, + speculative_use_rejection_sampling=use_rejection_sampling, + ) + handle_speculative_decoding(args) + self.assertFalse( + resolution_result(args, "speculative_use_block_verification") + ) + self.assertEqual( + resolution_result(args, "speculative_use_rejection_sampling"), + use_rejection_sampling, + ) + + def test_unsupported_configurations(self): + cases = [ + ({"speculative_algorithm": None}, "only supports EAGLE"), + ({"speculative_algorithm": "STANDALONE"}, "only supports EAGLE"), + ({"speculative_algorithm": "DFLASH"}, "only supports EAGLE"), + ({"device": "cpu"}, "only supports CUDA or ROCm"), + ({"device": "npu"}, "only supports CUDA or ROCm"), + ({"speculative_eagle_topk": 2}, "requires --speculative-eagle-topk=1"), + ({"speculative_accept_threshold_single": 0.5}, "incompatible"), + ({"speculative_accept_threshold_acc": 0.5}, "incompatible"), + ({"enable_deterministic_inference": True}, "incompatible"), + ] + for overrides, message in cases: + with self.subTest(overrides=overrides): + with self.assertRaisesRegex(ValueError, message): + handle_speculative_decoding(self._args(**overrides)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/spec/test_spec_registry.py b/test/registered/unit/spec/test_spec_registry.py index 8cda36a6c1..6d0c00a855 100644 --- a/test/registered/unit/spec/test_spec_registry.py +++ b/test/registered/unit/spec/test_spec_registry.py @@ -214,6 +214,7 @@ class TestServerArgsHook(_RegistryIsolated): speculative_moe_runner_backend=None, moe_runner_backend="auto", speculative_algorithm="my_handle_args", + speculative_use_block_verification=False, decrypted_draft_config_file=None, trust_remote_code=False, speculative_draft_window_size=None,