[Speculative] Add opt-in block verification (#42297)
This commit is contained in:
@@ -31,6 +31,8 @@ def speculative_sampling_classic_kernel(
|
||||
NUM_SLOTS: tl.constexpr,
|
||||
VOCAB_SIZE: tl.constexpr,
|
||||
BLOCK_V: tl.constexpr,
|
||||
BLOCK_VERIFICATION: tl.constexpr = False,
|
||||
BLOCK_REDUCE: tl.constexpr = 4096,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
cur_prob_row = 0
|
||||
@@ -44,60 +46,124 @@ def speculative_sampling_classic_kernel(
|
||||
last_accepted_global_idx = root_global_idx
|
||||
|
||||
num_accept = 0
|
||||
residual_scale = 1.0
|
||||
norm_sum = -1.0
|
||||
|
||||
# Verification Loop
|
||||
step = 1
|
||||
continue_verifying = 1
|
||||
|
||||
while (step < NUM_SLOTS) and (continue_verifying == 1):
|
||||
draft_token = tl.load(cand_ptr_base + step * stride_cand_s)
|
||||
|
||||
offset_prob = (
|
||||
(pid * stride_tp_b)
|
||||
+ (cur_prob_row * stride_tp_s)
|
||||
+ (draft_token * stride_tp_v)
|
||||
)
|
||||
offset_draft = (
|
||||
(pid * stride_dp_b)
|
||||
+ (cur_prob_row * stride_dp_s)
|
||||
+ (draft_token * stride_dp_v)
|
||||
)
|
||||
|
||||
p = tl.load(TargetProbs + offset_prob)
|
||||
q = tl.load(DraftProbs + offset_draft)
|
||||
|
||||
coin = tl.load(uni_ptr_base + (step - 1) * stride_uni_s)
|
||||
|
||||
# X was sampled from q, so q(X) has to be a positive probability.
|
||||
# Anything else means this row is not the distribution X came from, and
|
||||
# `coin * q < p` would then accept unconditionally -- -inf < p for an
|
||||
# -inf q, 0 < p for a zero one, and the range guard the residual passes
|
||||
# use lets zero through. Reject instead: the residual path resamples
|
||||
# from the target, which is the safe direction to fail in.
|
||||
q_is_prob = (q > 0.0) & (q <= 1.0)
|
||||
|
||||
if q_is_prob & (coin * q < p):
|
||||
num_accept += 1
|
||||
cur_prob_row = step
|
||||
tl.store(Predicts + last_accepted_global_idx, draft_token)
|
||||
|
||||
curr_global_idx = tl.load(idx_ptr_base + step * stride_idx_s)
|
||||
tl.store(
|
||||
AcceptIndex + pid * stride_idx_b + num_accept * stride_idx_s,
|
||||
curr_global_idx,
|
||||
if BLOCK_VERIFICATION:
|
||||
# Algorithm 2: https://arxiv.org/abs/2403.10444
|
||||
# Keep the short recurrence in registers; only the longest accepted prefix matters.
|
||||
prefix_prob = 1.0
|
||||
prefix_probs = ()
|
||||
for step in tl.static_range(1, NUM_SLOTS):
|
||||
draft_token = tl.load(cand_ptr_base + step * stride_cand_s)
|
||||
p = tl.load(
|
||||
TargetProbs
|
||||
+ pid * stride_tp_b
|
||||
+ (step - 1) * stride_tp_s
|
||||
+ draft_token * stride_tp_v
|
||||
)
|
||||
last_accepted_global_idx = curr_global_idx
|
||||
q = tl.load(
|
||||
DraftProbs
|
||||
+ pid * stride_dp_b
|
||||
+ (step - 1) * stride_dp_s
|
||||
+ draft_token * stride_dp_v
|
||||
)
|
||||
q_is_prob = (q > 0.0) & (q <= 1.0)
|
||||
prefix_prob = tl.where(q_is_prob, tl.minimum(prefix_prob * p / q, 1.0), 0.0)
|
||||
prefix_probs += (prefix_prob,)
|
||||
|
||||
step += 1
|
||||
else:
|
||||
continue_verifying = 0
|
||||
for step in tl.static_range(NUM_SLOTS - 1, 0, -1):
|
||||
if num_accept == 0:
|
||||
prefix_prob = prefix_probs[step - 1]
|
||||
coin = tl.load(uni_ptr_base + (step - 1) * stride_uni_s)
|
||||
# For normalized distributions h_i <= r_i; reject before scanning.
|
||||
if coin < prefix_prob:
|
||||
accept_prob = prefix_prob
|
||||
residual_mass = -1.0
|
||||
if (
|
||||
(step < NUM_SLOTS - 1)
|
||||
& (prefix_prob > 0.0)
|
||||
& (prefix_prob < 1.0)
|
||||
):
|
||||
residual_mass = 0.0
|
||||
for v_start in range(0, VOCAB_SIZE, BLOCK_REDUCE):
|
||||
v_offsets = v_start + tl.arange(0, BLOCK_REDUCE)
|
||||
mask = v_offsets < VOCAB_SIZE
|
||||
p_next = tl.load(
|
||||
TargetProbs
|
||||
+ pid * stride_tp_b
|
||||
+ step * stride_tp_s
|
||||
+ v_offsets * stride_tp_v,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
q_next = tl.load(
|
||||
DraftProbs
|
||||
+ pid * stride_dp_b
|
||||
+ step * stride_dp_s
|
||||
+ v_offsets * stride_dp_v,
|
||||
mask=mask,
|
||||
other=0.0,
|
||||
)
|
||||
residual_mass += tl.sum(
|
||||
tl.maximum(prefix_prob * p_next - q_next, 0.0)
|
||||
)
|
||||
accept_prob = residual_mass / (
|
||||
residual_mass + 1.0 - prefix_prob
|
||||
)
|
||||
if coin < accept_prob:
|
||||
num_accept = step
|
||||
residual_scale = prefix_prob
|
||||
norm_sum = residual_mass
|
||||
|
||||
cur_prob_row = num_accept
|
||||
for step in range(1, num_accept + 1):
|
||||
draft_token = tl.load(cand_ptr_base + step * stride_cand_s)
|
||||
tl.store(Predicts + last_accepted_global_idx, draft_token)
|
||||
last_accepted_global_idx = tl.load(idx_ptr_base + step * stride_idx_s)
|
||||
tl.store(
|
||||
AcceptIndex + pid * stride_idx_b + step * stride_idx_s,
|
||||
last_accepted_global_idx,
|
||||
)
|
||||
else:
|
||||
step = 1
|
||||
continue_verifying = 1
|
||||
# At each token-verification loop entry, cur_prob_row == num_accept == step - 1.
|
||||
while (step < NUM_SLOTS) and (continue_verifying == 1):
|
||||
draft_token = tl.load(cand_ptr_base + step * stride_cand_s)
|
||||
p = tl.load(
|
||||
TargetProbs
|
||||
+ pid * stride_tp_b
|
||||
+ cur_prob_row * stride_tp_s
|
||||
+ draft_token * stride_tp_v
|
||||
)
|
||||
q = tl.load(
|
||||
DraftProbs
|
||||
+ pid * stride_dp_b
|
||||
+ cur_prob_row * stride_dp_s
|
||||
+ draft_token * stride_dp_v
|
||||
)
|
||||
coin = tl.load(uni_ptr_base + (step - 1) * stride_uni_s)
|
||||
# A proposal token must have positive probability under its draft distribution.
|
||||
q_is_prob = (q > 0.0) & (q <= 1.0)
|
||||
if q_is_prob & (coin * q < p):
|
||||
num_accept += 1
|
||||
cur_prob_row = step
|
||||
tl.store(Predicts + last_accepted_global_idx, draft_token)
|
||||
last_accepted_global_idx = tl.load(idx_ptr_base + step * stride_idx_s)
|
||||
tl.store(
|
||||
AcceptIndex + pid * stride_idx_b + num_accept * stride_idx_s,
|
||||
last_accepted_global_idx,
|
||||
)
|
||||
step += 1
|
||||
else:
|
||||
continue_verifying = 0
|
||||
|
||||
tl.store(AcceptTokenNum + pid, num_accept)
|
||||
|
||||
# Final Sampling
|
||||
all_drafts_accepted = continue_verifying
|
||||
all_drafts_accepted = num_accept == NUM_SLOTS - 1
|
||||
coin_final = tl.load(UniformSamplesFinal + pid)
|
||||
norm_sum = 0.0
|
||||
|
||||
tp_base_ptr = TargetProbs + (pid * stride_tp_b) + (cur_prob_row * stride_tp_s)
|
||||
# DraftProbs has only num_steps rows (TargetProbs has num_steps + 1). When
|
||||
@@ -106,27 +172,30 @@ def speculative_sampling_classic_kernel(
|
||||
# dereferences this pointer; on rejection cur_prob_row <= num_steps - 1.
|
||||
dp_base_ptr_safe = DraftProbs + (pid * stride_dp_b) + (cur_prob_row * stride_dp_s)
|
||||
|
||||
# Pass 1: Sum
|
||||
for v_start in range(0, VOCAB_SIZE, BLOCK_V):
|
||||
v_offsets = v_start + tl.arange(0, BLOCK_V)
|
||||
mask = v_offsets < VOCAB_SIZE
|
||||
# Reuse the selected prefix mass as the correction normalizer.
|
||||
if norm_sum < 0.0:
|
||||
norm_sum = 0.0
|
||||
# Pass 1: Sum
|
||||
for v_start in range(0, VOCAB_SIZE, BLOCK_V):
|
||||
v_offsets = v_start + tl.arange(0, BLOCK_V)
|
||||
mask = v_offsets < VOCAB_SIZE
|
||||
|
||||
p_ptr = tp_base_ptr + v_offsets * stride_tp_v
|
||||
p_val = tl.load(p_ptr, mask=mask, other=0.0)
|
||||
p_ptr = tp_base_ptr + v_offsets * stride_tp_v
|
||||
p_val = tl.load(p_ptr, mask=mask, other=0.0)
|
||||
|
||||
if all_drafts_accepted:
|
||||
val = p_val
|
||||
else:
|
||||
q_ptr = dp_base_ptr_safe + v_offsets * stride_dp_v
|
||||
q_val = tl.load(q_ptr, mask=mask, other=0.0)
|
||||
# Treat any non-probability q (NaN, +-inf, negative) as 0: the
|
||||
# residual falls back to p. A comparison against NaN is false, so
|
||||
# the range test rejects it along with the infinities.
|
||||
q_val = tl.where((q_val >= 0.0) & (q_val <= 1.0), q_val, 0.0)
|
||||
diff = p_val - q_val
|
||||
val = tl.where(diff > 0.0, diff, 0.0)
|
||||
if all_drafts_accepted:
|
||||
val = p_val
|
||||
else:
|
||||
q_ptr = dp_base_ptr_safe + v_offsets * stride_dp_v
|
||||
q_val = tl.load(q_ptr, mask=mask, other=0.0)
|
||||
# Treat any non-probability q (NaN, +-inf, negative) as 0: the
|
||||
# residual falls back to p. A comparison against NaN is false, so
|
||||
# the range test rejects it along with the infinities.
|
||||
q_val = tl.where((q_val >= 0.0) & (q_val <= 1.0), q_val, 0.0)
|
||||
diff = residual_scale * p_val - q_val
|
||||
val = tl.where(diff > 0.0, diff, 0.0)
|
||||
|
||||
norm_sum += tl.sum(val)
|
||||
norm_sum += tl.sum(val)
|
||||
|
||||
# Pass 2: CDF. Degenerate residual (norm_sum == 0, i.e. p == q everywhere on
|
||||
# rejection) leaves the cumsum at 0 <= target_u, so final_token falls back to
|
||||
@@ -151,7 +220,7 @@ def speculative_sampling_classic_kernel(
|
||||
q_val = tl.load(q_ptr, mask=mask, other=0.0)
|
||||
# Same guard as pass 1.
|
||||
q_val = tl.where((q_val >= 0.0) & (q_val <= 1.0), q_val, 0.0)
|
||||
diff = p_val - q_val
|
||||
diff = residual_scale * p_val - q_val
|
||||
val = tl.where(diff > 0.0, diff, 0.0)
|
||||
|
||||
block_cumsum = tl.cumsum(val, axis=0)
|
||||
@@ -185,9 +254,14 @@ def chain_speculative_sampling_triton(
|
||||
threshold_single,
|
||||
threshold_acc,
|
||||
deterministic, # not used
|
||||
*,
|
||||
block_verification: bool = False,
|
||||
):
|
||||
batch_size, num_slots = candidates.shape
|
||||
vocab_size = target_probs.shape[-1]
|
||||
# Keep CDF tiles small, and cap multi-tile reductions to avoid register spills.
|
||||
block_reduce = min(triton.next_power_of_2(vocab_size), 32768)
|
||||
num_warps = 8 if block_verification and block_reduce > 4096 else 4
|
||||
|
||||
grid = (batch_size,)
|
||||
speculative_sampling_classic_kernel[grid](
|
||||
@@ -215,4 +289,7 @@ def chain_speculative_sampling_triton(
|
||||
NUM_SLOTS=num_slots,
|
||||
VOCAB_SIZE=vocab_size,
|
||||
BLOCK_V=4096,
|
||||
BLOCK_VERIFICATION=block_verification,
|
||||
BLOCK_REDUCE=block_reduce,
|
||||
num_warps=num_warps,
|
||||
)
|
||||
|
||||
@@ -116,6 +116,11 @@ class Spec(msgspec.Struct):
|
||||
speculative_use_rejection_sampling: A[
|
||||
bool, "Use rejection sampling for speculative decoding (requires topk=1)."
|
||||
] = False
|
||||
speculative_use_block_verification: A[
|
||||
bool,
|
||||
"Use block verification for EAGLE/EAGLE3/NEXTN on CUDA or ROCm "
|
||||
"(requires topk=1).",
|
||||
] = False
|
||||
speculative_token_map: A[
|
||||
Optional[str],
|
||||
"The path of the draft model's small vocab table.",
|
||||
|
||||
@@ -158,6 +158,23 @@ def handle_speculative_decoding(server_args: ServerArgs) -> None:
|
||||
),
|
||||
)
|
||||
|
||||
if cfg.speculative_use_block_verification:
|
||||
if cfg.speculative_algorithm not in ("EAGLE", "EAGLE3"):
|
||||
raise ValueError(
|
||||
"--speculative-use-block-verification only supports EAGLE / EAGLE3 / NEXTN."
|
||||
)
|
||||
if cfg.device != "cuda":
|
||||
raise ValueError(
|
||||
"--speculative-use-block-verification only supports CUDA or ROCm."
|
||||
)
|
||||
# Block verification needs sampled proposals and their full distributions.
|
||||
if not cfg.speculative_use_rejection_sampling:
|
||||
declare_resolution(
|
||||
server_args,
|
||||
"handle_speculative_decoding",
|
||||
speculative_use_rejection_sampling=True,
|
||||
)
|
||||
|
||||
# Validate --speculative-draft-window-size / --speculative-draft-sink-size once,
|
||||
# regardless of algorithm. Consumed by DFLASH (compact draft KV cache), Llama
|
||||
# EAGLE-3 (drafter attention SWA), and the built-in MTP/NEXTN + EAGLE draft-decode
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import math
|
||||
from enum import IntEnum
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
@@ -898,6 +899,8 @@ def eagle_sample(
|
||||
# branch not taken, and HIP only reaches here with rejection sampling on.
|
||||
if use_rejection_sampling:
|
||||
sampling_fn = chain_speculative_sampling_triton
|
||||
if get_spec().speculative_use_block_verification:
|
||||
sampling_fn = partial(sampling_fn, block_verification=True)
|
||||
else:
|
||||
if _is_cuda:
|
||||
from sglang.kernels.ops.speculative.sampling import (
|
||||
|
||||
Reference in New Issue
Block a user