[Refactor] spec: Move platform dispatch and Triton kernels into kernels/ops (#45504)
This commit is contained in:
@@ -47,6 +47,7 @@ _TRITON_KERNELS = [
|
||||
("pad", "unpad_draft_extend_output"),
|
||||
("pad", "seqlens_expand_triton"),
|
||||
("position", "compute_position_triton"),
|
||||
("table_qk_norm_rope", "table_qk_norm_rope_"),
|
||||
("dsv4.attn_metadata_kernels", "expand_prefill_causally"),
|
||||
("dsv4.attn_metadata_kernels", "build_page_table_positions"),
|
||||
("dsv4.attn_metadata_kernels", "build_causal_swa_page_indices"),
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _table_qk_norm_rope_kernel(
|
||||
qkv_ptr,
|
||||
q_weight_ptr,
|
||||
k_weight_ptr,
|
||||
cos_sin_ptr,
|
||||
pos_ptr,
|
||||
row_stride,
|
||||
q_size,
|
||||
NHQ: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
):
|
||||
t = tl.program_id(0).to(tl.int64)
|
||||
h = tl.program_id(1)
|
||||
pos = tl.load(pos_ptr + t).to(tl.int64)
|
||||
|
||||
HALF: tl.constexpr = D // 2
|
||||
half_ar = tl.arange(0, HALF)
|
||||
d_ar = tl.arange(0, D)
|
||||
cos = tl.load(cos_sin_ptr + pos * D + half_ar).to(tl.float32)
|
||||
sin = tl.load(cos_sin_ptr + pos * D + HALF + half_ar).to(tl.float32)
|
||||
|
||||
is_q = h < NHQ
|
||||
col0 = tl.where(is_q, h * D, q_size + (h - NHQ) * D).to(tl.int64)
|
||||
w_ptr = tl.where(is_q, q_weight_ptr.to(tl.int64), k_weight_ptr.to(tl.int64)).to(
|
||||
tl.pointer_type(tl.bfloat16)
|
||||
)
|
||||
|
||||
row = qkv_ptr + t * row_stride + col0
|
||||
x = tl.load(row + d_ar).to(tl.float32)
|
||||
ms = tl.sum(x * x, 0) / D
|
||||
inv = 1.0 / tl.sqrt(ms + EPS)
|
||||
w1 = tl.load(w_ptr + half_ar).to(tl.float32)
|
||||
w2 = tl.load(w_ptr + HALF + half_ar).to(tl.float32)
|
||||
x1 = tl.load(row + half_ar).to(tl.float32) * inv * w1
|
||||
x2 = tl.load(row + HALF + half_ar).to(tl.float32) * inv * w2
|
||||
x1 = x1.to(tl.bfloat16).to(tl.float32)
|
||||
x2 = x2.to(tl.bfloat16).to(tl.float32)
|
||||
o1 = x1 * cos - x2 * sin
|
||||
o2 = x2 * cos + x1 * sin
|
||||
tl.store(row + half_ar, o1.to(tl.bfloat16))
|
||||
tl.store(row + HALF + half_ar, o2.to(tl.bfloat16))
|
||||
|
||||
|
||||
def table_qk_norm_rope_(
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
num_q_heads: int,
|
||||
num_k_heads: int,
|
||||
head_dim: int,
|
||||
eps: float,
|
||||
) -> None:
|
||||
"""In-place QK RMSNorm + table-lookup neox RoPE on the fused QKV tensor.
|
||||
|
||||
Reads cos/sin from the SAME rotary table as the unfused path, so there is
|
||||
no large-position angle drift (unlike theta-recompute kernels). V columns
|
||||
are untouched.
|
||||
"""
|
||||
T = qkv.shape[0]
|
||||
if T == 0:
|
||||
return
|
||||
grid = (T, num_q_heads + num_k_heads)
|
||||
_table_qk_norm_rope_kernel[grid](
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
qkv.stride(0),
|
||||
num_q_heads * head_dim,
|
||||
NHQ=num_q_heads,
|
||||
D=head_dim,
|
||||
EPS=eps,
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Top-k / top-p probability renormalization, resolved once per platform.
|
||||
|
||||
A name is ``None`` where the platform has no kernel for it.
|
||||
"""
|
||||
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_musa, is_npu
|
||||
from sglang.utils import LazyImport
|
||||
|
||||
if is_cuda():
|
||||
# Resolved on first call: CUDA builds without FlashInfer must still import
|
||||
# this module.
|
||||
top_k_renorm_prob = LazyImport("flashinfer.sampling", "top_k_renorm_probs")
|
||||
top_p_renorm_prob = LazyImport("flashinfer.sampling", "top_p_renorm_probs")
|
||||
elif is_hip():
|
||||
from sglang.kernels.ops.sampling.renorm_triton import (
|
||||
top_k_renorm_probs_triton as top_k_renorm_prob,
|
||||
)
|
||||
from sglang.kernels.ops.sampling.renorm_triton import (
|
||||
top_p_renorm_probs_triton as top_p_renorm_prob,
|
||||
)
|
||||
elif is_npu():
|
||||
try:
|
||||
from sgl_kernel_npu.sample import top_k_renorm_prob, top_p_renorm_prob
|
||||
except ImportError:
|
||||
top_k_renorm_prob = None
|
||||
top_p_renorm_prob = None
|
||||
elif is_musa():
|
||||
try:
|
||||
from sgl_kernel import top_k_renorm_prob, top_p_renorm_prob
|
||||
except Exception:
|
||||
top_k_renorm_prob = None
|
||||
top_p_renorm_prob = None
|
||||
else:
|
||||
top_k_renorm_prob = None
|
||||
top_p_renorm_prob = None
|
||||
|
||||
__all__ = ["top_k_renorm_prob", "top_p_renorm_prob"]
|
||||
@@ -10,17 +10,23 @@ from sglang.kernels.spec import CUDA_ONLY, KernelBackend, KernelSpec
|
||||
|
||||
# (module, public_fn) migrated from speculative/triton_ops.
|
||||
_TRITON_KERNELS = [
|
||||
("cache_locs", "assign_draft_cache_locs_contiguous_func"),
|
||||
("cache_locs", "assign_extend_cache_locs_func"),
|
||||
("cache_locs", "assign_extend_cache_locs_uniform_func"),
|
||||
("cache_locs", "fill_front_out_cache_loc_func"),
|
||||
("cache_locs", "generate_draft_decode_kv_indices"),
|
||||
("cache_locs", "generate_draft_decode_window_kv_indices"),
|
||||
("dflash", "compute_dflash_correct_drafts_and_bonus"),
|
||||
("eagle", "fill_bonus_tokens"),
|
||||
("eagle", "fill_accept_out_cache_loc"),
|
||||
("gather_spec_extras", "gather_spec_extras"),
|
||||
("multi_layer_eagle", "rotate_input_ids"),
|
||||
("spec_tree", "sgl_build_tree_kernel_efficient_triton"),
|
||||
("spec_tree", "verify_tree_greedy_kernel_triton"),
|
||||
("spec_tree", "build_tree_kernel_efficient_func"),
|
||||
("spec_tree", "verify_tree_greedy_func"),
|
||||
("topk1", "draft_topk1_postprocess"),
|
||||
("uno_tree", "build_uno_tree_proposal"),
|
||||
("ragged_verify_kernels", "pad_verify_lens_to_bucket"),
|
||||
("ragged_verify_kernels", "build_qo_indptr"),
|
||||
("lilicorr", "lilicorr_topk_lse"),
|
||||
@@ -28,6 +34,7 @@ _TRITON_KERNELS = [
|
||||
("reject_sampling", "chain_speculative_sampling_triton"),
|
||||
("row_argmax", "row_argmax"),
|
||||
("reconstruct_tree", "reconstruct_indices_from_tree_mask_triton"),
|
||||
("reconstruct_tree", "reconstruct_indices_from_tree_mask_func"),
|
||||
]
|
||||
for _mod, _fn in _TRITON_KERNELS:
|
||||
register_kernel(
|
||||
|
||||
@@ -25,7 +25,10 @@ _is_musa = is_musa()
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
if _is_cpu:
|
||||
from sgl_kernel import assign_extend_cache_locs_cpu
|
||||
from sgl_kernel import (
|
||||
assign_draft_cache_locs_contiguous_cpu,
|
||||
assign_extend_cache_locs_cpu,
|
||||
)
|
||||
|
||||
|
||||
def _get_oot_speculative_cache_locs_fn() -> Callable[..., torch.Tensor] | None:
|
||||
@@ -79,6 +82,47 @@ def assign_draft_cache_locs_contiguous(
|
||||
)
|
||||
|
||||
|
||||
def assign_draft_cache_locs_contiguous_func(
|
||||
req_pool_indices: torch.Tensor,
|
||||
req_to_token: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
out_cache_loc: torch.Tensor, # mutable
|
||||
topk: int,
|
||||
num_cols: int,
|
||||
positions: Optional[torch.Tensor] = None, # mutable
|
||||
mrope: Optional[torch.Tensor] = None, # mutable
|
||||
):
|
||||
"""Copy each request's ``topk * num_cols`` slots from ``seq_lens`` on, and
|
||||
write the draft positions when ``positions`` is given."""
|
||||
if _is_cpu:
|
||||
assert positions is None and mrope is None, "CPU writes no positions"
|
||||
assign_draft_cache_locs_contiguous_cpu(
|
||||
req_pool_indices,
|
||||
req_to_token,
|
||||
seq_lens,
|
||||
out_cache_loc,
|
||||
req_to_token.shape[1],
|
||||
topk,
|
||||
num_cols,
|
||||
)
|
||||
return
|
||||
bs = seq_lens.shape[0]
|
||||
assign_draft_cache_locs_contiguous[(bs,)](
|
||||
req_pool_indices,
|
||||
req_to_token,
|
||||
seq_lens,
|
||||
out_cache_loc,
|
||||
req_to_token.shape[1],
|
||||
topk,
|
||||
num_cols,
|
||||
positions=positions,
|
||||
mrope=mrope,
|
||||
BS=bs,
|
||||
WRITE_POSITIONS=positions is not None,
|
||||
WRITE_MROPE=mrope is not None,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr):
|
||||
"""Virtual token ids -> physical ones through the page-level v2p table,
|
||||
@@ -384,131 +428,6 @@ def generate_draft_decode_window_kv_indices(
|
||||
tl.store(window_kv_indptr + zid, base)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def align_evict_mask_to_page_size(
|
||||
seq_lens,
|
||||
evict_mask,
|
||||
page_size: tl.constexpr,
|
||||
num_draft_tokens: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
t_range = tl.arange(0, BLOCK_SIZE)
|
||||
|
||||
bid = tl.program_id(axis=0)
|
||||
seq_len = tl.load(seq_lens + bid)
|
||||
io_mask = t_range < num_draft_tokens
|
||||
mask_row = tl.load(
|
||||
evict_mask + bid * num_draft_tokens + t_range, mask=io_mask, other=0
|
||||
)
|
||||
|
||||
num_trues = tl.sum(mask_row)
|
||||
num_false = num_draft_tokens - num_trues
|
||||
|
||||
start = (seq_len + num_false - 1) // page_size * page_size - seq_len
|
||||
for i in range(max(start, 0), min(start + page_size, num_draft_tokens)):
|
||||
tl.store(evict_mask + bid * num_draft_tokens + i, False)
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, disable=_is_npu)
|
||||
def get_src_tgt_cache_loc(
|
||||
seq_lens: torch.Tensor,
|
||||
out_cache_loc: torch.Tensor,
|
||||
accept_index: torch.Tensor,
|
||||
num_correct_drafts: torch.Tensor,
|
||||
draft_token_num: int,
|
||||
page_size: int,
|
||||
):
|
||||
src_cache_loc = out_cache_loc[accept_index]
|
||||
# zeros_like, not empty_like: any uncovered tail stays at slot 0 (padding)
|
||||
# instead of caching-allocator garbage.
|
||||
tgt_cache_loc = torch.zeros_like(src_cache_loc)
|
||||
extended_len = seq_lens + draft_token_num
|
||||
keep_len = torch.minimum(
|
||||
(seq_lens + num_correct_drafts + 1 + page_size - 1) // page_size * page_size,
|
||||
extended_len,
|
||||
)
|
||||
to_free_num_slots = extended_len - keep_len
|
||||
return src_cache_loc, tgt_cache_loc, to_free_num_slots
|
||||
|
||||
|
||||
@triton.jit
|
||||
def get_target_cache_loc(
|
||||
tgt_cache_loc,
|
||||
to_free_slots,
|
||||
num_correct_drafts,
|
||||
to_free_num_slots,
|
||||
out_cache_loc,
|
||||
num_verify_tokens: tl.constexpr,
|
||||
num_verify_tokens_upper: tl.constexpr,
|
||||
bs_upper: tl.constexpr,
|
||||
):
|
||||
bid = tl.program_id(axis=0)
|
||||
offset = tl.arange(0, num_verify_tokens_upper)
|
||||
bs_offset = tl.arange(0, bs_upper)
|
||||
|
||||
# write the first part to tgt_cache_loc
|
||||
accept_len_all = tl.load(num_correct_drafts + bs_offset, mask=bs_offset < bid)
|
||||
tgt_cache_loc_start = tl.sum(accept_len_all) + bid
|
||||
copy_len = tl.load(num_correct_drafts + bid) + 1
|
||||
out_cache_loc_row = tl.load(
|
||||
out_cache_loc + bid * num_verify_tokens + offset, mask=offset < copy_len
|
||||
)
|
||||
tl.store(
|
||||
tgt_cache_loc + tgt_cache_loc_start + offset,
|
||||
out_cache_loc_row,
|
||||
mask=offset < copy_len,
|
||||
)
|
||||
|
||||
# write the second part to to_free_num_pages
|
||||
to_free_num_slots_all = tl.load(to_free_num_slots + bs_offset, mask=bs_offset < bid)
|
||||
to_free_num_slots_cur = tl.load(to_free_num_slots + bid)
|
||||
out_cache_loc_start = num_verify_tokens - to_free_num_slots_cur
|
||||
to_free_slots_start = tl.sum(to_free_num_slots_all)
|
||||
|
||||
copy_len = to_free_num_slots_cur
|
||||
out_cache_loc_row = tl.load(
|
||||
out_cache_loc + bid * num_verify_tokens + out_cache_loc_start + offset,
|
||||
mask=offset < copy_len,
|
||||
)
|
||||
tl.store(
|
||||
to_free_slots + to_free_slots_start + offset,
|
||||
out_cache_loc_row,
|
||||
mask=offset < copy_len,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def filter_finished_cache_loc_kernel(
|
||||
out_cache_loc,
|
||||
tgt_cache_loc,
|
||||
num_correct_drafts,
|
||||
num_accept_tokens_filter,
|
||||
bs_upper: tl.constexpr,
|
||||
num_verify_tokens_upper: tl.constexpr,
|
||||
):
|
||||
bid = tl.program_id(0)
|
||||
bs_offset = tl.arange(0, bs_upper)
|
||||
|
||||
num_correct_drafts_all = tl.load(
|
||||
num_correct_drafts + bs_offset, mask=bs_offset < bid
|
||||
)
|
||||
old_start = tl.sum(num_correct_drafts_all) + bid
|
||||
|
||||
num_accept_tokens_filter_all = tl.load(
|
||||
num_accept_tokens_filter + bs_offset, mask=bs_offset < bid
|
||||
)
|
||||
new_start = tl.sum(num_accept_tokens_filter_all)
|
||||
|
||||
copy_len = tl.load(num_accept_tokens_filter + bid)
|
||||
copy_offset = tl.arange(0, num_verify_tokens_upper)
|
||||
value = tl.load(
|
||||
tgt_cache_loc + old_start + copy_offset, mask=copy_offset < copy_len
|
||||
)
|
||||
tl.store(
|
||||
out_cache_loc + new_start + copy_offset, value, mask=copy_offset < copy_len
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def rebuild_compact_draft_req_to_token(
|
||||
draft_req_to_token,
|
||||
@@ -800,3 +719,34 @@ def assign_extend_cache_locs_func(
|
||||
)
|
||||
|
||||
return out_cache_loc
|
||||
|
||||
|
||||
def fill_front_out_cache_loc_func(
|
||||
num_front: torch.Tensor,
|
||||
out_cache_loc: torch.Tensor,
|
||||
front_out_cache_loc: torch.Tensor, # mutable
|
||||
):
|
||||
"""Pack the first ``num_front[i]`` slots of request i's window in ``out_cache_loc``."""
|
||||
bs = num_front.numel()
|
||||
window = out_cache_loc.view(bs, -1)
|
||||
rows = torch.arange(bs, device=window.device)
|
||||
starts = torch.zeros_like(num_front)
|
||||
if _is_cpu:
|
||||
assign_extend_cache_locs_cpu(
|
||||
rows,
|
||||
window.int(),
|
||||
starts,
|
||||
num_front,
|
||||
front_out_cache_loc,
|
||||
window.shape[1],
|
||||
)
|
||||
return
|
||||
assign_extend_cache_locs[(bs,)](
|
||||
rows,
|
||||
window,
|
||||
starts,
|
||||
num_front,
|
||||
front_out_cache_loc,
|
||||
window.shape[1],
|
||||
next_power_of_2(bs),
|
||||
)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from typing import Tuple
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
@@ -318,3 +320,82 @@ def selector_walk_triton(
|
||||
num_warps=1,
|
||||
)
|
||||
return tokens, q_rows
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_correct_drafts_and_bonus_kernel(
|
||||
candidates_ptr,
|
||||
target_predict_ptr,
|
||||
num_correct_drafts_ptr,
|
||||
bonus_tokens_ptr,
|
||||
block_size,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
b = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.arange(0, BLOCK)
|
||||
in_row = offs < block_size - 1
|
||||
drafts = tl.load(candidates_ptr + b * block_size + 1 + offs, mask=in_row, other=-1)
|
||||
targets = tl.load(target_predict_ptr + b * block_size + offs, mask=in_row, other=-2)
|
||||
eq = (drafts == targets) & in_row
|
||||
# Leading-match count = index of the first mismatch lane; lanes past the
|
||||
# row and all-match rows both resolve to block_size - 1 via the min.
|
||||
num_correct = tl.min(tl.where(eq, BLOCK, offs), 0)
|
||||
bonus_token = tl.load(target_predict_ptr + b * block_size + num_correct)
|
||||
tl.store(num_correct_drafts_ptr + b, num_correct.to(tl.int32))
|
||||
tl.store(bonus_tokens_ptr + b, bonus_token.to(tl.int64))
|
||||
|
||||
|
||||
def compute_dflash_correct_drafts_and_bonus(
|
||||
*,
|
||||
candidates: torch.Tensor,
|
||||
target_predict: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute DFlash accept lengths and bonus tokens (greedy verify rule).
|
||||
|
||||
Args:
|
||||
candidates: Token ids proposed by the DFlash draft, including the current token.
|
||||
Shape: [bs, block_size]. candidates[:, 0] is the current token.
|
||||
target_predict: Token ids predicted by the target model for each position in the block.
|
||||
Shape: [bs, block_size]. target_predict[:, t] corresponds to argmax at position t.
|
||||
|
||||
Returns:
|
||||
correct_len: int32 tensor [bs], number of accepted *draft* tokens (excluding current token and bonus token).
|
||||
bonus: int64 tensor [bs], the target-predicted token at index correct_len (the "bonus" token to append).
|
||||
|
||||
Notes:
|
||||
Matches the reference implementation rule:
|
||||
accept while candidates[:, 1:] == target_predict[:, :-1] consecutively.
|
||||
"""
|
||||
if candidates.ndim != 2:
|
||||
raise ValueError(f"candidates must be 2D, got shape={tuple(candidates.shape)}")
|
||||
if target_predict.shape != candidates.shape:
|
||||
raise ValueError(
|
||||
"target_predict must have the same shape as candidates. "
|
||||
f"candidates.shape={tuple(candidates.shape)}, target_predict.shape={tuple(target_predict.shape)}"
|
||||
)
|
||||
|
||||
bs, block_size = candidates.shape
|
||||
if bs <= 0:
|
||||
raise ValueError(f"batch size must be positive, got {bs}.")
|
||||
if block_size <= 0:
|
||||
raise ValueError(f"block_size must be positive, got {block_size}.")
|
||||
|
||||
if candidates.is_cuda:
|
||||
num_correct_drafts = torch.empty(
|
||||
bs, dtype=torch.int32, device=candidates.device
|
||||
)
|
||||
bonus_tokens = torch.empty(bs, dtype=torch.int64, device=candidates.device)
|
||||
_fused_correct_drafts_and_bonus_kernel[(bs,)](
|
||||
candidates.contiguous(),
|
||||
target_predict.contiguous(),
|
||||
num_correct_drafts,
|
||||
bonus_tokens,
|
||||
block_size,
|
||||
BLOCK=triton.next_power_of_2(max(block_size - 1, 1)),
|
||||
)
|
||||
return num_correct_drafts, bonus_tokens
|
||||
|
||||
matches = candidates[:, 1:] == target_predict[:, :-1]
|
||||
correct_len = matches.to(torch.int32).cumprod(dim=1).sum(dim=1)
|
||||
bonus = target_predict[torch.arange(bs, device=target_predict.device), correct_len]
|
||||
return correct_len.to(torch.int32), bonus.to(torch.int64)
|
||||
|
||||
@@ -8,12 +8,14 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.speculative.dflash import (
|
||||
compute_dflash_correct_drafts_and_bonus,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.dispatch import inputs_on_cuda
|
||||
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
|
||||
from sglang.srt.speculative.dflash_utils import (
|
||||
_get_or_create_chain_verify_buffers,
|
||||
build_speculative_verify_target_probs,
|
||||
compute_dflash_correct_drafts_and_bonus,
|
||||
)
|
||||
from sglang.srt.utils import is_npu
|
||||
|
||||
|
||||
@@ -40,7 +40,14 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.utils import next_power_of_2
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_xpu, next_power_of_2
|
||||
|
||||
_is_xpu = is_xpu()
|
||||
|
||||
if is_cuda() or is_hip():
|
||||
from sglang.kernels.ops.speculative.tree import reconstruct_indices_from_tree_mask
|
||||
elif not _is_xpu:
|
||||
from sgl_kernel.speculative import reconstruct_indices_from_tree_mask
|
||||
|
||||
# Sentinel for "no ancestor / not-yet-found child": any value >= n. Reductions use
|
||||
# n for min-searches (first child/sibling) and -1 for max-searches (parent).
|
||||
@@ -151,3 +158,37 @@ def reconstruct_indices_from_tree_mask_triton(
|
||||
block_n,
|
||||
num_warps=_num_warps_for(block_n),
|
||||
)
|
||||
|
||||
|
||||
def reconstruct_indices_from_tree_mask_func(
|
||||
tree_mask: torch.Tensor,
|
||||
verified_seq_len: torch.Tensor,
|
||||
positions: torch.Tensor, # mutable
|
||||
retrieve_index: torch.Tensor, # mutable
|
||||
retrieve_next_token: torch.Tensor, # mutable
|
||||
retrieve_next_sibling: torch.Tensor, # mutable
|
||||
batch_size: int,
|
||||
draft_token_num: int,
|
||||
) -> None:
|
||||
if _is_xpu:
|
||||
reconstruct_indices_from_tree_mask_triton(
|
||||
tree_mask,
|
||||
verified_seq_len,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
batch_size,
|
||||
draft_token_num,
|
||||
)
|
||||
return
|
||||
reconstruct_indices_from_tree_mask(
|
||||
tree_mask,
|
||||
verified_seq_len,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
batch_size,
|
||||
draft_token_num,
|
||||
)
|
||||
|
||||
@@ -11,9 +11,41 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
from enum import IntEnum
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_musa, is_npu, is_xpu
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_hip = is_hip()
|
||||
_is_musa = is_musa()
|
||||
_is_npu = is_npu()
|
||||
_is_xpu = is_xpu()
|
||||
_is_cpu = is_cpu()
|
||||
|
||||
if _is_cuda or _is_hip:
|
||||
from sglang.kernels.ops.speculative.tree import (
|
||||
build_tree_kernel_efficient as sgl_build_tree_kernel_efficient,
|
||||
)
|
||||
elif _is_musa:
|
||||
from sgl_kernel import (
|
||||
build_tree_kernel_efficient as sgl_build_tree_kernel_efficient,
|
||||
)
|
||||
elif _is_cpu:
|
||||
from sgl_kernel import (
|
||||
build_tree_kernel_efficient_cpu as sgl_build_tree_kernel_efficient_cpu,
|
||||
)
|
||||
from sgl_kernel import verify_tree_greedy_cpu as sgl_verify_tree_greedy_cpu
|
||||
|
||||
|
||||
class TreeMaskMode(IntEnum):
|
||||
FULL_MASK = 0
|
||||
QLEN_ONLY = 1
|
||||
QLEN_ONLY_BITPACKING = 2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def sgl_build_tree_kernel_efficient_triton(
|
||||
@@ -279,3 +311,232 @@ def verify_tree_greedy_kernel_triton(
|
||||
target_predict_ptr + target_row * num_draft_tokens + target_col
|
||||
)
|
||||
tl.store(predicts_ptr + last_accept_retrieve_idx, final_target)
|
||||
|
||||
|
||||
def sgl_build_tree_kernel_triton(
|
||||
parent_list: torch.Tensor,
|
||||
selected_index: torch.Tensor,
|
||||
verified_seq_len: torch.Tensor,
|
||||
tree_mask: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
retrieve_index: torch.Tensor,
|
||||
retrieve_next_token: torch.Tensor,
|
||||
retrieve_next_sibling: torch.Tensor,
|
||||
topk: int,
|
||||
depth: int,
|
||||
draft_token_num: int,
|
||||
tree_mask_mode: TreeMaskMode = TreeMaskMode.FULL_MASK,
|
||||
):
|
||||
"""Triton-based implementation."""
|
||||
# TODO: Add support for QLEN_ONLY_BITPACKING mode
|
||||
if tree_mask_mode == TreeMaskMode.QLEN_ONLY_BITPACKING:
|
||||
raise NotImplementedError(
|
||||
"QLEN_ONLY_BITPACKING is not supported in Triton implementation"
|
||||
)
|
||||
|
||||
batch_size = verified_seq_len.shape[0]
|
||||
seq_len_prefix_sum = torch.cumsum(verified_seq_len, dim=0) - verified_seq_len
|
||||
|
||||
# Launch kernel with one program per batch item
|
||||
grid = (batch_size,)
|
||||
|
||||
sgl_build_tree_kernel_efficient_triton[grid](
|
||||
parent_list,
|
||||
selected_index,
|
||||
verified_seq_len,
|
||||
seq_len_prefix_sum,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk=topk,
|
||||
depth=depth,
|
||||
draft_token_num=draft_token_num,
|
||||
tree_mask_mode=int(tree_mask_mode),
|
||||
batch_size=batch_size,
|
||||
parent_list_stride=(
|
||||
parent_list.stride(0) if parent_list.dim() > 1 else parent_list.shape[0]
|
||||
),
|
||||
selected_index_stride=selected_index.stride(0),
|
||||
)
|
||||
|
||||
|
||||
def build_tree_kernel_efficient_func(
|
||||
parent_list: torch.Tensor,
|
||||
top_scores_index: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
tree_mask: torch.Tensor, # mutable
|
||||
positions: torch.Tensor, # mutable
|
||||
retrieve_index: torch.Tensor, # mutable
|
||||
retrieve_next_token: torch.Tensor, # mutable
|
||||
retrieve_next_sibling: torch.Tensor, # mutable
|
||||
topk: int,
|
||||
spec_steps: int,
|
||||
num_verify_tokens: int,
|
||||
tree_mask_mode: TreeMaskMode,
|
||||
) -> None:
|
||||
if _is_npu:
|
||||
torch.ops.npu.build_tree_kernel_efficient(
|
||||
parent_list.to(dtype=torch.int64),
|
||||
top_scores_index,
|
||||
seq_lens,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk,
|
||||
spec_steps,
|
||||
num_verify_tokens,
|
||||
tree_mask_mode,
|
||||
)
|
||||
elif _is_xpu:
|
||||
sgl_build_tree_kernel_triton(
|
||||
parent_list,
|
||||
top_scores_index,
|
||||
seq_lens,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk,
|
||||
spec_steps,
|
||||
num_verify_tokens,
|
||||
tree_mask_mode,
|
||||
)
|
||||
elif _is_cpu:
|
||||
sgl_build_tree_kernel_efficient_cpu(
|
||||
parent_list,
|
||||
top_scores_index,
|
||||
seq_lens,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk,
|
||||
spec_steps,
|
||||
num_verify_tokens,
|
||||
tree_mask_mode,
|
||||
)
|
||||
else:
|
||||
sgl_build_tree_kernel_efficient(
|
||||
parent_list,
|
||||
top_scores_index,
|
||||
seq_lens,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk,
|
||||
spec_steps,
|
||||
num_verify_tokens,
|
||||
tree_mask_mode,
|
||||
)
|
||||
|
||||
|
||||
def verify_tree_greedy_triton(
|
||||
predicts: torch.Tensor,
|
||||
accept_index: torch.Tensor,
|
||||
accept_token_num: torch.Tensor,
|
||||
candidates: torch.Tensor,
|
||||
retrieve_index: torch.Tensor,
|
||||
retrieve_next_token: torch.Tensor,
|
||||
retrieve_next_sibling: torch.Tensor,
|
||||
target_predict: torch.Tensor,
|
||||
):
|
||||
"""Triton-based implementation."""
|
||||
batch_size = candidates.shape[0]
|
||||
num_speculative_tokens = accept_index.shape[1]
|
||||
num_draft_tokens = candidates.shape[1]
|
||||
|
||||
# Launch kernel with one program per batch item
|
||||
grid = (batch_size,)
|
||||
|
||||
verify_tree_greedy_kernel_triton[grid](
|
||||
predicts,
|
||||
accept_index,
|
||||
accept_token_num,
|
||||
candidates,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
target_predict,
|
||||
batch_size=batch_size,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
)
|
||||
|
||||
|
||||
def verify_tree_greedy_func(
|
||||
predicts: torch.Tensor,
|
||||
accept_index: torch.Tensor,
|
||||
accept_token_num: torch.Tensor,
|
||||
candidates: torch.Tensor,
|
||||
retrieve_index: torch.Tensor,
|
||||
retrieve_next_token: torch.Tensor,
|
||||
retrieve_next_sibling: torch.Tensor,
|
||||
target_predict: torch.Tensor,
|
||||
topk: int = -1,
|
||||
):
|
||||
if _is_cuda or _is_hip or _is_musa:
|
||||
if _is_cuda or _is_hip:
|
||||
from sglang.kernels.ops.speculative.tree import verify_tree_greedy
|
||||
else:
|
||||
from sgl_kernel import verify_tree_greedy
|
||||
|
||||
verify_tree_greedy(
|
||||
predicts=predicts, # mutable
|
||||
accept_index=accept_index, # mutable
|
||||
accept_token_num=accept_token_num, # mutable
|
||||
candidates=candidates,
|
||||
# kwarg LHS retained as `retrive_*` to match sgl_kernel op schema.
|
||||
retrive_index=retrieve_index,
|
||||
retrive_next_token=retrieve_next_token,
|
||||
retrive_next_sibling=retrieve_next_sibling,
|
||||
target_predict=target_predict,
|
||||
)
|
||||
|
||||
elif _is_cpu:
|
||||
sgl_verify_tree_greedy_cpu(
|
||||
predicts=predicts, # mutable
|
||||
accept_index=accept_index, # mutable
|
||||
accept_token_num=accept_token_num, # mutable
|
||||
candidates=candidates,
|
||||
# kwarg LHS retained as `retrive_*` to match the CUDA op schema, so
|
||||
# the CPU/CUDA call sites stay grep-symmetric.
|
||||
retrive_index=retrieve_index,
|
||||
retrive_next_token=retrieve_next_token,
|
||||
retrive_next_sibling=retrieve_next_sibling,
|
||||
target_predict=target_predict,
|
||||
)
|
||||
|
||||
elif _is_npu:
|
||||
from sgl_kernel_npu.sample.verify_tree_greedy import verify_tree_greedy
|
||||
|
||||
verify_tree_greedy(
|
||||
predicts=predicts,
|
||||
accept_index=accept_index,
|
||||
accept_token_num=accept_token_num,
|
||||
candidates=candidates,
|
||||
# kwarg LHS retained as `retrive_*` to match sgl_kernel op schema.
|
||||
retrive_index=retrieve_index,
|
||||
retrive_next_token=retrieve_next_token,
|
||||
retrive_next_sibling=retrieve_next_sibling,
|
||||
target_predict=target_predict,
|
||||
)
|
||||
elif _is_xpu:
|
||||
verify_tree_greedy_triton(
|
||||
predicts=predicts,
|
||||
accept_index=accept_index,
|
||||
accept_token_num=accept_token_num,
|
||||
candidates=candidates,
|
||||
retrieve_index=retrieve_index,
|
||||
retrieve_next_token=retrieve_next_token,
|
||||
retrieve_next_sibling=retrieve_next_sibling,
|
||||
target_predict=target_predict,
|
||||
)
|
||||
return predicts, accept_index, accept_token_num
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Speculative verify sampling ops, resolved once per platform.
|
||||
|
||||
A name is ``None`` where the platform has no kernel for it.
|
||||
"""
|
||||
|
||||
from sglang.srt.utils import is_cuda, is_musa, is_npu
|
||||
|
||||
if is_npu():
|
||||
try:
|
||||
from sgl_kernel_npu.sample import (
|
||||
chain_speculative_sampling_triton,
|
||||
tree_speculative_sampling_target_only,
|
||||
)
|
||||
except ImportError:
|
||||
chain_speculative_sampling_triton = None
|
||||
tree_speculative_sampling_target_only = None
|
||||
else:
|
||||
from sglang.kernels.ops.speculative.reject_sampling import (
|
||||
chain_speculative_sampling_triton,
|
||||
)
|
||||
|
||||
if is_cuda():
|
||||
from sglang.kernels.ops.speculative.sampling import (
|
||||
tree_speculative_sampling_target_only,
|
||||
)
|
||||
elif is_musa():
|
||||
try:
|
||||
from sgl_kernel import tree_speculative_sampling_target_only
|
||||
except Exception:
|
||||
tree_speculative_sampling_target_only = None
|
||||
else:
|
||||
tree_speculative_sampling_target_only = None
|
||||
|
||||
__all__ = ["chain_speculative_sampling_triton", "tree_speculative_sampling_target_only"]
|
||||
@@ -369,7 +369,9 @@ class DFlashAttention(nn.Module):
|
||||
if _is_npu:
|
||||
q, k, v = self.forward_prepare_npu(positions, hidden_states)
|
||||
elif self.use_table_qk_norm_rope and qkv.dtype == torch.bfloat16:
|
||||
from sglang.srt.speculative.dflash_utils import table_qk_norm_rope_
|
||||
from sglang.kernels.ops.attention.table_qk_norm_rope import (
|
||||
table_qk_norm_rope_,
|
||||
)
|
||||
|
||||
table_qk_norm_rope_(
|
||||
qkv,
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Callable, Optional
|
||||
from typing import Optional
|
||||
|
||||
|
||||
class DraftMeshMessageType(str, Enum):
|
||||
@@ -23,23 +23,6 @@ class DraftReqKey:
|
||||
request_id: str
|
||||
|
||||
|
||||
def build_draft_scheduler_rid(draft_key: DraftReqKey) -> str:
|
||||
return f"draft:{int(draft_key.src_verifier_rank)}:{draft_key.request_id}"
|
||||
|
||||
|
||||
def parse_draft_scheduler_rid(rid: str) -> DraftReqKey:
|
||||
if rid.startswith("draft:"):
|
||||
encoded = rid[len("draft:") :]
|
||||
rank_text, sep, request_id = encoded.partition(":")
|
||||
if sep and request_id:
|
||||
return DraftReqKey(
|
||||
src_verifier_rank=int(rank_text),
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
raise ValueError(f"Invalid decoupled draft scheduler rid: {rid}")
|
||||
|
||||
|
||||
@dataclass
|
||||
class DraftSync:
|
||||
"""Open or re-open a drafter request from a verifier-owned prefix.
|
||||
@@ -246,114 +229,6 @@ class VerifierCommitSegment:
|
||||
return prefix_segment
|
||||
|
||||
|
||||
@dataclass
|
||||
class DraftControlInbox:
|
||||
"""Drafter-side inbox for verifier control messages.
|
||||
|
||||
The TokenSync thread temporarily stores incoming control messages here.
|
||||
The drafter scheduler extracts and consumes them each time it finishes a decoding step.
|
||||
"""
|
||||
|
||||
sync_messages: list[DraftSync] = field(default_factory=list)
|
||||
verifier_commit_segments: dict[DraftReqKey, VerifierCommitSegment] = field(
|
||||
default_factory=dict
|
||||
)
|
||||
close_keys: set[DraftReqKey] = field(default_factory=set)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return (
|
||||
not self.sync_messages
|
||||
and not self.verifier_commit_segments
|
||||
and not self.close_keys
|
||||
)
|
||||
|
||||
def pending_control_count(self) -> int:
|
||||
return (
|
||||
len(self.sync_messages)
|
||||
+ len(self.verifier_commit_segments)
|
||||
+ len(self.close_keys)
|
||||
)
|
||||
|
||||
def add_control_batch_locked(self, batch: DraftControlBatch) -> None:
|
||||
for message in batch.close_messages:
|
||||
self.add_close_key_locked(message.draft_key)
|
||||
for message in batch.sync_messages:
|
||||
if message.draft_key not in self.close_keys:
|
||||
self.sync_messages.append(message)
|
||||
for message in batch.verify_commit_messages:
|
||||
self.add_verify_commit_locked(message)
|
||||
|
||||
def add_close_key_locked(self, draft_key: DraftReqKey) -> None:
|
||||
self.close_keys.add(draft_key)
|
||||
self.verifier_commit_segments.pop(draft_key, None)
|
||||
self.sync_messages = [
|
||||
message for message in self.sync_messages if message.draft_key != draft_key
|
||||
]
|
||||
|
||||
def add_verify_commit_locked(self, message: VerifyCommit) -> None:
|
||||
if message.draft_key in self.close_keys:
|
||||
return
|
||||
segment = self.verifier_commit_segments.get(message.draft_key)
|
||||
if segment is None:
|
||||
segment = VerifierCommitSegment(
|
||||
draft_key=message.draft_key,
|
||||
dst_drafter_rank=int(message.dst_drafter_rank),
|
||||
pre_verify_committed_len=int(message.pre_verify_committed_len),
|
||||
)
|
||||
segment.append_message(message)
|
||||
self.verifier_commit_segments[message.draft_key] = segment
|
||||
return
|
||||
segment.append_message(message)
|
||||
|
||||
def extract_ready_controls_locked(
|
||||
self,
|
||||
consumable_commit_len: Callable[[VerifierCommitSegment], int],
|
||||
) -> ReadyDraftControls:
|
||||
ready_controls = ReadyDraftControls()
|
||||
|
||||
if self.close_keys:
|
||||
ready_controls.close_keys = self.close_keys
|
||||
self.close_keys = set()
|
||||
|
||||
if self.sync_messages:
|
||||
ready_controls.sync_messages = self.sync_messages
|
||||
self.sync_messages = []
|
||||
|
||||
for draft_key, segment in list(self.verifier_commit_segments.items()):
|
||||
consumable_len = consumable_commit_len(segment)
|
||||
if consumable_len <= 0:
|
||||
continue
|
||||
|
||||
ready_controls.ready_commit_segments.append(
|
||||
segment.extract_prefix(consumable_len)
|
||||
)
|
||||
if not segment.committed_tokens:
|
||||
self.verifier_commit_segments.pop(draft_key, None)
|
||||
|
||||
return ready_controls
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReadyDraftControls:
|
||||
sync_messages: list[DraftSync] = field(default_factory=list)
|
||||
close_keys: set[DraftReqKey] = field(default_factory=set)
|
||||
ready_commit_segments: list[VerifierCommitSegment] = field(default_factory=list)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return (
|
||||
not self.sync_messages
|
||||
and not self.close_keys
|
||||
and not self.ready_commit_segments
|
||||
)
|
||||
|
||||
def extracted_control_count(self) -> int:
|
||||
return (
|
||||
len(self.sync_messages)
|
||||
+ len(self.close_keys)
|
||||
+ len(self.ready_commit_segments)
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DraftMeshMessage:
|
||||
message_type: DraftMeshMessageType
|
||||
|
||||
@@ -8,9 +8,11 @@ from typing import Any, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.kernels.ops.sampling.renorm import top_k_renorm_prob, top_p_renorm_prob
|
||||
from sglang.kernels.ops.speculative.verify_sampling import (
|
||||
tree_speculative_sampling_target_only,
|
||||
)
|
||||
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
|
||||
from sglang.srt.layers.sampler import (
|
||||
apply_custom_logit_processor,
|
||||
@@ -20,14 +22,12 @@ from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.model_executor.runner_utils.pool import borrow_graph_pool
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
from sglang.srt.speculative.spec_utils import sample_simulated_acc_len
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_musa, is_npu
|
||||
from sglang.utils import LazyImport
|
||||
from sglang.srt.utils import is_npu
|
||||
|
||||
DEFAULT_DFLASH_MASK_TOKEN = "<|MASK|>"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_DFLASH_SAMPLING_VERIFY_AVAILABLE = False
|
||||
_DFLASH_CHAIN_VERIFY_BUFFERS: dict[tuple[Optional[int], int], dict[str, Any]] = {}
|
||||
_DFLASH_VERIFY_SKIP_CUSTOM_MASK_BACKENDS = frozenset(
|
||||
{
|
||||
@@ -42,43 +42,10 @@ _DFLASH_VERIFY_SKIP_CUSTOM_MASK_BACKENDS = frozenset(
|
||||
)
|
||||
|
||||
|
||||
if is_cuda():
|
||||
from sglang.kernels.ops.speculative.sampling import (
|
||||
tree_speculative_sampling_target_only,
|
||||
)
|
||||
|
||||
# Resolved on first call: CUDA builds without FlashInfer must still import
|
||||
# this module.
|
||||
top_k_renorm_prob = LazyImport("flashinfer.sampling", "top_k_renorm_probs")
|
||||
top_p_renorm_prob = LazyImport("flashinfer.sampling", "top_p_renorm_probs")
|
||||
|
||||
_DFLASH_SAMPLING_VERIFY_AVAILABLE = True
|
||||
elif is_musa():
|
||||
try:
|
||||
from sgl_kernel import (
|
||||
top_k_renorm_prob,
|
||||
top_p_renorm_prob,
|
||||
tree_speculative_sampling_target_only,
|
||||
)
|
||||
|
||||
_DFLASH_SAMPLING_VERIFY_AVAILABLE = True
|
||||
except Exception:
|
||||
top_k_renorm_prob = None
|
||||
top_p_renorm_prob = None
|
||||
tree_speculative_sampling_target_only = None
|
||||
elif is_hip():
|
||||
from sglang.kernels.ops.sampling.renorm_triton import (
|
||||
top_k_renorm_probs_triton as top_k_renorm_prob,
|
||||
)
|
||||
from sglang.kernels.ops.sampling.renorm_triton import (
|
||||
top_p_renorm_probs_triton as top_p_renorm_prob,
|
||||
)
|
||||
|
||||
tree_speculative_sampling_target_only = None
|
||||
else:
|
||||
top_k_renorm_prob = None
|
||||
top_p_renorm_prob = None
|
||||
tree_speculative_sampling_target_only = None
|
||||
# DFlash does not run sampling verify on NPU, even where the op exists.
|
||||
_DFLASH_SAMPLING_VERIFY_AVAILABLE = (
|
||||
tree_speculative_sampling_target_only is not None and not is_npu()
|
||||
)
|
||||
|
||||
|
||||
def is_dflash_sampling_verify_available() -> bool:
|
||||
@@ -881,85 +848,6 @@ def can_dflash_use_fused_qkv_proj(qkv_proj: Any) -> Tuple[bool, str]:
|
||||
return True, ""
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fused_correct_drafts_and_bonus_kernel(
|
||||
candidates_ptr,
|
||||
target_predict_ptr,
|
||||
num_correct_drafts_ptr,
|
||||
bonus_tokens_ptr,
|
||||
block_size,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
b = tl.program_id(0).to(tl.int64)
|
||||
offs = tl.arange(0, BLOCK)
|
||||
in_row = offs < block_size - 1
|
||||
drafts = tl.load(candidates_ptr + b * block_size + 1 + offs, mask=in_row, other=-1)
|
||||
targets = tl.load(target_predict_ptr + b * block_size + offs, mask=in_row, other=-2)
|
||||
eq = (drafts == targets) & in_row
|
||||
# Leading-match count = index of the first mismatch lane; lanes past the
|
||||
# row and all-match rows both resolve to block_size - 1 via the min.
|
||||
num_correct = tl.min(tl.where(eq, BLOCK, offs), 0)
|
||||
bonus_token = tl.load(target_predict_ptr + b * block_size + num_correct)
|
||||
tl.store(num_correct_drafts_ptr + b, num_correct.to(tl.int32))
|
||||
tl.store(bonus_tokens_ptr + b, bonus_token.to(tl.int64))
|
||||
|
||||
|
||||
def compute_dflash_correct_drafts_and_bonus(
|
||||
*,
|
||||
candidates: torch.Tensor,
|
||||
target_predict: torch.Tensor,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute DFlash accept lengths and bonus tokens (greedy verify rule).
|
||||
|
||||
Args:
|
||||
candidates: Token ids proposed by the DFlash draft, including the current token.
|
||||
Shape: [bs, block_size]. candidates[:, 0] is the current token.
|
||||
target_predict: Token ids predicted by the target model for each position in the block.
|
||||
Shape: [bs, block_size]. target_predict[:, t] corresponds to argmax at position t.
|
||||
|
||||
Returns:
|
||||
correct_len: int32 tensor [bs], number of accepted *draft* tokens (excluding current token and bonus token).
|
||||
bonus: int64 tensor [bs], the target-predicted token at index correct_len (the "bonus" token to append).
|
||||
|
||||
Notes:
|
||||
Matches the reference implementation rule:
|
||||
accept while candidates[:, 1:] == target_predict[:, :-1] consecutively.
|
||||
"""
|
||||
if candidates.ndim != 2:
|
||||
raise ValueError(f"candidates must be 2D, got shape={tuple(candidates.shape)}")
|
||||
if target_predict.shape != candidates.shape:
|
||||
raise ValueError(
|
||||
"target_predict must have the same shape as candidates. "
|
||||
f"candidates.shape={tuple(candidates.shape)}, target_predict.shape={tuple(target_predict.shape)}"
|
||||
)
|
||||
|
||||
bs, block_size = candidates.shape
|
||||
if bs <= 0:
|
||||
raise ValueError(f"batch size must be positive, got {bs}.")
|
||||
if block_size <= 0:
|
||||
raise ValueError(f"block_size must be positive, got {block_size}.")
|
||||
|
||||
if candidates.is_cuda:
|
||||
num_correct_drafts = torch.empty(
|
||||
bs, dtype=torch.int32, device=candidates.device
|
||||
)
|
||||
bonus_tokens = torch.empty(bs, dtype=torch.int64, device=candidates.device)
|
||||
_fused_correct_drafts_and_bonus_kernel[(bs,)](
|
||||
candidates.contiguous(),
|
||||
target_predict.contiguous(),
|
||||
num_correct_drafts,
|
||||
bonus_tokens,
|
||||
block_size,
|
||||
BLOCK=triton.next_power_of_2(max(block_size - 1, 1)),
|
||||
)
|
||||
return num_correct_drafts, bonus_tokens
|
||||
|
||||
matches = candidates[:, 1:] == target_predict[:, :-1]
|
||||
correct_len = matches.to(torch.int32).cumprod(dim=1).sum(dim=1)
|
||||
bonus = target_predict[torch.arange(bs, device=target_predict.device), correct_len]
|
||||
return correct_len.to(torch.int32), bonus.to(torch.int64)
|
||||
|
||||
|
||||
def apply_dflash_simulated_acceptance(
|
||||
*,
|
||||
candidates: torch.Tensor,
|
||||
@@ -1222,83 +1110,3 @@ def validate_dflash_request(req: Req, enable_overlap: bool) -> Optional[str]:
|
||||
return "DFLASH speculative decoding does not support return_hidden_states yet."
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _table_qk_norm_rope_kernel(
|
||||
qkv_ptr,
|
||||
q_weight_ptr,
|
||||
k_weight_ptr,
|
||||
cos_sin_ptr,
|
||||
pos_ptr,
|
||||
row_stride,
|
||||
q_size,
|
||||
NHQ: tl.constexpr,
|
||||
D: tl.constexpr,
|
||||
EPS: tl.constexpr,
|
||||
):
|
||||
t = tl.program_id(0).to(tl.int64)
|
||||
h = tl.program_id(1)
|
||||
pos = tl.load(pos_ptr + t).to(tl.int64)
|
||||
|
||||
HALF: tl.constexpr = D // 2
|
||||
half_ar = tl.arange(0, HALF)
|
||||
d_ar = tl.arange(0, D)
|
||||
cos = tl.load(cos_sin_ptr + pos * D + half_ar).to(tl.float32)
|
||||
sin = tl.load(cos_sin_ptr + pos * D + HALF + half_ar).to(tl.float32)
|
||||
|
||||
is_q = h < NHQ
|
||||
col0 = tl.where(is_q, h * D, q_size + (h - NHQ) * D).to(tl.int64)
|
||||
w_ptr = tl.where(is_q, q_weight_ptr.to(tl.int64), k_weight_ptr.to(tl.int64)).to(
|
||||
tl.pointer_type(tl.bfloat16)
|
||||
)
|
||||
|
||||
row = qkv_ptr + t * row_stride + col0
|
||||
x = tl.load(row + d_ar).to(tl.float32)
|
||||
ms = tl.sum(x * x, 0) / D
|
||||
inv = 1.0 / tl.sqrt(ms + EPS)
|
||||
w1 = tl.load(w_ptr + half_ar).to(tl.float32)
|
||||
w2 = tl.load(w_ptr + HALF + half_ar).to(tl.float32)
|
||||
x1 = tl.load(row + half_ar).to(tl.float32) * inv * w1
|
||||
x2 = tl.load(row + HALF + half_ar).to(tl.float32) * inv * w2
|
||||
x1 = x1.to(tl.bfloat16).to(tl.float32)
|
||||
x2 = x2.to(tl.bfloat16).to(tl.float32)
|
||||
o1 = x1 * cos - x2 * sin
|
||||
o2 = x2 * cos + x1 * sin
|
||||
tl.store(row + half_ar, o1.to(tl.bfloat16))
|
||||
tl.store(row + HALF + half_ar, o2.to(tl.bfloat16))
|
||||
|
||||
|
||||
def table_qk_norm_rope_(
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
num_q_heads: int,
|
||||
num_k_heads: int,
|
||||
head_dim: int,
|
||||
eps: float,
|
||||
) -> None:
|
||||
"""In-place QK RMSNorm + table-lookup neox RoPE on the fused QKV tensor.
|
||||
|
||||
Reads cos/sin from the SAME rotary table as the unfused path, so there is
|
||||
no large-position angle drift (unlike theta-recompute kernels). V columns
|
||||
are untouched.
|
||||
"""
|
||||
T = qkv.shape[0]
|
||||
if T == 0:
|
||||
return
|
||||
grid = (T, num_q_heads + num_k_heads)
|
||||
_table_qk_norm_rope_kernel[grid](
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
qkv.stride(0),
|
||||
num_q_heads * head_dim,
|
||||
NHQ=num_q_heads,
|
||||
D=head_dim,
|
||||
EPS=eps,
|
||||
)
|
||||
|
||||
@@ -13,6 +13,7 @@ from sglang.kernels.ops.speculative.cache_locs import (
|
||||
from sglang.kernels.ops.speculative.dflash import (
|
||||
_compute_dflash_accept_bonus_triton_unchecked,
|
||||
_prepare_dflash_draft_block_unchecked,
|
||||
compute_dflash_correct_drafts_and_bonus,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.dspark.dspark_accept import (
|
||||
accept_sampling,
|
||||
@@ -62,7 +63,6 @@ from sglang.srt.speculative.dflash_utils import (
|
||||
apply_dflash_simulated_acceptance,
|
||||
apply_dflash_verify_logits_adjustments,
|
||||
can_dflash_use_fused_qkv_proj,
|
||||
compute_dflash_correct_drafts_and_bonus,
|
||||
compute_dflash_sampling_correct_drafts_and_bonus,
|
||||
is_dense_head_weight,
|
||||
is_dflash_sampling_verify_available,
|
||||
|
||||
@@ -155,12 +155,6 @@ def read_draft_checkpoint_config(*, server_args: ServerArgs) -> DSparkDraftConfi
|
||||
return parse_dspark_draft_config(draft_hf_config=draft_hf_config)
|
||||
|
||||
|
||||
def read_draft_checkpoint_gamma(*, server_args: ServerArgs) -> Optional[int]:
|
||||
return read_draft_checkpoint_config(server_args=server_args).resolve_gamma(
|
||||
default=None
|
||||
)
|
||||
|
||||
|
||||
def checkpoint_bundles_dspark_draft(hf_config: Any) -> bool:
|
||||
"""The checkpoint carries a bundled DSpark draft head, marked by the
|
||||
prefixed dspark_* keys on the target hf config. Single source of truth
|
||||
|
||||
@@ -12,11 +12,13 @@ from typing import Callable, ContextManager, Iterator, Optional, Union
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.dflash import (
|
||||
compute_dflash_correct_drafts_and_bonus,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.kv_canary.runner.future_tensor import FutureTensors
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.sampling.sampling_params import TOP_K_ALL
|
||||
from sglang.srt.speculative.dflash_utils import compute_dflash_correct_drafts_and_bonus
|
||||
from sglang.srt.speculative.dspark_components.dspark_block_accept_estimator import (
|
||||
create_block_accept_estimate_recorder,
|
||||
)
|
||||
|
||||
@@ -2,16 +2,21 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from enum import IntEnum
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.sampling import softmax as sampling_softmax
|
||||
from sglang.kernels.ops.sampling.renorm import top_k_renorm_prob, top_p_renorm_prob
|
||||
from sglang.kernels.ops.speculative.spec_tree import (
|
||||
sgl_build_tree_kernel_efficient_triton,
|
||||
verify_tree_greedy_kernel_triton,
|
||||
TreeMaskMode,
|
||||
build_tree_kernel_efficient_func,
|
||||
verify_tree_greedy_func,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.verify_sampling import (
|
||||
chain_speculative_sampling_triton,
|
||||
tree_speculative_sampling_target_only,
|
||||
)
|
||||
from sglang.srt.hardware_backend.npu.dsv4.dsv4_common_hooks import (
|
||||
maybe_build_dsv4_verify_bundle,
|
||||
@@ -26,8 +31,6 @@ from sglang.srt.utils import (
|
||||
is_cpu,
|
||||
is_cuda,
|
||||
is_hip,
|
||||
is_musa,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
)
|
||||
from sglang.srt.utils.async_probe import maybe_detect_oob
|
||||
@@ -45,27 +48,11 @@ if TYPE_CHECKING:
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
_is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
_is_musa = is_musa()
|
||||
_is_xpu = is_xpu()
|
||||
_is_cpu = is_cpu()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
if _is_cuda or _is_hip:
|
||||
from sglang.kernels.ops.speculative.tree import (
|
||||
build_tree_kernel_efficient as sgl_build_tree_kernel_efficient,
|
||||
)
|
||||
elif _is_musa:
|
||||
from sgl_kernel import (
|
||||
build_tree_kernel_efficient as sgl_build_tree_kernel_efficient,
|
||||
)
|
||||
elif _is_cpu:
|
||||
from sgl_kernel import (
|
||||
build_tree_kernel_efficient_cpu as sgl_build_tree_kernel_efficient_cpu,
|
||||
)
|
||||
from sgl_kernel import verify_tree_greedy_cpu as sgl_verify_tree_greedy_cpu
|
||||
|
||||
|
||||
def per_step_draft_out_cache_loc(
|
||||
out_cache_loc: torch.Tensor,
|
||||
@@ -142,12 +129,6 @@ def organize_draft_results(
|
||||
return parent_list, top_scores_index, draft_tokens
|
||||
|
||||
|
||||
class TreeMaskMode(IntEnum):
|
||||
FULL_MASK = 0
|
||||
QLEN_ONLY = 1
|
||||
QLEN_ONLY_BITPACKING = 2
|
||||
|
||||
|
||||
def default_tree_mask_mode() -> TreeMaskMode:
|
||||
# The CPU verify attention kernel (intel_amx) consumes the qlen x qlen
|
||||
# QLEN_ONLY tree mask directly; FULL_MASK is for the GPU kernels.
|
||||
@@ -229,66 +210,20 @@ def build_tree_kernel_efficient(
|
||||
# then, positions = [7, 8, 8, 9]
|
||||
positions = torch.empty((bs * num_verify_tokens,), device=device, dtype=torch.long)
|
||||
|
||||
if _is_npu:
|
||||
torch.ops.npu.build_tree_kernel_efficient(
|
||||
parent_list.to(dtype=torch.int64),
|
||||
top_scores_index,
|
||||
seq_lens,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk,
|
||||
spec_steps,
|
||||
num_verify_tokens,
|
||||
tree_mask_mode,
|
||||
)
|
||||
elif _is_xpu:
|
||||
sgl_build_tree_kernel_triton(
|
||||
parent_list,
|
||||
top_scores_index,
|
||||
seq_lens,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk,
|
||||
spec_steps,
|
||||
num_verify_tokens,
|
||||
tree_mask_mode,
|
||||
)
|
||||
elif _is_cpu:
|
||||
sgl_build_tree_kernel_efficient_cpu(
|
||||
parent_list,
|
||||
top_scores_index,
|
||||
seq_lens,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk,
|
||||
spec_steps,
|
||||
num_verify_tokens,
|
||||
tree_mask_mode,
|
||||
)
|
||||
else:
|
||||
sgl_build_tree_kernel_efficient(
|
||||
parent_list,
|
||||
top_scores_index,
|
||||
seq_lens,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk,
|
||||
spec_steps,
|
||||
num_verify_tokens,
|
||||
tree_mask_mode,
|
||||
)
|
||||
build_tree_kernel_efficient_func(
|
||||
parent_list,
|
||||
top_scores_index,
|
||||
seq_lens,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk,
|
||||
spec_steps,
|
||||
num_verify_tokens,
|
||||
tree_mask_mode,
|
||||
)
|
||||
return (
|
||||
tree_mask,
|
||||
positions,
|
||||
@@ -299,159 +234,6 @@ def build_tree_kernel_efficient(
|
||||
)
|
||||
|
||||
|
||||
def sgl_build_tree_kernel_triton(
|
||||
parent_list: torch.Tensor,
|
||||
selected_index: torch.Tensor,
|
||||
verified_seq_len: torch.Tensor,
|
||||
tree_mask: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
retrieve_index: torch.Tensor,
|
||||
retrieve_next_token: torch.Tensor,
|
||||
retrieve_next_sibling: torch.Tensor,
|
||||
topk: int,
|
||||
depth: int,
|
||||
draft_token_num: int,
|
||||
tree_mask_mode: TreeMaskMode = TreeMaskMode.FULL_MASK,
|
||||
):
|
||||
"""Triton-based implementation."""
|
||||
# TODO: Add support for QLEN_ONLY_BITPACKING mode
|
||||
if tree_mask_mode == TreeMaskMode.QLEN_ONLY_BITPACKING:
|
||||
raise NotImplementedError(
|
||||
"QLEN_ONLY_BITPACKING is not supported in Triton implementation"
|
||||
)
|
||||
|
||||
batch_size = verified_seq_len.shape[0]
|
||||
seq_len_prefix_sum = torch.cumsum(verified_seq_len, dim=0) - verified_seq_len
|
||||
|
||||
# Launch kernel with one program per batch item
|
||||
grid = (batch_size,)
|
||||
|
||||
sgl_build_tree_kernel_efficient_triton[grid](
|
||||
parent_list,
|
||||
selected_index,
|
||||
verified_seq_len,
|
||||
seq_len_prefix_sum,
|
||||
tree_mask,
|
||||
positions,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
topk=topk,
|
||||
depth=depth,
|
||||
draft_token_num=draft_token_num,
|
||||
tree_mask_mode=int(tree_mask_mode),
|
||||
batch_size=batch_size,
|
||||
parent_list_stride=(
|
||||
parent_list.stride(0) if parent_list.dim() > 1 else parent_list.shape[0]
|
||||
),
|
||||
selected_index_stride=selected_index.stride(0),
|
||||
)
|
||||
|
||||
|
||||
def verify_tree_greedy_triton(
|
||||
predicts: torch.Tensor,
|
||||
accept_index: torch.Tensor,
|
||||
accept_token_num: torch.Tensor,
|
||||
candidates: torch.Tensor,
|
||||
retrieve_index: torch.Tensor,
|
||||
retrieve_next_token: torch.Tensor,
|
||||
retrieve_next_sibling: torch.Tensor,
|
||||
target_predict: torch.Tensor,
|
||||
):
|
||||
"""Triton-based implementation."""
|
||||
batch_size = candidates.shape[0]
|
||||
num_speculative_tokens = accept_index.shape[1]
|
||||
num_draft_tokens = candidates.shape[1]
|
||||
|
||||
# Launch kernel with one program per batch item
|
||||
grid = (batch_size,)
|
||||
|
||||
verify_tree_greedy_kernel_triton[grid](
|
||||
predicts,
|
||||
accept_index,
|
||||
accept_token_num,
|
||||
candidates,
|
||||
retrieve_index,
|
||||
retrieve_next_token,
|
||||
retrieve_next_sibling,
|
||||
target_predict,
|
||||
batch_size=batch_size,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
)
|
||||
|
||||
|
||||
def verify_tree_greedy_func(
|
||||
predicts: torch.Tensor,
|
||||
accept_index: torch.Tensor,
|
||||
accept_token_num: torch.Tensor,
|
||||
candidates: torch.Tensor,
|
||||
retrieve_index: torch.Tensor,
|
||||
retrieve_next_token: torch.Tensor,
|
||||
retrieve_next_sibling: torch.Tensor,
|
||||
target_predict: torch.Tensor,
|
||||
topk: int = -1,
|
||||
):
|
||||
if _is_cuda or _is_hip or _is_musa:
|
||||
if _is_cuda or _is_hip:
|
||||
from sglang.kernels.ops.speculative.tree import verify_tree_greedy
|
||||
else:
|
||||
from sgl_kernel import verify_tree_greedy
|
||||
|
||||
verify_tree_greedy(
|
||||
predicts=predicts, # mutable
|
||||
accept_index=accept_index, # mutable
|
||||
accept_token_num=accept_token_num, # mutable
|
||||
candidates=candidates,
|
||||
# kwarg LHS retained as `retrive_*` to match sgl_kernel op schema.
|
||||
retrive_index=retrieve_index,
|
||||
retrive_next_token=retrieve_next_token,
|
||||
retrive_next_sibling=retrieve_next_sibling,
|
||||
target_predict=target_predict,
|
||||
)
|
||||
|
||||
elif _is_cpu:
|
||||
sgl_verify_tree_greedy_cpu(
|
||||
predicts=predicts, # mutable
|
||||
accept_index=accept_index, # mutable
|
||||
accept_token_num=accept_token_num, # mutable
|
||||
candidates=candidates,
|
||||
# kwarg LHS retained as `retrive_*` to match the CUDA op schema, so
|
||||
# the CPU/CUDA call sites stay grep-symmetric.
|
||||
retrive_index=retrieve_index,
|
||||
retrive_next_token=retrieve_next_token,
|
||||
retrive_next_sibling=retrieve_next_sibling,
|
||||
target_predict=target_predict,
|
||||
)
|
||||
|
||||
elif _is_npu:
|
||||
from sgl_kernel_npu.sample.verify_tree_greedy import verify_tree_greedy
|
||||
|
||||
verify_tree_greedy(
|
||||
predicts=predicts,
|
||||
accept_index=accept_index,
|
||||
accept_token_num=accept_token_num,
|
||||
candidates=candidates,
|
||||
# kwarg LHS retained as `retrive_*` to match sgl_kernel op schema.
|
||||
retrive_index=retrieve_index,
|
||||
retrive_next_token=retrieve_next_token,
|
||||
retrive_next_sibling=retrieve_next_sibling,
|
||||
target_predict=target_predict,
|
||||
)
|
||||
elif _is_xpu:
|
||||
verify_tree_greedy_triton(
|
||||
predicts=predicts,
|
||||
accept_index=accept_index,
|
||||
accept_token_num=accept_token_num,
|
||||
candidates=candidates,
|
||||
retrieve_index=retrieve_index,
|
||||
retrieve_next_token=retrieve_next_token,
|
||||
retrieve_next_sibling=retrieve_next_sibling,
|
||||
target_predict=target_predict,
|
||||
)
|
||||
return predicts, accept_index, accept_token_num
|
||||
|
||||
|
||||
def get_draft_input_from_target_hidden_dim(model_runner: ModelRunner) -> int:
|
||||
"""Width of the target hidden states fed into the draft model.
|
||||
|
||||
@@ -930,48 +712,13 @@ def eagle_sample(
|
||||
tp_group.broadcast(accept_index, src=0)
|
||||
tp_group.broadcast(num_correct_drafts, src=0)
|
||||
else:
|
||||
if _is_npu:
|
||||
from sgl_kernel_npu.sample import (
|
||||
chain_speculative_sampling_triton,
|
||||
top_k_renorm_prob,
|
||||
top_p_renorm_prob,
|
||||
tree_speculative_sampling_target_only,
|
||||
)
|
||||
else:
|
||||
from sglang.kernels.ops.speculative.reject_sampling import (
|
||||
chain_speculative_sampling_triton,
|
||||
)
|
||||
|
||||
# if/else, not a ternary: the CUDA-only name still has to resolve in the
|
||||
# branch not taken, and HIP only reaches here with rejection sampling on.
|
||||
if use_rejection_sampling:
|
||||
sampling_fn = chain_speculative_sampling_triton
|
||||
if get_spec().speculative_use_block_verification:
|
||||
sampling_fn = partial(sampling_fn, block_verification=True)
|
||||
else:
|
||||
if _is_cuda:
|
||||
from sglang.kernels.ops.speculative.sampling import (
|
||||
tree_speculative_sampling_target_only,
|
||||
)
|
||||
elif not _is_npu:
|
||||
from sgl_kernel import tree_speculative_sampling_target_only
|
||||
|
||||
sampling_fn = tree_speculative_sampling_target_only
|
||||
|
||||
if _is_hip:
|
||||
# Same names, same contract: dflash_utils.py aliases these too.
|
||||
from sglang.kernels.ops.sampling.renorm_triton import (
|
||||
top_k_renorm_probs_triton as top_k_renorm_prob,
|
||||
)
|
||||
from sglang.kernels.ops.sampling.renorm_triton import (
|
||||
top_p_renorm_probs_triton as top_p_renorm_prob,
|
||||
)
|
||||
elif _is_cuda:
|
||||
from flashinfer.sampling import top_k_renorm_probs as top_k_renorm_prob
|
||||
from flashinfer.sampling import top_p_renorm_probs as top_p_renorm_prob
|
||||
elif not _is_npu:
|
||||
from sgl_kernel import top_k_renorm_prob, top_p_renorm_prob
|
||||
|
||||
expanded_temperature = torch.repeat_interleave(
|
||||
sampling_info.temperatures, verify_input.draft_token_num, dim=0
|
||||
) # (bs * num_draft_tokens, 1)
|
||||
|
||||
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Optional
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
assign_draft_cache_locs_contiguous,
|
||||
assign_draft_cache_locs_contiguous_func,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.eagle import (
|
||||
build_chain_tree,
|
||||
@@ -49,9 +49,6 @@ from sglang.srt.utils.common import is_npu
|
||||
_is_cpu = is_cpu()
|
||||
_is_npu = is_npu()
|
||||
|
||||
if _is_cpu:
|
||||
from sgl_kernel import assign_draft_cache_locs_contiguous_cpu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.tp_worker import TpModelWorker
|
||||
@@ -281,40 +278,25 @@ def prepare_for_draft(
|
||||
dtype=torch.int64,
|
||||
device=batch.device,
|
||||
)
|
||||
if _is_cpu:
|
||||
assign_draft_cache_locs_contiguous_cpu(
|
||||
batch.req_pool_indices,
|
||||
req_to_token_pool.req_to_token,
|
||||
batch.seq_lens,
|
||||
batch.out_cache_loc,
|
||||
req_to_token_pool.req_to_token.shape[1],
|
||||
topk,
|
||||
num_steps,
|
||||
)
|
||||
else:
|
||||
if batch.seq_lens.is_cuda and topk == 1:
|
||||
prepared_positions = torch.empty_like(batch.seq_lens)
|
||||
if draft_model_runner.model_config.model_is_mrope and all(
|
||||
x is None for x in batch.multimodal_inputs
|
||||
):
|
||||
prepared_mrope_positions = torch.empty(
|
||||
(3, bs), dtype=torch.int64, device=batch.device
|
||||
)
|
||||
# FIXME(lsyin): align with the default code path
|
||||
assign_draft_cache_locs_contiguous[(bs,)](
|
||||
batch.req_pool_indices,
|
||||
req_to_token_pool.req_to_token,
|
||||
batch.seq_lens,
|
||||
batch.out_cache_loc,
|
||||
req_to_token_pool.req_to_token.shape[1],
|
||||
topk,
|
||||
window_cols,
|
||||
positions=prepared_positions,
|
||||
mrope=prepared_mrope_positions,
|
||||
BS=bs,
|
||||
WRITE_POSITIONS=prepared_positions is not None,
|
||||
WRITE_MROPE=prepared_mrope_positions is not None,
|
||||
)
|
||||
if batch.seq_lens.is_cuda and topk == 1:
|
||||
prepared_positions = torch.empty_like(batch.seq_lens)
|
||||
if draft_model_runner.model_config.model_is_mrope and all(
|
||||
x is None for x in batch.multimodal_inputs
|
||||
):
|
||||
prepared_mrope_positions = torch.empty(
|
||||
(3, bs), dtype=torch.int64, device=batch.device
|
||||
)
|
||||
# TODO(lsyin): align with the default code path
|
||||
assign_draft_cache_locs_contiguous_func(
|
||||
batch.req_pool_indices,
|
||||
req_to_token_pool.req_to_token,
|
||||
batch.seq_lens,
|
||||
batch.out_cache_loc,
|
||||
topk,
|
||||
window_cols,
|
||||
positions=prepared_positions,
|
||||
mrope=prepared_mrope_positions,
|
||||
)
|
||||
if plans_window:
|
||||
kv_loc_plan = target_translator.plan(
|
||||
req_pool_indices=batch.req_pool_indices,
|
||||
|
||||
@@ -7,6 +7,9 @@ import torch
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
assign_extend_cache_locs_func as assign_extend_cache_locs_func,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.reconstruct_tree import (
|
||||
reconstruct_indices_from_tree_mask_func,
|
||||
)
|
||||
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
from sglang.srt.managers.scheduler import GenerationBatchResult
|
||||
@@ -31,20 +34,11 @@ from sglang.srt.speculative.spec_utils import (
|
||||
prepare_mamba_track_for_verify,
|
||||
record_stream_for_v2_verify,
|
||||
)
|
||||
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_xpu
|
||||
from sglang.srt.utils import is_cpu
|
||||
from sglang.srt.utils.async_probe import maybe_detect_inf, maybe_detect_nan
|
||||
|
||||
_is_cpu = is_cpu()
|
||||
|
||||
if is_xpu():
|
||||
from sglang.kernels.ops.speculative.reconstruct_tree import (
|
||||
reconstruct_indices_from_tree_mask_triton as reconstruct_indices_from_tree_mask,
|
||||
)
|
||||
elif is_cuda() or is_hip():
|
||||
from sglang.kernels.ops.speculative.tree import reconstruct_indices_from_tree_mask
|
||||
else:
|
||||
from sgl_kernel.speculative import reconstruct_indices_from_tree_mask
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -345,7 +339,7 @@ class NGRAMWorker(BaseSpecWorker):
|
||||
self.grammar_tree_host = (mask, req_drafts) if batch.has_grammar else None
|
||||
|
||||
# generate positions and some indices using tree_mask
|
||||
reconstruct_indices_from_tree_mask(
|
||||
reconstruct_indices_from_tree_mask_func(
|
||||
tree_mask,
|
||||
batch.seq_lens,
|
||||
positions, # mutable
|
||||
|
||||
@@ -302,25 +302,3 @@ def compute_uniform_extend_lengths(
|
||||
num_tokens=num_tokens,
|
||||
extend_start_loc=None,
|
||||
)
|
||||
|
||||
|
||||
def compute_ragged_extend_lengths(
|
||||
*,
|
||||
seq_lens: torch.Tensor,
|
||||
seq_lens_cpu: List[int],
|
||||
ragged_layout: RaggedVerifyLayout,
|
||||
) -> VerifyExtendLengths:
|
||||
extend_seq_lens_cpu = list(ragged_layout.verify_lens_cpu)
|
||||
seq_lens_extended = seq_lens + ragged_layout.verify_lens
|
||||
seq_lens_cpu_extended = [
|
||||
raw + length for raw, length in zip(seq_lens_cpu, extend_seq_lens_cpu)
|
||||
]
|
||||
num_tokens = ragged_layout.total_verify_tokens
|
||||
extend_start_loc = ragged_layout.extend_start_loc
|
||||
return VerifyExtendLengths(
|
||||
seq_lens_extended=seq_lens_extended,
|
||||
seq_lens_cpu_extended=seq_lens_cpu_extended,
|
||||
extend_seq_lens_cpu=extend_seq_lens_cpu,
|
||||
num_tokens=num_tokens,
|
||||
extend_start_loc=extend_start_loc,
|
||||
)
|
||||
|
||||
@@ -12,13 +12,7 @@ from huggingface_hub import snapshot_download
|
||||
|
||||
from sglang.kernels.ops.sampling import softmax as sampling_softmax
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
align_evict_mask_to_page_size as align_evict_mask_to_page_size,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
assign_extend_cache_locs as assign_extend_cache_locs,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
filter_finished_cache_loc_kernel as filter_finished_cache_loc_kernel,
|
||||
fill_front_out_cache_loc_func as fill_front_out_cache_loc_func,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
generate_draft_decode_kv_indices as generate_draft_decode_kv_indices,
|
||||
@@ -26,12 +20,6 @@ from sglang.kernels.ops.speculative.cache_locs import (
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
generate_draft_decode_window_kv_indices as generate_draft_decode_window_kv_indices,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
get_src_tgt_cache_loc as get_src_tgt_cache_loc,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.cache_locs import (
|
||||
get_target_cache_loc as get_target_cache_loc,
|
||||
)
|
||||
from sglang.kernels.ops.speculative.eagle import (
|
||||
fill_accept_out_cache_loc_func as fill_accept_out_cache_loc_func,
|
||||
)
|
||||
@@ -48,9 +36,6 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
|
||||
from sglang.srt.managers.schedule_batch import set_mamba_track_indices_from_reqs
|
||||
from sglang.srt.managers.utils import _async_d2h
|
||||
from sglang.srt.mem_cache.allocation import (
|
||||
assign_req_to_token_pool as assign_req_to_token_pool,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocation import (
|
||||
assign_req_to_token_pool_func as assign_req_to_token_pool_func,
|
||||
)
|
||||
@@ -62,13 +47,11 @@ from sglang.srt.runtime_context import (
|
||||
max_speculative_num_draft_tokens,
|
||||
)
|
||||
from sglang.srt.utils import (
|
||||
is_cpu,
|
||||
is_cuda,
|
||||
is_hip,
|
||||
is_musa,
|
||||
is_npu,
|
||||
is_xpu,
|
||||
next_power_of_2,
|
||||
)
|
||||
from sglang.srt.utils.async_probe import maybe_detect_oob
|
||||
from sglang.srt.utils.common import fast_topk
|
||||
@@ -79,7 +62,6 @@ _is_hip = is_hip()
|
||||
_is_npu = is_npu()
|
||||
_is_musa = is_musa()
|
||||
_is_xpu = is_xpu()
|
||||
_is_cpu = is_cpu()
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.constrained.base_grammar_backend import BaseGrammarObject
|
||||
@@ -89,10 +71,6 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
|
||||
|
||||
if _is_cpu:
|
||||
from sgl_kernel import assign_extend_cache_locs_cpu
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -351,20 +329,6 @@ def spec_need_hidden_states() -> bool:
|
||||
return not spec.enable_multi_layer_eagle
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, disable=_is_npu or _is_xpu)
|
||||
def create_num_accept_tokens_filter(
|
||||
num_correct_drafts: torch.Tensor,
|
||||
unfinished_index_device: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
):
|
||||
num_accept_tokens_filter = torch.zeros_like(num_correct_drafts)
|
||||
num_accept_tokens_filter[unfinished_index_device] = (
|
||||
num_correct_drafts[unfinished_index_device] + 1
|
||||
)
|
||||
seq_lens.add_(num_correct_drafts + 1)
|
||||
return num_accept_tokens_filter
|
||||
|
||||
|
||||
def _select_top_k_tokens_first(
|
||||
topk_p: torch.Tensor,
|
||||
topk_index: torch.Tensor,
|
||||
@@ -826,28 +790,11 @@ def move_accept_tokens_to_target_kvcache(
|
||||
# The targets lead each request's verify window. Read them from that window,
|
||||
# not req_to_token: after verify's read-done the scheduler may hand a finished
|
||||
# request's row to a new request.
|
||||
window = batch.out_cache_loc.view(bs, -1)
|
||||
window_rows = torch.arange(bs, device=device)
|
||||
window_starts = torch.zeros_like(num_correct_drafts)
|
||||
if _is_cpu:
|
||||
assign_extend_cache_locs_cpu(
|
||||
window_rows,
|
||||
window.int(),
|
||||
window_starts,
|
||||
num_correct_drafts + 1,
|
||||
tgt_cache_loc,
|
||||
window.shape[1],
|
||||
)
|
||||
else:
|
||||
assign_extend_cache_locs[(bs,)](
|
||||
window_rows,
|
||||
window,
|
||||
window_starts,
|
||||
num_correct_drafts + 1,
|
||||
tgt_cache_loc,
|
||||
window.shape[1],
|
||||
next_power_of_2(bs),
|
||||
)
|
||||
fill_front_out_cache_loc_func(
|
||||
num_correct_drafts + 1,
|
||||
batch.out_cache_loc,
|
||||
tgt_cache_loc,
|
||||
)
|
||||
fill_accept_out_cache_loc_func(
|
||||
accept_index,
|
||||
batch.out_cache_loc,
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.ops.speculative.uno_tree import build_uno_tree_proposal
|
||||
from sglang.srt.managers.utils import GenerationBatchResult
|
||||
from sglang.srt.model_executor.forward_batch_info import (
|
||||
CaptureHiddenMode,
|
||||
@@ -32,7 +33,6 @@ from sglang.srt.speculative.uno_cuda_graph_runner import (
|
||||
UnoDecodeCudaGraphRunner,
|
||||
)
|
||||
from sglang.srt.speculative.uno_info import UnoDraftInput, UnoForwardInput
|
||||
from sglang.srt.speculative.uno_tree import build_uno_tree_proposal
|
||||
from sglang.srt.speculative.uno_utils import (
|
||||
build_uno_draft_input,
|
||||
pack_uno_tree_result,
|
||||
|
||||
Reference in New Issue
Block a user