[Refactor] spec: Move platform dispatch and Triton kernels into kernels/ops (#45504)

This commit is contained in:
Liangsheng Yin
2026-10-21 01:22:16 -07:00
committed by GitHub
parent 6cac4dc3fe
commit 6efdd33d07
26 changed files with 707 additions and 1168 deletions
@@ -47,6 +47,7 @@ _TRITON_KERNELS = [
("pad", "unpad_draft_extend_output"),
("pad", "seqlens_expand_triton"),
("position", "compute_position_triton"),
("table_qk_norm_rope", "table_qk_norm_rope_"),
("dsv4.attn_metadata_kernels", "expand_prefill_causally"),
("dsv4.attn_metadata_kernels", "build_page_table_positions"),
("dsv4.attn_metadata_kernels", "build_causal_swa_page_indices"),
@@ -0,0 +1,83 @@
import torch
import triton
import triton.language as tl
@triton.jit
def _table_qk_norm_rope_kernel(
qkv_ptr,
q_weight_ptr,
k_weight_ptr,
cos_sin_ptr,
pos_ptr,
row_stride,
q_size,
NHQ: tl.constexpr,
D: tl.constexpr,
EPS: tl.constexpr,
):
t = tl.program_id(0).to(tl.int64)
h = tl.program_id(1)
pos = tl.load(pos_ptr + t).to(tl.int64)
HALF: tl.constexpr = D // 2
half_ar = tl.arange(0, HALF)
d_ar = tl.arange(0, D)
cos = tl.load(cos_sin_ptr + pos * D + half_ar).to(tl.float32)
sin = tl.load(cos_sin_ptr + pos * D + HALF + half_ar).to(tl.float32)
is_q = h < NHQ
col0 = tl.where(is_q, h * D, q_size + (h - NHQ) * D).to(tl.int64)
w_ptr = tl.where(is_q, q_weight_ptr.to(tl.int64), k_weight_ptr.to(tl.int64)).to(
tl.pointer_type(tl.bfloat16)
)
row = qkv_ptr + t * row_stride + col0
x = tl.load(row + d_ar).to(tl.float32)
ms = tl.sum(x * x, 0) / D
inv = 1.0 / tl.sqrt(ms + EPS)
w1 = tl.load(w_ptr + half_ar).to(tl.float32)
w2 = tl.load(w_ptr + HALF + half_ar).to(tl.float32)
x1 = tl.load(row + half_ar).to(tl.float32) * inv * w1
x2 = tl.load(row + HALF + half_ar).to(tl.float32) * inv * w2
x1 = x1.to(tl.bfloat16).to(tl.float32)
x2 = x2.to(tl.bfloat16).to(tl.float32)
o1 = x1 * cos - x2 * sin
o2 = x2 * cos + x1 * sin
tl.store(row + half_ar, o1.to(tl.bfloat16))
tl.store(row + HALF + half_ar, o2.to(tl.bfloat16))
def table_qk_norm_rope_(
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
num_q_heads: int,
num_k_heads: int,
head_dim: int,
eps: float,
) -> None:
"""In-place QK RMSNorm + table-lookup neox RoPE on the fused QKV tensor.
Reads cos/sin from the SAME rotary table as the unfused path, so there is
no large-position angle drift (unlike theta-recompute kernels). V columns
are untouched.
"""
T = qkv.shape[0]
if T == 0:
return
grid = (T, num_q_heads + num_k_heads)
_table_qk_norm_rope_kernel[grid](
qkv,
q_weight,
k_weight,
cos_sin_cache,
positions,
qkv.stride(0),
num_q_heads * head_dim,
NHQ=num_q_heads,
D=head_dim,
EPS=eps,
)
@@ -0,0 +1,37 @@
"""Top-k / top-p probability renormalization, resolved once per platform.
A name is ``None`` where the platform has no kernel for it.
"""
from sglang.srt.utils import is_cuda, is_hip, is_musa, is_npu
from sglang.utils import LazyImport
if is_cuda():
# Resolved on first call: CUDA builds without FlashInfer must still import
# this module.
top_k_renorm_prob = LazyImport("flashinfer.sampling", "top_k_renorm_probs")
top_p_renorm_prob = LazyImport("flashinfer.sampling", "top_p_renorm_probs")
elif is_hip():
from sglang.kernels.ops.sampling.renorm_triton import (
top_k_renorm_probs_triton as top_k_renorm_prob,
)
from sglang.kernels.ops.sampling.renorm_triton import (
top_p_renorm_probs_triton as top_p_renorm_prob,
)
elif is_npu():
try:
from sgl_kernel_npu.sample import top_k_renorm_prob, top_p_renorm_prob
except ImportError:
top_k_renorm_prob = None
top_p_renorm_prob = None
elif is_musa():
try:
from sgl_kernel import top_k_renorm_prob, top_p_renorm_prob
except Exception:
top_k_renorm_prob = None
top_p_renorm_prob = None
else:
top_k_renorm_prob = None
top_p_renorm_prob = None
__all__ = ["top_k_renorm_prob", "top_p_renorm_prob"]
@@ -10,17 +10,23 @@ from sglang.kernels.spec import CUDA_ONLY, KernelBackend, KernelSpec
# (module, public_fn) migrated from speculative/triton_ops.
_TRITON_KERNELS = [
("cache_locs", "assign_draft_cache_locs_contiguous_func"),
("cache_locs", "assign_extend_cache_locs_func"),
("cache_locs", "assign_extend_cache_locs_uniform_func"),
("cache_locs", "fill_front_out_cache_loc_func"),
("cache_locs", "generate_draft_decode_kv_indices"),
("cache_locs", "generate_draft_decode_window_kv_indices"),
("dflash", "compute_dflash_correct_drafts_and_bonus"),
("eagle", "fill_bonus_tokens"),
("eagle", "fill_accept_out_cache_loc"),
("gather_spec_extras", "gather_spec_extras"),
("multi_layer_eagle", "rotate_input_ids"),
("spec_tree", "sgl_build_tree_kernel_efficient_triton"),
("spec_tree", "verify_tree_greedy_kernel_triton"),
("spec_tree", "build_tree_kernel_efficient_func"),
("spec_tree", "verify_tree_greedy_func"),
("topk1", "draft_topk1_postprocess"),
("uno_tree", "build_uno_tree_proposal"),
("ragged_verify_kernels", "pad_verify_lens_to_bucket"),
("ragged_verify_kernels", "build_qo_indptr"),
("lilicorr", "lilicorr_topk_lse"),
@@ -28,6 +34,7 @@ _TRITON_KERNELS = [
("reject_sampling", "chain_speculative_sampling_triton"),
("row_argmax", "row_argmax"),
("reconstruct_tree", "reconstruct_indices_from_tree_mask_triton"),
("reconstruct_tree", "reconstruct_indices_from_tree_mask_func"),
]
for _mod, _fn in _TRITON_KERNELS:
register_kernel(
@@ -25,7 +25,10 @@ _is_musa = is_musa()
_is_xpu = is_xpu()
if _is_cpu:
from sgl_kernel import assign_extend_cache_locs_cpu
from sgl_kernel import (
assign_draft_cache_locs_contiguous_cpu,
assign_extend_cache_locs_cpu,
)
def _get_oot_speculative_cache_locs_fn() -> Callable[..., torch.Tensor] | None:
@@ -79,6 +82,47 @@ def assign_draft_cache_locs_contiguous(
)
def assign_draft_cache_locs_contiguous_func(
req_pool_indices: torch.Tensor,
req_to_token: torch.Tensor,
seq_lens: torch.Tensor,
out_cache_loc: torch.Tensor, # mutable
topk: int,
num_cols: int,
positions: Optional[torch.Tensor] = None, # mutable
mrope: Optional[torch.Tensor] = None, # mutable
):
"""Copy each request's ``topk * num_cols`` slots from ``seq_lens`` on, and
write the draft positions when ``positions`` is given."""
if _is_cpu:
assert positions is None and mrope is None, "CPU writes no positions"
assign_draft_cache_locs_contiguous_cpu(
req_pool_indices,
req_to_token,
seq_lens,
out_cache_loc,
req_to_token.shape[1],
topk,
num_cols,
)
return
bs = seq_lens.shape[0]
assign_draft_cache_locs_contiguous[(bs,)](
req_pool_indices,
req_to_token,
seq_lens,
out_cache_loc,
req_to_token.shape[1],
topk,
num_cols,
positions=positions,
mrope=mrope,
BS=bs,
WRITE_POSITIONS=positions is not None,
WRITE_MROPE=mrope is not None,
)
@triton.jit
def _translate_token_ids(ids, v2p, mask, page_size: tl.constexpr):
"""Virtual token ids -> physical ones through the page-level v2p table,
@@ -384,131 +428,6 @@ def generate_draft_decode_window_kv_indices(
tl.store(window_kv_indptr + zid, base)
@triton.jit
def align_evict_mask_to_page_size(
seq_lens,
evict_mask,
page_size: tl.constexpr,
num_draft_tokens: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
t_range = tl.arange(0, BLOCK_SIZE)
bid = tl.program_id(axis=0)
seq_len = tl.load(seq_lens + bid)
io_mask = t_range < num_draft_tokens
mask_row = tl.load(
evict_mask + bid * num_draft_tokens + t_range, mask=io_mask, other=0
)
num_trues = tl.sum(mask_row)
num_false = num_draft_tokens - num_trues
start = (seq_len + num_false - 1) // page_size * page_size - seq_len
for i in range(max(start, 0), min(start + page_size, num_draft_tokens)):
tl.store(evict_mask + bid * num_draft_tokens + i, False)
@torch.compile(dynamic=True, disable=_is_npu)
def get_src_tgt_cache_loc(
seq_lens: torch.Tensor,
out_cache_loc: torch.Tensor,
accept_index: torch.Tensor,
num_correct_drafts: torch.Tensor,
draft_token_num: int,
page_size: int,
):
src_cache_loc = out_cache_loc[accept_index]
# zeros_like, not empty_like: any uncovered tail stays at slot 0 (padding)
# instead of caching-allocator garbage.
tgt_cache_loc = torch.zeros_like(src_cache_loc)
extended_len = seq_lens + draft_token_num
keep_len = torch.minimum(
(seq_lens + num_correct_drafts + 1 + page_size - 1) // page_size * page_size,
extended_len,
)
to_free_num_slots = extended_len - keep_len
return src_cache_loc, tgt_cache_loc, to_free_num_slots
@triton.jit
def get_target_cache_loc(
tgt_cache_loc,
to_free_slots,
num_correct_drafts,
to_free_num_slots,
out_cache_loc,
num_verify_tokens: tl.constexpr,
num_verify_tokens_upper: tl.constexpr,
bs_upper: tl.constexpr,
):
bid = tl.program_id(axis=0)
offset = tl.arange(0, num_verify_tokens_upper)
bs_offset = tl.arange(0, bs_upper)
# write the first part to tgt_cache_loc
accept_len_all = tl.load(num_correct_drafts + bs_offset, mask=bs_offset < bid)
tgt_cache_loc_start = tl.sum(accept_len_all) + bid
copy_len = tl.load(num_correct_drafts + bid) + 1
out_cache_loc_row = tl.load(
out_cache_loc + bid * num_verify_tokens + offset, mask=offset < copy_len
)
tl.store(
tgt_cache_loc + tgt_cache_loc_start + offset,
out_cache_loc_row,
mask=offset < copy_len,
)
# write the second part to to_free_num_pages
to_free_num_slots_all = tl.load(to_free_num_slots + bs_offset, mask=bs_offset < bid)
to_free_num_slots_cur = tl.load(to_free_num_slots + bid)
out_cache_loc_start = num_verify_tokens - to_free_num_slots_cur
to_free_slots_start = tl.sum(to_free_num_slots_all)
copy_len = to_free_num_slots_cur
out_cache_loc_row = tl.load(
out_cache_loc + bid * num_verify_tokens + out_cache_loc_start + offset,
mask=offset < copy_len,
)
tl.store(
to_free_slots + to_free_slots_start + offset,
out_cache_loc_row,
mask=offset < copy_len,
)
@triton.jit
def filter_finished_cache_loc_kernel(
out_cache_loc,
tgt_cache_loc,
num_correct_drafts,
num_accept_tokens_filter,
bs_upper: tl.constexpr,
num_verify_tokens_upper: tl.constexpr,
):
bid = tl.program_id(0)
bs_offset = tl.arange(0, bs_upper)
num_correct_drafts_all = tl.load(
num_correct_drafts + bs_offset, mask=bs_offset < bid
)
old_start = tl.sum(num_correct_drafts_all) + bid
num_accept_tokens_filter_all = tl.load(
num_accept_tokens_filter + bs_offset, mask=bs_offset < bid
)
new_start = tl.sum(num_accept_tokens_filter_all)
copy_len = tl.load(num_accept_tokens_filter + bid)
copy_offset = tl.arange(0, num_verify_tokens_upper)
value = tl.load(
tgt_cache_loc + old_start + copy_offset, mask=copy_offset < copy_len
)
tl.store(
out_cache_loc + new_start + copy_offset, value, mask=copy_offset < copy_len
)
@triton.jit
def rebuild_compact_draft_req_to_token(
draft_req_to_token,
@@ -800,3 +719,34 @@ def assign_extend_cache_locs_func(
)
return out_cache_loc
def fill_front_out_cache_loc_func(
num_front: torch.Tensor,
out_cache_loc: torch.Tensor,
front_out_cache_loc: torch.Tensor, # mutable
):
"""Pack the first ``num_front[i]`` slots of request i's window in ``out_cache_loc``."""
bs = num_front.numel()
window = out_cache_loc.view(bs, -1)
rows = torch.arange(bs, device=window.device)
starts = torch.zeros_like(num_front)
if _is_cpu:
assign_extend_cache_locs_cpu(
rows,
window.int(),
starts,
num_front,
front_out_cache_loc,
window.shape[1],
)
return
assign_extend_cache_locs[(bs,)](
rows,
window,
starts,
num_front,
front_out_cache_loc,
window.shape[1],
next_power_of_2(bs),
)
@@ -1,3 +1,5 @@
from typing import Tuple
import torch
import triton
import triton.language as tl
@@ -318,3 +320,82 @@ def selector_walk_triton(
num_warps=1,
)
return tokens, q_rows
@triton.jit
def _fused_correct_drafts_and_bonus_kernel(
candidates_ptr,
target_predict_ptr,
num_correct_drafts_ptr,
bonus_tokens_ptr,
block_size,
BLOCK: tl.constexpr,
):
b = tl.program_id(0).to(tl.int64)
offs = tl.arange(0, BLOCK)
in_row = offs < block_size - 1
drafts = tl.load(candidates_ptr + b * block_size + 1 + offs, mask=in_row, other=-1)
targets = tl.load(target_predict_ptr + b * block_size + offs, mask=in_row, other=-2)
eq = (drafts == targets) & in_row
# Leading-match count = index of the first mismatch lane; lanes past the
# row and all-match rows both resolve to block_size - 1 via the min.
num_correct = tl.min(tl.where(eq, BLOCK, offs), 0)
bonus_token = tl.load(target_predict_ptr + b * block_size + num_correct)
tl.store(num_correct_drafts_ptr + b, num_correct.to(tl.int32))
tl.store(bonus_tokens_ptr + b, bonus_token.to(tl.int64))
def compute_dflash_correct_drafts_and_bonus(
*,
candidates: torch.Tensor,
target_predict: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Compute DFlash accept lengths and bonus tokens (greedy verify rule).
Args:
candidates: Token ids proposed by the DFlash draft, including the current token.
Shape: [bs, block_size]. candidates[:, 0] is the current token.
target_predict: Token ids predicted by the target model for each position in the block.
Shape: [bs, block_size]. target_predict[:, t] corresponds to argmax at position t.
Returns:
correct_len: int32 tensor [bs], number of accepted *draft* tokens (excluding current token and bonus token).
bonus: int64 tensor [bs], the target-predicted token at index correct_len (the "bonus" token to append).
Notes:
Matches the reference implementation rule:
accept while candidates[:, 1:] == target_predict[:, :-1] consecutively.
"""
if candidates.ndim != 2:
raise ValueError(f"candidates must be 2D, got shape={tuple(candidates.shape)}")
if target_predict.shape != candidates.shape:
raise ValueError(
"target_predict must have the same shape as candidates. "
f"candidates.shape={tuple(candidates.shape)}, target_predict.shape={tuple(target_predict.shape)}"
)
bs, block_size = candidates.shape
if bs <= 0:
raise ValueError(f"batch size must be positive, got {bs}.")
if block_size <= 0:
raise ValueError(f"block_size must be positive, got {block_size}.")
if candidates.is_cuda:
num_correct_drafts = torch.empty(
bs, dtype=torch.int32, device=candidates.device
)
bonus_tokens = torch.empty(bs, dtype=torch.int64, device=candidates.device)
_fused_correct_drafts_and_bonus_kernel[(bs,)](
candidates.contiguous(),
target_predict.contiguous(),
num_correct_drafts,
bonus_tokens,
block_size,
BLOCK=triton.next_power_of_2(max(block_size - 1, 1)),
)
return num_correct_drafts, bonus_tokens
matches = candidates[:, 1:] == target_predict[:, :-1]
correct_len = matches.to(torch.int32).cumprod(dim=1).sum(dim=1)
bonus = target_predict[torch.arange(bs, device=target_predict.device), correct_len]
return correct_len.to(torch.int32), bonus.to(torch.int64)
@@ -8,12 +8,14 @@ import torch
import triton
import triton.language as tl
from sglang.kernels.ops.speculative.dflash import (
compute_dflash_correct_drafts_and_bonus,
)
from sglang.kernels.ops.speculative.dspark.dispatch import inputs_on_cuda
from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2
from sglang.srt.speculative.dflash_utils import (
_get_or_create_chain_verify_buffers,
build_speculative_verify_target_probs,
compute_dflash_correct_drafts_and_bonus,
)
from sglang.srt.utils import is_npu
@@ -40,7 +40,14 @@ import torch
import triton
import triton.language as tl
from sglang.srt.utils import next_power_of_2
from sglang.srt.utils import is_cuda, is_hip, is_xpu, next_power_of_2
_is_xpu = is_xpu()
if is_cuda() or is_hip():
from sglang.kernels.ops.speculative.tree import reconstruct_indices_from_tree_mask
elif not _is_xpu:
from sgl_kernel.speculative import reconstruct_indices_from_tree_mask
# Sentinel for "no ancestor / not-yet-found child": any value >= n. Reductions use
# n for min-searches (first child/sibling) and -1 for max-searches (parent).
@@ -151,3 +158,37 @@ def reconstruct_indices_from_tree_mask_triton(
block_n,
num_warps=_num_warps_for(block_n),
)
def reconstruct_indices_from_tree_mask_func(
tree_mask: torch.Tensor,
verified_seq_len: torch.Tensor,
positions: torch.Tensor, # mutable
retrieve_index: torch.Tensor, # mutable
retrieve_next_token: torch.Tensor, # mutable
retrieve_next_sibling: torch.Tensor, # mutable
batch_size: int,
draft_token_num: int,
) -> None:
if _is_xpu:
reconstruct_indices_from_tree_mask_triton(
tree_mask,
verified_seq_len,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
batch_size,
draft_token_num,
)
return
reconstruct_indices_from_tree_mask(
tree_mask,
verified_seq_len,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
batch_size,
draft_token_num,
)
@@ -11,9 +11,41 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
from enum import IntEnum
import torch
import triton
import triton.language as tl
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_musa, is_npu, is_xpu
_is_cuda = is_cuda()
_is_hip = is_hip()
_is_musa = is_musa()
_is_npu = is_npu()
_is_xpu = is_xpu()
_is_cpu = is_cpu()
if _is_cuda or _is_hip:
from sglang.kernels.ops.speculative.tree import (
build_tree_kernel_efficient as sgl_build_tree_kernel_efficient,
)
elif _is_musa:
from sgl_kernel import (
build_tree_kernel_efficient as sgl_build_tree_kernel_efficient,
)
elif _is_cpu:
from sgl_kernel import (
build_tree_kernel_efficient_cpu as sgl_build_tree_kernel_efficient_cpu,
)
from sgl_kernel import verify_tree_greedy_cpu as sgl_verify_tree_greedy_cpu
class TreeMaskMode(IntEnum):
FULL_MASK = 0
QLEN_ONLY = 1
QLEN_ONLY_BITPACKING = 2
@triton.jit
def sgl_build_tree_kernel_efficient_triton(
@@ -279,3 +311,232 @@ def verify_tree_greedy_kernel_triton(
target_predict_ptr + target_row * num_draft_tokens + target_col
)
tl.store(predicts_ptr + last_accept_retrieve_idx, final_target)
def sgl_build_tree_kernel_triton(
parent_list: torch.Tensor,
selected_index: torch.Tensor,
verified_seq_len: torch.Tensor,
tree_mask: torch.Tensor,
positions: torch.Tensor,
retrieve_index: torch.Tensor,
retrieve_next_token: torch.Tensor,
retrieve_next_sibling: torch.Tensor,
topk: int,
depth: int,
draft_token_num: int,
tree_mask_mode: TreeMaskMode = TreeMaskMode.FULL_MASK,
):
"""Triton-based implementation."""
# TODO: Add support for QLEN_ONLY_BITPACKING mode
if tree_mask_mode == TreeMaskMode.QLEN_ONLY_BITPACKING:
raise NotImplementedError(
"QLEN_ONLY_BITPACKING is not supported in Triton implementation"
)
batch_size = verified_seq_len.shape[0]
seq_len_prefix_sum = torch.cumsum(verified_seq_len, dim=0) - verified_seq_len
# Launch kernel with one program per batch item
grid = (batch_size,)
sgl_build_tree_kernel_efficient_triton[grid](
parent_list,
selected_index,
verified_seq_len,
seq_len_prefix_sum,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk=topk,
depth=depth,
draft_token_num=draft_token_num,
tree_mask_mode=int(tree_mask_mode),
batch_size=batch_size,
parent_list_stride=(
parent_list.stride(0) if parent_list.dim() > 1 else parent_list.shape[0]
),
selected_index_stride=selected_index.stride(0),
)
def build_tree_kernel_efficient_func(
parent_list: torch.Tensor,
top_scores_index: torch.Tensor,
seq_lens: torch.Tensor,
tree_mask: torch.Tensor, # mutable
positions: torch.Tensor, # mutable
retrieve_index: torch.Tensor, # mutable
retrieve_next_token: torch.Tensor, # mutable
retrieve_next_sibling: torch.Tensor, # mutable
topk: int,
spec_steps: int,
num_verify_tokens: int,
tree_mask_mode: TreeMaskMode,
) -> None:
if _is_npu:
torch.ops.npu.build_tree_kernel_efficient(
parent_list.to(dtype=torch.int64),
top_scores_index,
seq_lens,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk,
spec_steps,
num_verify_tokens,
tree_mask_mode,
)
elif _is_xpu:
sgl_build_tree_kernel_triton(
parent_list,
top_scores_index,
seq_lens,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk,
spec_steps,
num_verify_tokens,
tree_mask_mode,
)
elif _is_cpu:
sgl_build_tree_kernel_efficient_cpu(
parent_list,
top_scores_index,
seq_lens,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk,
spec_steps,
num_verify_tokens,
tree_mask_mode,
)
else:
sgl_build_tree_kernel_efficient(
parent_list,
top_scores_index,
seq_lens,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk,
spec_steps,
num_verify_tokens,
tree_mask_mode,
)
def verify_tree_greedy_triton(
predicts: torch.Tensor,
accept_index: torch.Tensor,
accept_token_num: torch.Tensor,
candidates: torch.Tensor,
retrieve_index: torch.Tensor,
retrieve_next_token: torch.Tensor,
retrieve_next_sibling: torch.Tensor,
target_predict: torch.Tensor,
):
"""Triton-based implementation."""
batch_size = candidates.shape[0]
num_speculative_tokens = accept_index.shape[1]
num_draft_tokens = candidates.shape[1]
# Launch kernel with one program per batch item
grid = (batch_size,)
verify_tree_greedy_kernel_triton[grid](
predicts,
accept_index,
accept_token_num,
candidates,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
target_predict,
batch_size=batch_size,
num_speculative_tokens=num_speculative_tokens,
num_draft_tokens=num_draft_tokens,
)
def verify_tree_greedy_func(
predicts: torch.Tensor,
accept_index: torch.Tensor,
accept_token_num: torch.Tensor,
candidates: torch.Tensor,
retrieve_index: torch.Tensor,
retrieve_next_token: torch.Tensor,
retrieve_next_sibling: torch.Tensor,
target_predict: torch.Tensor,
topk: int = -1,
):
if _is_cuda or _is_hip or _is_musa:
if _is_cuda or _is_hip:
from sglang.kernels.ops.speculative.tree import verify_tree_greedy
else:
from sgl_kernel import verify_tree_greedy
verify_tree_greedy(
predicts=predicts, # mutable
accept_index=accept_index, # mutable
accept_token_num=accept_token_num, # mutable
candidates=candidates,
# kwarg LHS retained as `retrive_*` to match sgl_kernel op schema.
retrive_index=retrieve_index,
retrive_next_token=retrieve_next_token,
retrive_next_sibling=retrieve_next_sibling,
target_predict=target_predict,
)
elif _is_cpu:
sgl_verify_tree_greedy_cpu(
predicts=predicts, # mutable
accept_index=accept_index, # mutable
accept_token_num=accept_token_num, # mutable
candidates=candidates,
# kwarg LHS retained as `retrive_*` to match the CUDA op schema, so
# the CPU/CUDA call sites stay grep-symmetric.
retrive_index=retrieve_index,
retrive_next_token=retrieve_next_token,
retrive_next_sibling=retrieve_next_sibling,
target_predict=target_predict,
)
elif _is_npu:
from sgl_kernel_npu.sample.verify_tree_greedy import verify_tree_greedy
verify_tree_greedy(
predicts=predicts,
accept_index=accept_index,
accept_token_num=accept_token_num,
candidates=candidates,
# kwarg LHS retained as `retrive_*` to match sgl_kernel op schema.
retrive_index=retrieve_index,
retrive_next_token=retrieve_next_token,
retrive_next_sibling=retrieve_next_sibling,
target_predict=target_predict,
)
elif _is_xpu:
verify_tree_greedy_triton(
predicts=predicts,
accept_index=accept_index,
accept_token_num=accept_token_num,
candidates=candidates,
retrieve_index=retrieve_index,
retrieve_next_token=retrieve_next_token,
retrieve_next_sibling=retrieve_next_sibling,
target_predict=target_predict,
)
return predicts, accept_index, accept_token_num
@@ -0,0 +1,34 @@
"""Speculative verify sampling ops, resolved once per platform.
A name is ``None`` where the platform has no kernel for it.
"""
from sglang.srt.utils import is_cuda, is_musa, is_npu
if is_npu():
try:
from sgl_kernel_npu.sample import (
chain_speculative_sampling_triton,
tree_speculative_sampling_target_only,
)
except ImportError:
chain_speculative_sampling_triton = None
tree_speculative_sampling_target_only = None
else:
from sglang.kernels.ops.speculative.reject_sampling import (
chain_speculative_sampling_triton,
)
if is_cuda():
from sglang.kernels.ops.speculative.sampling import (
tree_speculative_sampling_target_only,
)
elif is_musa():
try:
from sgl_kernel import tree_speculative_sampling_target_only
except Exception:
tree_speculative_sampling_target_only = None
else:
tree_speculative_sampling_target_only = None
__all__ = ["chain_speculative_sampling_triton", "tree_speculative_sampling_target_only"]
+3 -1
View File
@@ -369,7 +369,9 @@ class DFlashAttention(nn.Module):
if _is_npu:
q, k, v = self.forward_prepare_npu(positions, hidden_states)
elif self.use_table_qk_norm_rope and qkv.dtype == torch.bfloat16:
from sglang.srt.speculative.dflash_utils import table_qk_norm_rope_
from sglang.kernels.ops.attention.table_qk_norm_rope import (
table_qk_norm_rope_,
)
table_qk_norm_rope_(
qkv,
@@ -2,7 +2,7 @@ from __future__ import annotations
from dataclasses import dataclass, field
from enum import Enum
from typing import Callable, Optional
from typing import Optional
class DraftMeshMessageType(str, Enum):
@@ -23,23 +23,6 @@ class DraftReqKey:
request_id: str
def build_draft_scheduler_rid(draft_key: DraftReqKey) -> str:
return f"draft:{int(draft_key.src_verifier_rank)}:{draft_key.request_id}"
def parse_draft_scheduler_rid(rid: str) -> DraftReqKey:
if rid.startswith("draft:"):
encoded = rid[len("draft:") :]
rank_text, sep, request_id = encoded.partition(":")
if sep and request_id:
return DraftReqKey(
src_verifier_rank=int(rank_text),
request_id=request_id,
)
raise ValueError(f"Invalid decoupled draft scheduler rid: {rid}")
@dataclass
class DraftSync:
"""Open or re-open a drafter request from a verifier-owned prefix.
@@ -246,114 +229,6 @@ class VerifierCommitSegment:
return prefix_segment
@dataclass
class DraftControlInbox:
"""Drafter-side inbox for verifier control messages.
The TokenSync thread temporarily stores incoming control messages here.
The drafter scheduler extracts and consumes them each time it finishes a decoding step.
"""
sync_messages: list[DraftSync] = field(default_factory=list)
verifier_commit_segments: dict[DraftReqKey, VerifierCommitSegment] = field(
default_factory=dict
)
close_keys: set[DraftReqKey] = field(default_factory=set)
def is_empty(self) -> bool:
return (
not self.sync_messages
and not self.verifier_commit_segments
and not self.close_keys
)
def pending_control_count(self) -> int:
return (
len(self.sync_messages)
+ len(self.verifier_commit_segments)
+ len(self.close_keys)
)
def add_control_batch_locked(self, batch: DraftControlBatch) -> None:
for message in batch.close_messages:
self.add_close_key_locked(message.draft_key)
for message in batch.sync_messages:
if message.draft_key not in self.close_keys:
self.sync_messages.append(message)
for message in batch.verify_commit_messages:
self.add_verify_commit_locked(message)
def add_close_key_locked(self, draft_key: DraftReqKey) -> None:
self.close_keys.add(draft_key)
self.verifier_commit_segments.pop(draft_key, None)
self.sync_messages = [
message for message in self.sync_messages if message.draft_key != draft_key
]
def add_verify_commit_locked(self, message: VerifyCommit) -> None:
if message.draft_key in self.close_keys:
return
segment = self.verifier_commit_segments.get(message.draft_key)
if segment is None:
segment = VerifierCommitSegment(
draft_key=message.draft_key,
dst_drafter_rank=int(message.dst_drafter_rank),
pre_verify_committed_len=int(message.pre_verify_committed_len),
)
segment.append_message(message)
self.verifier_commit_segments[message.draft_key] = segment
return
segment.append_message(message)
def extract_ready_controls_locked(
self,
consumable_commit_len: Callable[[VerifierCommitSegment], int],
) -> ReadyDraftControls:
ready_controls = ReadyDraftControls()
if self.close_keys:
ready_controls.close_keys = self.close_keys
self.close_keys = set()
if self.sync_messages:
ready_controls.sync_messages = self.sync_messages
self.sync_messages = []
for draft_key, segment in list(self.verifier_commit_segments.items()):
consumable_len = consumable_commit_len(segment)
if consumable_len <= 0:
continue
ready_controls.ready_commit_segments.append(
segment.extract_prefix(consumable_len)
)
if not segment.committed_tokens:
self.verifier_commit_segments.pop(draft_key, None)
return ready_controls
@dataclass
class ReadyDraftControls:
sync_messages: list[DraftSync] = field(default_factory=list)
close_keys: set[DraftReqKey] = field(default_factory=set)
ready_commit_segments: list[VerifierCommitSegment] = field(default_factory=list)
def is_empty(self) -> bool:
return (
not self.sync_messages
and not self.close_keys
and not self.ready_commit_segments
)
def extracted_control_count(self) -> int:
return (
len(self.sync_messages)
+ len(self.close_keys)
+ len(self.ready_commit_segments)
)
@dataclass
class DraftMeshMessage:
message_type: DraftMeshMessageType
+9 -201
View File
@@ -8,9 +8,11 @@ from typing import Any, List, Optional, Tuple
import torch
import torch.nn.functional as F
import triton
import triton.language as tl
from sglang.kernels.ops.sampling.renorm import top_k_renorm_prob, top_p_renorm_prob
from sglang.kernels.ops.speculative.verify_sampling import (
tree_speculative_sampling_target_only,
)
from sglang.srt.layers.quantization.unquant import UnquantizedLinearMethod
from sglang.srt.layers.sampler import (
apply_custom_logit_processor,
@@ -20,14 +22,12 @@ from sglang.srt.managers.schedule_batch import Req
from sglang.srt.model_executor.runner_utils.pool import borrow_graph_pool
from sglang.srt.runtime_context import get_spec
from sglang.srt.speculative.spec_utils import sample_simulated_acc_len
from sglang.srt.utils import is_cuda, is_hip, is_musa, is_npu
from sglang.utils import LazyImport
from sglang.srt.utils import is_npu
DEFAULT_DFLASH_MASK_TOKEN = "<|MASK|>"
logger = logging.getLogger(__name__)
_DFLASH_SAMPLING_VERIFY_AVAILABLE = False
_DFLASH_CHAIN_VERIFY_BUFFERS: dict[tuple[Optional[int], int], dict[str, Any]] = {}
_DFLASH_VERIFY_SKIP_CUSTOM_MASK_BACKENDS = frozenset(
{
@@ -42,43 +42,10 @@ _DFLASH_VERIFY_SKIP_CUSTOM_MASK_BACKENDS = frozenset(
)
if is_cuda():
from sglang.kernels.ops.speculative.sampling import (
tree_speculative_sampling_target_only,
)
# Resolved on first call: CUDA builds without FlashInfer must still import
# this module.
top_k_renorm_prob = LazyImport("flashinfer.sampling", "top_k_renorm_probs")
top_p_renorm_prob = LazyImport("flashinfer.sampling", "top_p_renorm_probs")
_DFLASH_SAMPLING_VERIFY_AVAILABLE = True
elif is_musa():
try:
from sgl_kernel import (
top_k_renorm_prob,
top_p_renorm_prob,
tree_speculative_sampling_target_only,
)
_DFLASH_SAMPLING_VERIFY_AVAILABLE = True
except Exception:
top_k_renorm_prob = None
top_p_renorm_prob = None
tree_speculative_sampling_target_only = None
elif is_hip():
from sglang.kernels.ops.sampling.renorm_triton import (
top_k_renorm_probs_triton as top_k_renorm_prob,
)
from sglang.kernels.ops.sampling.renorm_triton import (
top_p_renorm_probs_triton as top_p_renorm_prob,
)
tree_speculative_sampling_target_only = None
else:
top_k_renorm_prob = None
top_p_renorm_prob = None
tree_speculative_sampling_target_only = None
# DFlash does not run sampling verify on NPU, even where the op exists.
_DFLASH_SAMPLING_VERIFY_AVAILABLE = (
tree_speculative_sampling_target_only is not None and not is_npu()
)
def is_dflash_sampling_verify_available() -> bool:
@@ -881,85 +848,6 @@ def can_dflash_use_fused_qkv_proj(qkv_proj: Any) -> Tuple[bool, str]:
return True, ""
@triton.jit
def _fused_correct_drafts_and_bonus_kernel(
candidates_ptr,
target_predict_ptr,
num_correct_drafts_ptr,
bonus_tokens_ptr,
block_size,
BLOCK: tl.constexpr,
):
b = tl.program_id(0).to(tl.int64)
offs = tl.arange(0, BLOCK)
in_row = offs < block_size - 1
drafts = tl.load(candidates_ptr + b * block_size + 1 + offs, mask=in_row, other=-1)
targets = tl.load(target_predict_ptr + b * block_size + offs, mask=in_row, other=-2)
eq = (drafts == targets) & in_row
# Leading-match count = index of the first mismatch lane; lanes past the
# row and all-match rows both resolve to block_size - 1 via the min.
num_correct = tl.min(tl.where(eq, BLOCK, offs), 0)
bonus_token = tl.load(target_predict_ptr + b * block_size + num_correct)
tl.store(num_correct_drafts_ptr + b, num_correct.to(tl.int32))
tl.store(bonus_tokens_ptr + b, bonus_token.to(tl.int64))
def compute_dflash_correct_drafts_and_bonus(
*,
candidates: torch.Tensor,
target_predict: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""Compute DFlash accept lengths and bonus tokens (greedy verify rule).
Args:
candidates: Token ids proposed by the DFlash draft, including the current token.
Shape: [bs, block_size]. candidates[:, 0] is the current token.
target_predict: Token ids predicted by the target model for each position in the block.
Shape: [bs, block_size]. target_predict[:, t] corresponds to argmax at position t.
Returns:
correct_len: int32 tensor [bs], number of accepted *draft* tokens (excluding current token and bonus token).
bonus: int64 tensor [bs], the target-predicted token at index correct_len (the "bonus" token to append).
Notes:
Matches the reference implementation rule:
accept while candidates[:, 1:] == target_predict[:, :-1] consecutively.
"""
if candidates.ndim != 2:
raise ValueError(f"candidates must be 2D, got shape={tuple(candidates.shape)}")
if target_predict.shape != candidates.shape:
raise ValueError(
"target_predict must have the same shape as candidates. "
f"candidates.shape={tuple(candidates.shape)}, target_predict.shape={tuple(target_predict.shape)}"
)
bs, block_size = candidates.shape
if bs <= 0:
raise ValueError(f"batch size must be positive, got {bs}.")
if block_size <= 0:
raise ValueError(f"block_size must be positive, got {block_size}.")
if candidates.is_cuda:
num_correct_drafts = torch.empty(
bs, dtype=torch.int32, device=candidates.device
)
bonus_tokens = torch.empty(bs, dtype=torch.int64, device=candidates.device)
_fused_correct_drafts_and_bonus_kernel[(bs,)](
candidates.contiguous(),
target_predict.contiguous(),
num_correct_drafts,
bonus_tokens,
block_size,
BLOCK=triton.next_power_of_2(max(block_size - 1, 1)),
)
return num_correct_drafts, bonus_tokens
matches = candidates[:, 1:] == target_predict[:, :-1]
correct_len = matches.to(torch.int32).cumprod(dim=1).sum(dim=1)
bonus = target_predict[torch.arange(bs, device=target_predict.device), correct_len]
return correct_len.to(torch.int32), bonus.to(torch.int64)
def apply_dflash_simulated_acceptance(
*,
candidates: torch.Tensor,
@@ -1222,83 +1110,3 @@ def validate_dflash_request(req: Req, enable_overlap: bool) -> Optional[str]:
return "DFLASH speculative decoding does not support return_hidden_states yet."
return None
@triton.jit
def _table_qk_norm_rope_kernel(
qkv_ptr,
q_weight_ptr,
k_weight_ptr,
cos_sin_ptr,
pos_ptr,
row_stride,
q_size,
NHQ: tl.constexpr,
D: tl.constexpr,
EPS: tl.constexpr,
):
t = tl.program_id(0).to(tl.int64)
h = tl.program_id(1)
pos = tl.load(pos_ptr + t).to(tl.int64)
HALF: tl.constexpr = D // 2
half_ar = tl.arange(0, HALF)
d_ar = tl.arange(0, D)
cos = tl.load(cos_sin_ptr + pos * D + half_ar).to(tl.float32)
sin = tl.load(cos_sin_ptr + pos * D + HALF + half_ar).to(tl.float32)
is_q = h < NHQ
col0 = tl.where(is_q, h * D, q_size + (h - NHQ) * D).to(tl.int64)
w_ptr = tl.where(is_q, q_weight_ptr.to(tl.int64), k_weight_ptr.to(tl.int64)).to(
tl.pointer_type(tl.bfloat16)
)
row = qkv_ptr + t * row_stride + col0
x = tl.load(row + d_ar).to(tl.float32)
ms = tl.sum(x * x, 0) / D
inv = 1.0 / tl.sqrt(ms + EPS)
w1 = tl.load(w_ptr + half_ar).to(tl.float32)
w2 = tl.load(w_ptr + HALF + half_ar).to(tl.float32)
x1 = tl.load(row + half_ar).to(tl.float32) * inv * w1
x2 = tl.load(row + HALF + half_ar).to(tl.float32) * inv * w2
x1 = x1.to(tl.bfloat16).to(tl.float32)
x2 = x2.to(tl.bfloat16).to(tl.float32)
o1 = x1 * cos - x2 * sin
o2 = x2 * cos + x1 * sin
tl.store(row + half_ar, o1.to(tl.bfloat16))
tl.store(row + HALF + half_ar, o2.to(tl.bfloat16))
def table_qk_norm_rope_(
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
num_q_heads: int,
num_k_heads: int,
head_dim: int,
eps: float,
) -> None:
"""In-place QK RMSNorm + table-lookup neox RoPE on the fused QKV tensor.
Reads cos/sin from the SAME rotary table as the unfused path, so there is
no large-position angle drift (unlike theta-recompute kernels). V columns
are untouched.
"""
T = qkv.shape[0]
if T == 0:
return
grid = (T, num_q_heads + num_k_heads)
_table_qk_norm_rope_kernel[grid](
qkv,
q_weight,
k_weight,
cos_sin_cache,
positions,
qkv.stride(0),
num_q_heads * head_dim,
NHQ=num_q_heads,
D=head_dim,
EPS=eps,
)
@@ -13,6 +13,7 @@ from sglang.kernels.ops.speculative.cache_locs import (
from sglang.kernels.ops.speculative.dflash import (
_compute_dflash_accept_bonus_triton_unchecked,
_prepare_dflash_draft_block_unchecked,
compute_dflash_correct_drafts_and_bonus,
)
from sglang.kernels.ops.speculative.dspark.dspark_accept import (
accept_sampling,
@@ -62,7 +63,6 @@ from sglang.srt.speculative.dflash_utils import (
apply_dflash_simulated_acceptance,
apply_dflash_verify_logits_adjustments,
can_dflash_use_fused_qkv_proj,
compute_dflash_correct_drafts_and_bonus,
compute_dflash_sampling_correct_drafts_and_bonus,
is_dense_head_weight,
is_dflash_sampling_verify_available,
@@ -155,12 +155,6 @@ def read_draft_checkpoint_config(*, server_args: ServerArgs) -> DSparkDraftConfi
return parse_dspark_draft_config(draft_hf_config=draft_hf_config)
def read_draft_checkpoint_gamma(*, server_args: ServerArgs) -> Optional[int]:
return read_draft_checkpoint_config(server_args=server_args).resolve_gamma(
default=None
)
def checkpoint_bundles_dspark_draft(hf_config: Any) -> bool:
"""The checkpoint carries a bundled DSpark draft head, marked by the
prefixed dspark_* keys on the target hf config. Single source of truth
@@ -12,11 +12,13 @@ from typing import Callable, ContextManager, Iterator, Optional, Union
import msgspec
import torch
from sglang.kernels.ops.speculative.dflash import (
compute_dflash_correct_drafts_and_bonus,
)
from sglang.srt.environ import envs
from sglang.srt.kv_canary.runner.future_tensor import FutureTensors
from sglang.srt.runtime_context import get_parallel
from sglang.srt.sampling.sampling_params import TOP_K_ALL
from sglang.srt.speculative.dflash_utils import compute_dflash_correct_drafts_and_bonus
from sglang.srt.speculative.dspark_components.dspark_block_accept_estimator import (
create_block_accept_estimate_recorder,
)
+22 -275
View File
@@ -2,16 +2,21 @@ from __future__ import annotations
import logging
import math
from enum import IntEnum
from functools import partial
from typing import TYPE_CHECKING, List, Optional, Tuple
import torch
from sglang.kernels.ops.sampling import softmax as sampling_softmax
from sglang.kernels.ops.sampling.renorm import top_k_renorm_prob, top_p_renorm_prob
from sglang.kernels.ops.speculative.spec_tree import (
sgl_build_tree_kernel_efficient_triton,
verify_tree_greedy_kernel_triton,
TreeMaskMode,
build_tree_kernel_efficient_func,
verify_tree_greedy_func,
)
from sglang.kernels.ops.speculative.verify_sampling import (
chain_speculative_sampling_triton,
tree_speculative_sampling_target_only,
)
from sglang.srt.hardware_backend.npu.dsv4.dsv4_common_hooks import (
maybe_build_dsv4_verify_bundle,
@@ -26,8 +31,6 @@ from sglang.srt.utils import (
is_cpu,
is_cuda,
is_hip,
is_musa,
is_npu,
is_xpu,
)
from sglang.srt.utils.async_probe import maybe_detect_oob
@@ -45,27 +48,11 @@ if TYPE_CHECKING:
_is_cuda = is_cuda()
_is_hip = is_hip()
_is_npu = is_npu()
_is_musa = is_musa()
_is_xpu = is_xpu()
_is_cpu = is_cpu()
logger = logging.getLogger(__name__)
if _is_cuda or _is_hip:
from sglang.kernels.ops.speculative.tree import (
build_tree_kernel_efficient as sgl_build_tree_kernel_efficient,
)
elif _is_musa:
from sgl_kernel import (
build_tree_kernel_efficient as sgl_build_tree_kernel_efficient,
)
elif _is_cpu:
from sgl_kernel import (
build_tree_kernel_efficient_cpu as sgl_build_tree_kernel_efficient_cpu,
)
from sgl_kernel import verify_tree_greedy_cpu as sgl_verify_tree_greedy_cpu
def per_step_draft_out_cache_loc(
out_cache_loc: torch.Tensor,
@@ -142,12 +129,6 @@ def organize_draft_results(
return parent_list, top_scores_index, draft_tokens
class TreeMaskMode(IntEnum):
FULL_MASK = 0
QLEN_ONLY = 1
QLEN_ONLY_BITPACKING = 2
def default_tree_mask_mode() -> TreeMaskMode:
# The CPU verify attention kernel (intel_amx) consumes the qlen x qlen
# QLEN_ONLY tree mask directly; FULL_MASK is for the GPU kernels.
@@ -229,66 +210,20 @@ def build_tree_kernel_efficient(
# then, positions = [7, 8, 8, 9]
positions = torch.empty((bs * num_verify_tokens,), device=device, dtype=torch.long)
if _is_npu:
torch.ops.npu.build_tree_kernel_efficient(
parent_list.to(dtype=torch.int64),
top_scores_index,
seq_lens,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk,
spec_steps,
num_verify_tokens,
tree_mask_mode,
)
elif _is_xpu:
sgl_build_tree_kernel_triton(
parent_list,
top_scores_index,
seq_lens,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk,
spec_steps,
num_verify_tokens,
tree_mask_mode,
)
elif _is_cpu:
sgl_build_tree_kernel_efficient_cpu(
parent_list,
top_scores_index,
seq_lens,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk,
spec_steps,
num_verify_tokens,
tree_mask_mode,
)
else:
sgl_build_tree_kernel_efficient(
parent_list,
top_scores_index,
seq_lens,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk,
spec_steps,
num_verify_tokens,
tree_mask_mode,
)
build_tree_kernel_efficient_func(
parent_list,
top_scores_index,
seq_lens,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk,
spec_steps,
num_verify_tokens,
tree_mask_mode,
)
return (
tree_mask,
positions,
@@ -299,159 +234,6 @@ def build_tree_kernel_efficient(
)
def sgl_build_tree_kernel_triton(
parent_list: torch.Tensor,
selected_index: torch.Tensor,
verified_seq_len: torch.Tensor,
tree_mask: torch.Tensor,
positions: torch.Tensor,
retrieve_index: torch.Tensor,
retrieve_next_token: torch.Tensor,
retrieve_next_sibling: torch.Tensor,
topk: int,
depth: int,
draft_token_num: int,
tree_mask_mode: TreeMaskMode = TreeMaskMode.FULL_MASK,
):
"""Triton-based implementation."""
# TODO: Add support for QLEN_ONLY_BITPACKING mode
if tree_mask_mode == TreeMaskMode.QLEN_ONLY_BITPACKING:
raise NotImplementedError(
"QLEN_ONLY_BITPACKING is not supported in Triton implementation"
)
batch_size = verified_seq_len.shape[0]
seq_len_prefix_sum = torch.cumsum(verified_seq_len, dim=0) - verified_seq_len
# Launch kernel with one program per batch item
grid = (batch_size,)
sgl_build_tree_kernel_efficient_triton[grid](
parent_list,
selected_index,
verified_seq_len,
seq_len_prefix_sum,
tree_mask,
positions,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
topk=topk,
depth=depth,
draft_token_num=draft_token_num,
tree_mask_mode=int(tree_mask_mode),
batch_size=batch_size,
parent_list_stride=(
parent_list.stride(0) if parent_list.dim() > 1 else parent_list.shape[0]
),
selected_index_stride=selected_index.stride(0),
)
def verify_tree_greedy_triton(
predicts: torch.Tensor,
accept_index: torch.Tensor,
accept_token_num: torch.Tensor,
candidates: torch.Tensor,
retrieve_index: torch.Tensor,
retrieve_next_token: torch.Tensor,
retrieve_next_sibling: torch.Tensor,
target_predict: torch.Tensor,
):
"""Triton-based implementation."""
batch_size = candidates.shape[0]
num_speculative_tokens = accept_index.shape[1]
num_draft_tokens = candidates.shape[1]
# Launch kernel with one program per batch item
grid = (batch_size,)
verify_tree_greedy_kernel_triton[grid](
predicts,
accept_index,
accept_token_num,
candidates,
retrieve_index,
retrieve_next_token,
retrieve_next_sibling,
target_predict,
batch_size=batch_size,
num_speculative_tokens=num_speculative_tokens,
num_draft_tokens=num_draft_tokens,
)
def verify_tree_greedy_func(
predicts: torch.Tensor,
accept_index: torch.Tensor,
accept_token_num: torch.Tensor,
candidates: torch.Tensor,
retrieve_index: torch.Tensor,
retrieve_next_token: torch.Tensor,
retrieve_next_sibling: torch.Tensor,
target_predict: torch.Tensor,
topk: int = -1,
):
if _is_cuda or _is_hip or _is_musa:
if _is_cuda or _is_hip:
from sglang.kernels.ops.speculative.tree import verify_tree_greedy
else:
from sgl_kernel import verify_tree_greedy
verify_tree_greedy(
predicts=predicts, # mutable
accept_index=accept_index, # mutable
accept_token_num=accept_token_num, # mutable
candidates=candidates,
# kwarg LHS retained as `retrive_*` to match sgl_kernel op schema.
retrive_index=retrieve_index,
retrive_next_token=retrieve_next_token,
retrive_next_sibling=retrieve_next_sibling,
target_predict=target_predict,
)
elif _is_cpu:
sgl_verify_tree_greedy_cpu(
predicts=predicts, # mutable
accept_index=accept_index, # mutable
accept_token_num=accept_token_num, # mutable
candidates=candidates,
# kwarg LHS retained as `retrive_*` to match the CUDA op schema, so
# the CPU/CUDA call sites stay grep-symmetric.
retrive_index=retrieve_index,
retrive_next_token=retrieve_next_token,
retrive_next_sibling=retrieve_next_sibling,
target_predict=target_predict,
)
elif _is_npu:
from sgl_kernel_npu.sample.verify_tree_greedy import verify_tree_greedy
verify_tree_greedy(
predicts=predicts,
accept_index=accept_index,
accept_token_num=accept_token_num,
candidates=candidates,
# kwarg LHS retained as `retrive_*` to match sgl_kernel op schema.
retrive_index=retrieve_index,
retrive_next_token=retrieve_next_token,
retrive_next_sibling=retrieve_next_sibling,
target_predict=target_predict,
)
elif _is_xpu:
verify_tree_greedy_triton(
predicts=predicts,
accept_index=accept_index,
accept_token_num=accept_token_num,
candidates=candidates,
retrieve_index=retrieve_index,
retrieve_next_token=retrieve_next_token,
retrieve_next_sibling=retrieve_next_sibling,
target_predict=target_predict,
)
return predicts, accept_index, accept_token_num
def get_draft_input_from_target_hidden_dim(model_runner: ModelRunner) -> int:
"""Width of the target hidden states fed into the draft model.
@@ -930,48 +712,13 @@ def eagle_sample(
tp_group.broadcast(accept_index, src=0)
tp_group.broadcast(num_correct_drafts, src=0)
else:
if _is_npu:
from sgl_kernel_npu.sample import (
chain_speculative_sampling_triton,
top_k_renorm_prob,
top_p_renorm_prob,
tree_speculative_sampling_target_only,
)
else:
from sglang.kernels.ops.speculative.reject_sampling import (
chain_speculative_sampling_triton,
)
# if/else, not a ternary: the CUDA-only name still has to resolve in the
# branch not taken, and HIP only reaches here with rejection sampling on.
if use_rejection_sampling:
sampling_fn = chain_speculative_sampling_triton
if get_spec().speculative_use_block_verification:
sampling_fn = partial(sampling_fn, block_verification=True)
else:
if _is_cuda:
from sglang.kernels.ops.speculative.sampling import (
tree_speculative_sampling_target_only,
)
elif not _is_npu:
from sgl_kernel import tree_speculative_sampling_target_only
sampling_fn = tree_speculative_sampling_target_only
if _is_hip:
# Same names, same contract: dflash_utils.py aliases these too.
from sglang.kernels.ops.sampling.renorm_triton import (
top_k_renorm_probs_triton as top_k_renorm_prob,
)
from sglang.kernels.ops.sampling.renorm_triton import (
top_p_renorm_probs_triton as top_p_renorm_prob,
)
elif _is_cuda:
from flashinfer.sampling import top_k_renorm_probs as top_k_renorm_prob
from flashinfer.sampling import top_p_renorm_probs as top_p_renorm_prob
elif not _is_npu:
from sgl_kernel import top_k_renorm_prob, top_p_renorm_prob
expanded_temperature = torch.repeat_interleave(
sampling_info.temperatures, verify_input.draft_token_num, dim=0
) # (bs * num_draft_tokens, 1)
@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Any, Optional
import torch
from sglang.kernels.ops.speculative.cache_locs import (
assign_draft_cache_locs_contiguous,
assign_draft_cache_locs_contiguous_func,
)
from sglang.kernels.ops.speculative.eagle import (
build_chain_tree,
@@ -49,9 +49,6 @@ from sglang.srt.utils.common import is_npu
_is_cpu = is_cpu()
_is_npu = is_npu()
if _is_cpu:
from sgl_kernel import assign_draft_cache_locs_contiguous_cpu
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.tp_worker import TpModelWorker
@@ -281,40 +278,25 @@ def prepare_for_draft(
dtype=torch.int64,
device=batch.device,
)
if _is_cpu:
assign_draft_cache_locs_contiguous_cpu(
batch.req_pool_indices,
req_to_token_pool.req_to_token,
batch.seq_lens,
batch.out_cache_loc,
req_to_token_pool.req_to_token.shape[1],
topk,
num_steps,
)
else:
if batch.seq_lens.is_cuda and topk == 1:
prepared_positions = torch.empty_like(batch.seq_lens)
if draft_model_runner.model_config.model_is_mrope and all(
x is None for x in batch.multimodal_inputs
):
prepared_mrope_positions = torch.empty(
(3, bs), dtype=torch.int64, device=batch.device
)
# FIXME(lsyin): align with the default code path
assign_draft_cache_locs_contiguous[(bs,)](
batch.req_pool_indices,
req_to_token_pool.req_to_token,
batch.seq_lens,
batch.out_cache_loc,
req_to_token_pool.req_to_token.shape[1],
topk,
window_cols,
positions=prepared_positions,
mrope=prepared_mrope_positions,
BS=bs,
WRITE_POSITIONS=prepared_positions is not None,
WRITE_MROPE=prepared_mrope_positions is not None,
)
if batch.seq_lens.is_cuda and topk == 1:
prepared_positions = torch.empty_like(batch.seq_lens)
if draft_model_runner.model_config.model_is_mrope and all(
x is None for x in batch.multimodal_inputs
):
prepared_mrope_positions = torch.empty(
(3, bs), dtype=torch.int64, device=batch.device
)
# TODO(lsyin): align with the default code path
assign_draft_cache_locs_contiguous_func(
batch.req_pool_indices,
req_to_token_pool.req_to_token,
batch.seq_lens,
batch.out_cache_loc,
topk,
window_cols,
positions=prepared_positions,
mrope=prepared_mrope_positions,
)
if plans_window:
kv_loc_plan = target_translator.plan(
req_pool_indices=batch.req_pool_indices,
+5 -11
View File
@@ -7,6 +7,9 @@ import torch
from sglang.kernels.ops.speculative.cache_locs import (
assign_extend_cache_locs_func as assign_extend_cache_locs_func,
)
from sglang.kernels.ops.speculative.reconstruct_tree import (
reconstruct_indices_from_tree_mask_func,
)
from sglang.srt.layers.logprob_processor import compute_spec_logprobs
from sglang.srt.managers.schedule_batch import ScheduleBatch
from sglang.srt.managers.scheduler import GenerationBatchResult
@@ -31,20 +34,11 @@ from sglang.srt.speculative.spec_utils import (
prepare_mamba_track_for_verify,
record_stream_for_v2_verify,
)
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_xpu
from sglang.srt.utils import is_cpu
from sglang.srt.utils.async_probe import maybe_detect_inf, maybe_detect_nan
_is_cpu = is_cpu()
if is_xpu():
from sglang.kernels.ops.speculative.reconstruct_tree import (
reconstruct_indices_from_tree_mask_triton as reconstruct_indices_from_tree_mask,
)
elif is_cuda() or is_hip():
from sglang.kernels.ops.speculative.tree import reconstruct_indices_from_tree_mask
else:
from sgl_kernel.speculative import reconstruct_indices_from_tree_mask
logger = logging.getLogger(__name__)
@@ -345,7 +339,7 @@ class NGRAMWorker(BaseSpecWorker):
self.grammar_tree_host = (mask, req_drafts) if batch.has_grammar else None
# generate positions and some indices using tree_mask
reconstruct_indices_from_tree_mask(
reconstruct_indices_from_tree_mask_func(
tree_mask,
batch.seq_lens,
positions, # mutable
@@ -302,25 +302,3 @@ def compute_uniform_extend_lengths(
num_tokens=num_tokens,
extend_start_loc=None,
)
def compute_ragged_extend_lengths(
*,
seq_lens: torch.Tensor,
seq_lens_cpu: List[int],
ragged_layout: RaggedVerifyLayout,
) -> VerifyExtendLengths:
extend_seq_lens_cpu = list(ragged_layout.verify_lens_cpu)
seq_lens_extended = seq_lens + ragged_layout.verify_lens
seq_lens_cpu_extended = [
raw + length for raw, length in zip(seq_lens_cpu, extend_seq_lens_cpu)
]
num_tokens = ragged_layout.total_verify_tokens
extend_start_loc = ragged_layout.extend_start_loc
return VerifyExtendLengths(
seq_lens_extended=seq_lens_extended,
seq_lens_cpu_extended=seq_lens_cpu_extended,
extend_seq_lens_cpu=extend_seq_lens_cpu,
num_tokens=num_tokens,
extend_start_loc=extend_start_loc,
)
+6 -59
View File
@@ -12,13 +12,7 @@ from huggingface_hub import snapshot_download
from sglang.kernels.ops.sampling import softmax as sampling_softmax
from sglang.kernels.ops.speculative.cache_locs import (
align_evict_mask_to_page_size as align_evict_mask_to_page_size,
)
from sglang.kernels.ops.speculative.cache_locs import (
assign_extend_cache_locs as assign_extend_cache_locs,
)
from sglang.kernels.ops.speculative.cache_locs import (
filter_finished_cache_loc_kernel as filter_finished_cache_loc_kernel,
fill_front_out_cache_loc_func as fill_front_out_cache_loc_func,
)
from sglang.kernels.ops.speculative.cache_locs import (
generate_draft_decode_kv_indices as generate_draft_decode_kv_indices,
@@ -26,12 +20,6 @@ from sglang.kernels.ops.speculative.cache_locs import (
from sglang.kernels.ops.speculative.cache_locs import (
generate_draft_decode_window_kv_indices as generate_draft_decode_window_kv_indices,
)
from sglang.kernels.ops.speculative.cache_locs import (
get_src_tgt_cache_loc as get_src_tgt_cache_loc,
)
from sglang.kernels.ops.speculative.cache_locs import (
get_target_cache_loc as get_target_cache_loc,
)
from sglang.kernels.ops.speculative.eagle import (
fill_accept_out_cache_loc_func as fill_accept_out_cache_loc_func,
)
@@ -48,9 +36,6 @@ from sglang.srt.environ import envs
from sglang.srt.layers.attention.linear.utils import pp_spec_stable_rows_enabled
from sglang.srt.managers.schedule_batch import set_mamba_track_indices_from_reqs
from sglang.srt.managers.utils import _async_d2h
from sglang.srt.mem_cache.allocation import (
assign_req_to_token_pool as assign_req_to_token_pool,
)
from sglang.srt.mem_cache.allocation import (
assign_req_to_token_pool_func as assign_req_to_token_pool_func,
)
@@ -62,13 +47,11 @@ from sglang.srt.runtime_context import (
max_speculative_num_draft_tokens,
)
from sglang.srt.utils import (
is_cpu,
is_cuda,
is_hip,
is_musa,
is_npu,
is_xpu,
next_power_of_2,
)
from sglang.srt.utils.async_probe import maybe_detect_oob
from sglang.srt.utils.common import fast_topk
@@ -79,7 +62,6 @@ _is_hip = is_hip()
_is_npu = is_npu()
_is_musa = is_musa()
_is_xpu = is_xpu()
_is_cpu = is_cpu()
if TYPE_CHECKING:
from sglang.srt.constrained.base_grammar_backend import BaseGrammarObject
@@ -89,10 +71,6 @@ if TYPE_CHECKING:
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
if _is_cpu:
from sgl_kernel import assign_extend_cache_locs_cpu
logger = logging.getLogger(__name__)
@@ -351,20 +329,6 @@ def spec_need_hidden_states() -> bool:
return not spec.enable_multi_layer_eagle
@torch.compile(dynamic=True, disable=_is_npu or _is_xpu)
def create_num_accept_tokens_filter(
num_correct_drafts: torch.Tensor,
unfinished_index_device: torch.Tensor,
seq_lens: torch.Tensor,
):
num_accept_tokens_filter = torch.zeros_like(num_correct_drafts)
num_accept_tokens_filter[unfinished_index_device] = (
num_correct_drafts[unfinished_index_device] + 1
)
seq_lens.add_(num_correct_drafts + 1)
return num_accept_tokens_filter
def _select_top_k_tokens_first(
topk_p: torch.Tensor,
topk_index: torch.Tensor,
@@ -826,28 +790,11 @@ def move_accept_tokens_to_target_kvcache(
# The targets lead each request's verify window. Read them from that window,
# not req_to_token: after verify's read-done the scheduler may hand a finished
# request's row to a new request.
window = batch.out_cache_loc.view(bs, -1)
window_rows = torch.arange(bs, device=device)
window_starts = torch.zeros_like(num_correct_drafts)
if _is_cpu:
assign_extend_cache_locs_cpu(
window_rows,
window.int(),
window_starts,
num_correct_drafts + 1,
tgt_cache_loc,
window.shape[1],
)
else:
assign_extend_cache_locs[(bs,)](
window_rows,
window,
window_starts,
num_correct_drafts + 1,
tgt_cache_loc,
window.shape[1],
next_power_of_2(bs),
)
fill_front_out_cache_loc_func(
num_correct_drafts + 1,
batch.out_cache_loc,
tgt_cache_loc,
)
fill_accept_out_cache_loc_func(
accept_index,
batch.out_cache_loc,
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Optional
import torch
from sglang.kernels.ops.speculative.uno_tree import build_uno_tree_proposal
from sglang.srt.managers.utils import GenerationBatchResult
from sglang.srt.model_executor.forward_batch_info import (
CaptureHiddenMode,
@@ -32,7 +33,6 @@ from sglang.srt.speculative.uno_cuda_graph_runner import (
UnoDecodeCudaGraphRunner,
)
from sglang.srt.speculative.uno_info import UnoDraftInput, UnoForwardInput
from sglang.srt.speculative.uno_tree import build_uno_tree_proposal
from sglang.srt.speculative.uno_utils import (
build_uno_draft_input,
pack_uno_tree_result,