Support speculative decoding on CPU (#27862)
Co-authored-by: Valentine233 <xuan.liao@intel.com>
This commit is contained in:
co-authored by
Valentine233
parent
177c048c68
commit
3b43df5b6d
@@ -76,6 +76,7 @@ else:
|
||||
max_pooling_1d_varlen,
|
||||
)
|
||||
from sgl_kernel.kvcacheio import (
|
||||
copy_all_layer_kv_cache_cpu,
|
||||
transfer_kv_all_layer,
|
||||
transfer_kv_all_layer_mla,
|
||||
transfer_kv_per_layer,
|
||||
@@ -113,11 +114,20 @@ else:
|
||||
top_p_renorm_prob,
|
||||
)
|
||||
from sgl_kernel.speculative import (
|
||||
assign_draft_cache_locs_contiguous_cpu,
|
||||
assign_extend_cache_locs_cpu,
|
||||
assign_req_to_token_pool_cpu,
|
||||
build_draft_decode_metadata_cpu,
|
||||
build_tree_kernel_efficient,
|
||||
build_tree_kernel_efficient_cpu,
|
||||
fill_accept_out_cache_loc_cpu,
|
||||
fill_bonus_tokens_cpu,
|
||||
reconstruct_indices_from_tree_mask,
|
||||
rotate_input_ids_cpu,
|
||||
segment_packbits,
|
||||
tree_speculative_sampling_target_only,
|
||||
verify_tree_greedy,
|
||||
verify_tree_greedy_cpu,
|
||||
)
|
||||
from sgl_kernel.top_k import (
|
||||
fast_topk,
|
||||
|
||||
Reference in New Issue
Block a user