qwen 3.8 rebase (#35758)
Co-authored-by: cherichy <cherichy@outlook.com> Co-authored-by: guangyunh-nv <guangyunh@nvidia.com> Co-authored-by: jiahanc <jiahanc@nvidia.com> Co-authored-by: jinyangyuan-nvidia <joyuan@nvidia.com> Co-authored-by: Cheng Hang <chang@nvidia.com> Co-authored-by: Yicheng Qiang <yqiang@nvidia.com> Co-authored-by: Sam Li <lsam@nvidia.com> Co-authored-by: Tom-Zheng <tizheng@nvidia.com> Co-authored-by: Yangmin Li <yangminl@nvidia.com> Co-authored-by: xiaoweiw-nv <xiaoweiw@nvidia.com> Co-authored-by: Zheng Li <lizheng.cs@zju.edu.cn> Co-authored-by: yizhang2077 <1109276519@qq.com> Co-authored-by: Ke Bao <ispobaoke@gmail.com> Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Co-authored-by: Yuhao Yang <47235274+yhyang201@users.noreply.github.com> Co-authored-by: Zijie Xia <zijie.xia@radixark.ai> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
cherichy
guangyunh-nv
jiahanc
jinyangyuan-nvidia
Cheng Hang
Yicheng Qiang
Sam Li
Tom-Zheng
Yangmin Li
xiaoweiw-nv
Zheng Li
yizhang2077
Ke Bao
Xinyuan Tong
Yuhao Yang
Zijie Xia
github-actions[bot]
parent
ca8cc101b8
commit
5f216fc33f
@@ -23,6 +23,7 @@ class KVCacheBuildResult:
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.arg_groups.overrides import resolving_view
|
||||
from sglang.srt.configs.hybrid_arch import (
|
||||
hybrid_gdn_config,
|
||||
hybrid_lightning_config,
|
||||
@@ -62,6 +63,29 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
|
||||
def get_draft_kv_pool(
|
||||
*,
|
||||
draft_worker: BaseTpWorker,
|
||||
spec_algorithm: SpeculativeAlgorithm,
|
||||
server_args: ServerArgs,
|
||||
):
|
||||
"""Return the draft token-to-KV pool for the current draft worker,
|
||||
or None when no draft KV pool is available."""
|
||||
if draft_worker is None or spec_algorithm.is_ngram():
|
||||
return None
|
||||
|
||||
# V2 draft workers exist only on their hosting PP stage; other ranks own no
|
||||
# nested draft worker or draft KV pool.
|
||||
if draft_worker.draft_worker is None:
|
||||
return None
|
||||
|
||||
if resolving_view(server_args).enable_multi_layer_eagle:
|
||||
draft_runner = draft_worker.draft_worker.draft_runner_list[0]
|
||||
else:
|
||||
draft_runner = draft_worker.draft_worker.draft_runner
|
||||
return draft_runner.token_to_kv_pool
|
||||
|
||||
|
||||
def maybe_register_hicache_draft(
|
||||
*,
|
||||
tree_cache,
|
||||
|
||||
Reference in New Issue
Block a user