[Feat] spec: Let out-of-tree platforms provide their own speculative worker (#45144)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yihao Wang
2026-10-23 21:24:58 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent f09cec65e9
commit af3f9e4c8f
7 changed files with 206 additions and 0 deletions
@@ -17,6 +17,8 @@ import torch
import triton
import triton.language as tl
from sglang.srt.platforms import current_platform
from sglang.srt.platforms.interface import reject_out_of_tree_path
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_musa, is_npu, is_xpu
_is_cuda = is_cuda()
@@ -422,6 +424,10 @@ def build_tree_kernel_efficient_func(
tree_mask_mode,
)
else:
if not (_is_cuda or _is_hip or _is_musa):
reject_out_of_tree_path(
current_platform, subsystem="EAGLE tree construction"
)
sgl_build_tree_kernel_efficient(
parent_list,
top_scores_index,
@@ -539,4 +545,8 @@ def verify_tree_greedy_func(
retrieve_next_sibling=retrieve_next_sibling,
target_predict=target_predict,
)
else:
reject_out_of_tree_path(
current_platform, subsystem="EAGLE greedy tree verification"
)
return predicts, accept_index, accept_token_num
@@ -991,8 +991,21 @@ def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool:
return True
def _check_oot_speculative_algorithm(algorithm: str, device: str) -> None:
if current_platform.is_out_of_tree() and not (
current_platform.supports_speculative_algorithm(algorithm)
):
raise ValueError(
f"{algorithm} speculative decoding is not supported by "
f"{type(current_platform).__name__} on device {device!r}."
)
def _handle_eagle_family(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
_check_oot_speculative_algorithm(
algorithm=cfg.speculative_algorithm, device=cfg.device
)
if cfg.speculative_algorithm == "STANDALONE" and attn_dp_enabled_of(
resolved_view(server_args)
+10
View File
@@ -186,6 +186,16 @@ class SRTPlatform(DeviceMixin):
"""Return a platform implementation for speculative KV-cache locations."""
return None
def get_speculative_worker_cls(self, algorithm: str) -> Optional[type]:
"""Return a worker class replacing the built-in one, or None to keep it.
Consulted for every speculative algorithm on out-of-tree platforms. The
class is constructed with the built-in worker's keyword arguments, must
expose the same scheduler-facing interface, and must reject flags it
cannot serve (for example multi-layer EAGLE).
"""
return None
def get_quantization_config(
self, quantization: str
) -> Optional[Type[QuantizationConfig]]:
@@ -17,6 +17,7 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
)
from sglang.srt.runtime_context import attn_dp_enabled_of
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.utils import log_info_on_rank0
if TYPE_CHECKING:
@@ -62,6 +63,15 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
f"speculative_algorithm={cfg.speculative_algorithm} "
"(only EAGLE/EAGLE3 are supported)"
)
platform_worker_cls = SpeculativeAlgorithm.from_string(
cfg.speculative_algorithm
).get_platform_worker_cls()
if platform_worker_cls is not None:
return (
f"the platform serves {cfg.speculative_algorithm} with "
f"{platform_worker_cls.__name__} (only the built-in worker "
"implements adaptive)"
)
if cfg.speculative_eagle_topk is not None and cfg.speculative_eagle_topk != 1:
return (
f"speculative_eagle_topk={cfg.speculative_eagle_topk} "
@@ -300,6 +300,14 @@ class SpeculativeAlgorithm(Enum):
return num_draft_tokens - 1
return num_draft_tokens
def get_platform_worker_cls(self) -> Optional[type]:
"""Out-of-tree platform's replacement worker, or None for the built-in one."""
from sglang.srt.platforms import current_platform
if not current_platform.is_out_of_tree():
return None
return current_platform.get_speculative_worker_cls(algorithm=self.name)
def create_worker(
self, server_args: ServerArgs
) -> Optional[Union[Type[BaseSpecWorker], Type[TpModelWorker], Type[NGRAMWorker]]]:
@@ -309,6 +317,10 @@ class SpeculativeAlgorithm(Enum):
"Cannot create worker for NONE speculative algorithm."
)
platform_worker_cls = self.get_platform_worker_cls()
if platform_worker_cls is not None:
return platform_worker_cls
if self.is_dflash():
# V2 worker drives both overlap and non-overlap (scheduler runs it
# synchronously when overlap is disabled), same as EAGLE.