[Feat] spec: Let out-of-tree platforms provide their own speculative worker (#45144)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
f09cec65e9
commit
af3f9e4c8f
@@ -17,6 +17,8 @@ import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.platforms.interface import reject_out_of_tree_path
|
||||
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_musa, is_npu, is_xpu
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
@@ -422,6 +424,10 @@ def build_tree_kernel_efficient_func(
|
||||
tree_mask_mode,
|
||||
)
|
||||
else:
|
||||
if not (_is_cuda or _is_hip or _is_musa):
|
||||
reject_out_of_tree_path(
|
||||
current_platform, subsystem="EAGLE tree construction"
|
||||
)
|
||||
sgl_build_tree_kernel_efficient(
|
||||
parent_list,
|
||||
top_scores_index,
|
||||
@@ -539,4 +545,8 @@ def verify_tree_greedy_func(
|
||||
retrieve_next_sibling=retrieve_next_sibling,
|
||||
target_predict=target_predict,
|
||||
)
|
||||
else:
|
||||
reject_out_of_tree_path(
|
||||
current_platform, subsystem="EAGLE greedy tree verification"
|
||||
)
|
||||
return predicts, accept_index, accept_token_num
|
||||
|
||||
@@ -991,8 +991,21 @@ def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def _check_oot_speculative_algorithm(algorithm: str, device: str) -> None:
|
||||
if current_platform.is_out_of_tree() and not (
|
||||
current_platform.supports_speculative_algorithm(algorithm)
|
||||
):
|
||||
raise ValueError(
|
||||
f"{algorithm} speculative decoding is not supported by "
|
||||
f"{type(current_platform).__name__} on device {device!r}."
|
||||
)
|
||||
|
||||
|
||||
def _handle_eagle_family(server_args: ServerArgs) -> None:
|
||||
cfg = resolving_view(server_args)
|
||||
_check_oot_speculative_algorithm(
|
||||
algorithm=cfg.speculative_algorithm, device=cfg.device
|
||||
)
|
||||
|
||||
if cfg.speculative_algorithm == "STANDALONE" and attn_dp_enabled_of(
|
||||
resolved_view(server_args)
|
||||
|
||||
@@ -186,6 +186,16 @@ class SRTPlatform(DeviceMixin):
|
||||
"""Return a platform implementation for speculative KV-cache locations."""
|
||||
return None
|
||||
|
||||
def get_speculative_worker_cls(self, algorithm: str) -> Optional[type]:
|
||||
"""Return a worker class replacing the built-in one, or None to keep it.
|
||||
|
||||
Consulted for every speculative algorithm on out-of-tree platforms. The
|
||||
class is constructed with the built-in worker's keyword arguments, must
|
||||
expose the same scheduler-facing interface, and must reject flags it
|
||||
cannot serve (for example multi-layer EAGLE).
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_quantization_config(
|
||||
self, quantization: str
|
||||
) -> Optional[Type[QuantizationConfig]]:
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
resolving_view,
|
||||
)
|
||||
from sglang.srt.runtime_context import attn_dp_enabled_of
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.utils import log_info_on_rank0
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -62,6 +63,15 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
|
||||
f"speculative_algorithm={cfg.speculative_algorithm} "
|
||||
"(only EAGLE/EAGLE3 are supported)"
|
||||
)
|
||||
platform_worker_cls = SpeculativeAlgorithm.from_string(
|
||||
cfg.speculative_algorithm
|
||||
).get_platform_worker_cls()
|
||||
if platform_worker_cls is not None:
|
||||
return (
|
||||
f"the platform serves {cfg.speculative_algorithm} with "
|
||||
f"{platform_worker_cls.__name__} (only the built-in worker "
|
||||
"implements adaptive)"
|
||||
)
|
||||
if cfg.speculative_eagle_topk is not None and cfg.speculative_eagle_topk != 1:
|
||||
return (
|
||||
f"speculative_eagle_topk={cfg.speculative_eagle_topk} "
|
||||
|
||||
@@ -300,6 +300,14 @@ class SpeculativeAlgorithm(Enum):
|
||||
return num_draft_tokens - 1
|
||||
return num_draft_tokens
|
||||
|
||||
def get_platform_worker_cls(self) -> Optional[type]:
|
||||
"""Out-of-tree platform's replacement worker, or None for the built-in one."""
|
||||
from sglang.srt.platforms import current_platform
|
||||
|
||||
if not current_platform.is_out_of_tree():
|
||||
return None
|
||||
return current_platform.get_speculative_worker_cls(algorithm=self.name)
|
||||
|
||||
def create_worker(
|
||||
self, server_args: ServerArgs
|
||||
) -> Optional[Union[Type[BaseSpecWorker], Type[TpModelWorker], Type[NGRAMWorker]]]:
|
||||
@@ -309,6 +317,10 @@ class SpeculativeAlgorithm(Enum):
|
||||
"Cannot create worker for NONE speculative algorithm."
|
||||
)
|
||||
|
||||
platform_worker_cls = self.get_platform_worker_cls()
|
||||
if platform_worker_cls is not None:
|
||||
return platform_worker_cls
|
||||
|
||||
if self.is_dflash():
|
||||
# V2 worker drives both overlap and non-overlap (scheduler runs it
|
||||
# synchronously when overlap is disabled), same as EAGLE.
|
||||
|
||||
Reference in New Issue
Block a user