[Feat] spec: Let out-of-tree platforms provide their own speculative worker (#45144)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yihao Wang
2026-10-23 21:24:58 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent f09cec65e9
commit af3f9e4c8f
7 changed files with 206 additions and 0 deletions
+5
View File
@@ -610,6 +610,11 @@ field defaults to the conservative answer.
<td><code>return None</code></td>
<td>Graph Runner class</td>
</tr>
<tr>
<td><code>get_speculative_worker_cls(algorithm)</code></td>
<td><code>return None</code></td>
<td>Worker class that replaces the built-in speculative worker, for any algorithm. It is constructed with the built-in worker&#39;s keyword arguments, must expose the same scheduler-facing interface, and must reject flags it cannot serve (such as multi-layer EAGLE). <code>--speculative-adaptive</code> falls back to static parameters with a platform worker, since only the built-in worker implements it. EAGLE, EAGLE3, NEXTN (asked as <code>"EAGLE"</code>) and STANDALONE also require <code>supports_speculative_algorithm(algorithm)</code> to return True.</td>
</tr>
<tr>
<td><code>get_kv_pool_cls(kind=...)</code></td>
<td><code>return None</code></td>
@@ -17,6 +17,8 @@ import torch
import triton
import triton.language as tl
from sglang.srt.platforms import current_platform
from sglang.srt.platforms.interface import reject_out_of_tree_path
from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_musa, is_npu, is_xpu
_is_cuda = is_cuda()
@@ -422,6 +424,10 @@ def build_tree_kernel_efficient_func(
tree_mask_mode,
)
else:
if not (_is_cuda or _is_hip or _is_musa):
reject_out_of_tree_path(
current_platform, subsystem="EAGLE tree construction"
)
sgl_build_tree_kernel_efficient(
parent_list,
top_scores_index,
@@ -539,4 +545,8 @@ def verify_tree_greedy_func(
retrieve_next_sibling=retrieve_next_sibling,
target_predict=target_predict,
)
else:
reject_out_of_tree_path(
current_platform, subsystem="EAGLE greedy tree verification"
)
return predicts, accept_index, accept_token_num
@@ -991,8 +991,21 @@ def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool:
return True
def _check_oot_speculative_algorithm(algorithm: str, device: str) -> None:
if current_platform.is_out_of_tree() and not (
current_platform.supports_speculative_algorithm(algorithm)
):
raise ValueError(
f"{algorithm} speculative decoding is not supported by "
f"{type(current_platform).__name__} on device {device!r}."
)
def _handle_eagle_family(server_args: ServerArgs) -> None:
cfg = resolving_view(server_args)
_check_oot_speculative_algorithm(
algorithm=cfg.speculative_algorithm, device=cfg.device
)
if cfg.speculative_algorithm == "STANDALONE" and attn_dp_enabled_of(
resolved_view(server_args)
+10
View File
@@ -186,6 +186,16 @@ class SRTPlatform(DeviceMixin):
"""Return a platform implementation for speculative KV-cache locations."""
return None
def get_speculative_worker_cls(self, algorithm: str) -> Optional[type]:
"""Return a worker class replacing the built-in one, or None to keep it.
Consulted for every speculative algorithm on out-of-tree platforms. The
class is constructed with the built-in worker's keyword arguments, must
expose the same scheduler-facing interface, and must reject flags it
cannot serve (for example multi-layer EAGLE).
"""
return None
def get_quantization_config(
self, quantization: str
) -> Optional[Type[QuantizationConfig]]:
@@ -17,6 +17,7 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
)
from sglang.srt.runtime_context import attn_dp_enabled_of
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.utils import log_info_on_rank0
if TYPE_CHECKING:
@@ -62,6 +63,15 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None:
f"speculative_algorithm={cfg.speculative_algorithm} "
"(only EAGLE/EAGLE3 are supported)"
)
platform_worker_cls = SpeculativeAlgorithm.from_string(
cfg.speculative_algorithm
).get_platform_worker_cls()
if platform_worker_cls is not None:
return (
f"the platform serves {cfg.speculative_algorithm} with "
f"{platform_worker_cls.__name__} (only the built-in worker "
"implements adaptive)"
)
if cfg.speculative_eagle_topk is not None and cfg.speculative_eagle_topk != 1:
return (
f"speculative_eagle_topk={cfg.speculative_eagle_topk} "
@@ -300,6 +300,14 @@ class SpeculativeAlgorithm(Enum):
return num_draft_tokens - 1
return num_draft_tokens
def get_platform_worker_cls(self) -> Optional[type]:
"""Out-of-tree platform's replacement worker, or None for the built-in one."""
from sglang.srt.platforms import current_platform
if not current_platform.is_out_of_tree():
return None
return current_platform.get_speculative_worker_cls(algorithm=self.name)
def create_worker(
self, server_args: ServerArgs
) -> Optional[Union[Type[BaseSpecWorker], Type[TpModelWorker], Type[NGRAMWorker]]]:
@@ -309,6 +317,10 @@ class SpeculativeAlgorithm(Enum):
"Cannot create worker for NONE speculative algorithm."
)
platform_worker_cls = self.get_platform_worker_cls()
if platform_worker_cls is not None:
return platform_worker_cls
if self.is_dflash():
# V2 worker drives both overlap and non-overlap (scheduler runs it
# synchronously when overlap is disabled), same as EAGLE.
@@ -0,0 +1,146 @@
import contextlib
import sys
import types
import unittest
from unittest.mock import Mock, patch, sentinel
import torch
from sglang.kernels.ops.speculative import spec_tree
from sglang.srt.arg_groups.speculative_hook import _check_oot_speculative_algorithm
from sglang.srt.platforms.interface import SRTPlatform
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative import eagle_utils
from sglang.srt.speculative.adaptive_spec_params import adaptive_unsupported_reason
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
HOOK_MODULE = "sglang.srt.arg_groups.speculative_hook"
WORKER_MODULE = "sglang.srt.speculative.eagle_worker_v2"
DEVICE_FLAGS = ("_is_cuda", "_is_hip", "_is_musa", "_is_cpu", "_is_npu", "_is_xpu")
class _PlatformWorker:
pass
def _platform(
*, out_of_tree: bool = True, supported: bool = True, worker_cls=None
) -> Mock:
platform = Mock(spec=SRTPlatform)
platform.device_name = "custom"
platform.is_out_of_tree.return_value = out_of_tree
platform.supports_speculative_algorithm.return_value = supported
platform.get_speculative_worker_cls.return_value = worker_cls
return platform
class TestOOTEagleWorkerSelection(CustomTestCase):
def _create_worker(self, platform: Mock):
args = ServerArgs(model_path="dummy", device="custom")
builtin = types.ModuleType(WORKER_MODULE)
builtin.EAGLEWorkerV2 = sentinel.builtin_worker
with (
patch("sglang.srt.platforms.current_platform", platform),
patch.dict(sys.modules, {WORKER_MODULE: builtin}),
):
return SpeculativeAlgorithm.EAGLE.create_worker(args)
def test_platform_worker_replaces_builtin(self):
platform = _platform(worker_cls=sentinel.platform_worker)
self.assertIs(self._create_worker(platform), sentinel.platform_worker)
platform.get_speculative_worker_cls.assert_called_once_with(algorithm="EAGLE")
def test_builtin_worker_when_platform_declines(self):
declining = _platform(worker_cls=None)
self.assertIs(self._create_worker(declining), sentinel.builtin_worker)
in_tree = _platform(out_of_tree=False, worker_cls=sentinel.platform_worker)
self.assertIs(self._create_worker(in_tree), sentinel.builtin_worker)
in_tree.get_speculative_worker_cls.assert_not_called()
class TestOOTEagleAlgorithmGate(CustomTestCase):
def test_unsupported_out_of_tree_platform_is_rejected(self):
with patch(f"{HOOK_MODULE}.current_platform", _platform(supported=False)):
with self.assertRaisesRegex(ValueError, "EAGLE speculative decoding"):
_check_oot_speculative_algorithm(algorithm="EAGLE", device="custom")
def test_builtin_platforms_are_not_gated(self):
platform = _platform(out_of_tree=False, supported=False)
with patch(f"{HOOK_MODULE}.current_platform", platform):
_check_oot_speculative_algorithm(algorithm="EAGLE", device="cuda")
platform.supports_speculative_algorithm.assert_not_called()
class TestOOTPlatformWorkerAdaptive(CustomTestCase):
def _reason(self, platform: Mock):
args = ServerArgs(
model_path="dummy", device="custom", speculative_algorithm="EAGLE"
)
with patch("sglang.srt.platforms.current_platform", platform):
return adaptive_unsupported_reason(args)
def test_platform_worker_disables_adaptive(self):
reason = self._reason(_platform(worker_cls=_PlatformWorker))
self.assertIn("_PlatformWorker", reason)
def test_builtin_worker_keeps_adaptive(self):
self.assertIsNone(self._reason(_platform(worker_cls=None)))
self.assertIsNone(
self._reason(_platform(out_of_tree=False, worker_cls=_PlatformWorker))
)
class TestOOTEagleKernelDispatch(CustomTestCase):
"""An OOT device has no tree kernel; it must fail loudly, not compute nothing."""
def _run(self, platform: Mock, fn):
with contextlib.ExitStack() as stack:
for flag in DEVICE_FLAGS:
stack.enter_context(patch.object(spec_tree, flag, False))
stack.enter_context(patch.object(spec_tree, "current_platform", platform))
return fn()
def _build_tree(self):
return eagle_utils.build_tree_kernel_efficient(
bonus_tokens=torch.tensor([5]),
parent_list=torch.zeros((1, 2), dtype=torch.long),
top_scores_index=torch.zeros((1, 2), dtype=torch.long),
draft_tokens=torch.tensor([[6, 7]]),
seq_lens=torch.tensor([4]),
seq_lens_sum=4,
topk=1,
spec_steps=2,
num_verify_tokens=3,
tree_mask_mode=spec_tree.TreeMaskMode.QLEN_ONLY,
)
def _verify(self):
return spec_tree.verify_tree_greedy_func(
predicts=None,
accept_index=None,
accept_token_num=None,
candidates=None,
retrieve_index=None,
retrieve_next_token=None,
retrieve_next_sibling=None,
target_predict=None,
)
def test_out_of_tree_device_is_rejected(self):
for name, fn in (("build_tree", self._build_tree), ("verify", self._verify)):
with self.subTest(op=name):
with self.assertRaisesRegex(NotImplementedError, "no platform seam"):
self._run(_platform(), fn)
def test_in_tree_verify_fallthrough_is_unchanged(self):
self.assertEqual(
self._run(_platform(out_of_tree=False), self._verify), (None, None, None)
)
if __name__ == "__main__":
unittest.main()