From af3f9e4c8f7a3bfc60b95fded402769d8b50f7be Mon Sep 17 00:00:00 2001 From: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:24:58 -0700 Subject: [PATCH] [Feat] spec: Let out-of-tree platforms provide their own speculative worker (#45144) Co-authored-by: Claude Opus 5.5 --- docs/docs/hardware-platforms/plugin.mdx | 5 + .../kernels/ops/speculative/spec_tree.py | 10 ++ .../sglang/srt/arg_groups/speculative_hook.py | 13 ++ python/sglang/srt/platforms/interface.py | 10 ++ .../srt/speculative/adaptive_spec_params.py | 10 ++ python/sglang/srt/speculative/spec_info.py | 12 ++ .../unit/spec/test_oot_eagle_hooks.py | 146 ++++++++++++++++++ 7 files changed, 206 insertions(+) create mode 100644 test/registered/unit/spec/test_oot_eagle_hooks.py diff --git a/docs/docs/hardware-platforms/plugin.mdx b/docs/docs/hardware-platforms/plugin.mdx index 9d88ce6d09..381b60ce23 100644 --- a/docs/docs/hardware-platforms/plugin.mdx +++ b/docs/docs/hardware-platforms/plugin.mdx @@ -610,6 +610,11 @@ field defaults to the conservative answer. return None Graph Runner class + + get_speculative_worker_cls(algorithm) + return None + Worker class that replaces the built-in speculative worker, for any algorithm. It is constructed with the built-in worker's keyword arguments, must expose the same scheduler-facing interface, and must reject flags it cannot serve (such as multi-layer EAGLE). --speculative-adaptive falls back to static parameters with a platform worker, since only the built-in worker implements it. EAGLE, EAGLE3, NEXTN (asked as "EAGLE") and STANDALONE also require supports_speculative_algorithm(algorithm) to return True. + get_kv_pool_cls(kind=...) return None diff --git a/python/sglang/kernels/ops/speculative/spec_tree.py b/python/sglang/kernels/ops/speculative/spec_tree.py index 138145d44d..18009a8ea5 100644 --- a/python/sglang/kernels/ops/speculative/spec_tree.py +++ b/python/sglang/kernels/ops/speculative/spec_tree.py @@ -17,6 +17,8 @@ import torch import triton import triton.language as tl +from sglang.srt.platforms import current_platform +from sglang.srt.platforms.interface import reject_out_of_tree_path from sglang.srt.utils import is_cpu, is_cuda, is_hip, is_musa, is_npu, is_xpu _is_cuda = is_cuda() @@ -422,6 +424,10 @@ def build_tree_kernel_efficient_func( tree_mask_mode, ) else: + if not (_is_cuda or _is_hip or _is_musa): + reject_out_of_tree_path( + current_platform, subsystem="EAGLE tree construction" + ) sgl_build_tree_kernel_efficient( parent_list, top_scores_index, @@ -539,4 +545,8 @@ def verify_tree_greedy_func( retrieve_next_sibling=retrieve_next_sibling, target_predict=target_predict, ) + else: + reject_out_of_tree_path( + current_platform, subsystem="EAGLE greedy tree verification" + ) return predicts, accept_index, accept_token_num diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py index fb9e7912ce..18ceaa1b58 100644 --- a/python/sglang/srt/arg_groups/speculative_hook.py +++ b/python/sglang/srt/arg_groups/speculative_hook.py @@ -991,8 +991,21 @@ def _handle_iquest_q1_mtp_draft(server_args: ServerArgs) -> bool: return True +def _check_oot_speculative_algorithm(algorithm: str, device: str) -> None: + if current_platform.is_out_of_tree() and not ( + current_platform.supports_speculative_algorithm(algorithm) + ): + raise ValueError( + f"{algorithm} speculative decoding is not supported by " + f"{type(current_platform).__name__} on device {device!r}." + ) + + def _handle_eagle_family(server_args: ServerArgs) -> None: cfg = resolving_view(server_args) + _check_oot_speculative_algorithm( + algorithm=cfg.speculative_algorithm, device=cfg.device + ) if cfg.speculative_algorithm == "STANDALONE" and attn_dp_enabled_of( resolved_view(server_args) diff --git a/python/sglang/srt/platforms/interface.py b/python/sglang/srt/platforms/interface.py index 843ff70903..8ff75c6b0f 100644 --- a/python/sglang/srt/platforms/interface.py +++ b/python/sglang/srt/platforms/interface.py @@ -186,6 +186,16 @@ class SRTPlatform(DeviceMixin): """Return a platform implementation for speculative KV-cache locations.""" return None + def get_speculative_worker_cls(self, algorithm: str) -> Optional[type]: + """Return a worker class replacing the built-in one, or None to keep it. + + Consulted for every speculative algorithm on out-of-tree platforms. The + class is constructed with the built-in worker's keyword arguments, must + expose the same scheduler-facing interface, and must reject flags it + cannot serve (for example multi-layer EAGLE). + """ + return None + def get_quantization_config( self, quantization: str ) -> Optional[Type[QuantizationConfig]]: diff --git a/python/sglang/srt/speculative/adaptive_spec_params.py b/python/sglang/srt/speculative/adaptive_spec_params.py index 268ceaeafa..091a070f68 100644 --- a/python/sglang/srt/speculative/adaptive_spec_params.py +++ b/python/sglang/srt/speculative/adaptive_spec_params.py @@ -17,6 +17,7 @@ from sglang.srt.arg_groups.overrides import ( resolving_view, ) from sglang.srt.runtime_context import attn_dp_enabled_of +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.utils import log_info_on_rank0 if TYPE_CHECKING: @@ -62,6 +63,15 @@ def adaptive_unsupported_reason(server_args: ServerArgs) -> str | None: f"speculative_algorithm={cfg.speculative_algorithm} " "(only EAGLE/EAGLE3 are supported)" ) + platform_worker_cls = SpeculativeAlgorithm.from_string( + cfg.speculative_algorithm + ).get_platform_worker_cls() + if platform_worker_cls is not None: + return ( + f"the platform serves {cfg.speculative_algorithm} with " + f"{platform_worker_cls.__name__} (only the built-in worker " + "implements adaptive)" + ) if cfg.speculative_eagle_topk is not None and cfg.speculative_eagle_topk != 1: return ( f"speculative_eagle_topk={cfg.speculative_eagle_topk} " diff --git a/python/sglang/srt/speculative/spec_info.py b/python/sglang/srt/speculative/spec_info.py index f3f4ab04fd..3576481be0 100644 --- a/python/sglang/srt/speculative/spec_info.py +++ b/python/sglang/srt/speculative/spec_info.py @@ -300,6 +300,14 @@ class SpeculativeAlgorithm(Enum): return num_draft_tokens - 1 return num_draft_tokens + def get_platform_worker_cls(self) -> Optional[type]: + """Out-of-tree platform's replacement worker, or None for the built-in one.""" + from sglang.srt.platforms import current_platform + + if not current_platform.is_out_of_tree(): + return None + return current_platform.get_speculative_worker_cls(algorithm=self.name) + def create_worker( self, server_args: ServerArgs ) -> Optional[Union[Type[BaseSpecWorker], Type[TpModelWorker], Type[NGRAMWorker]]]: @@ -309,6 +317,10 @@ class SpeculativeAlgorithm(Enum): "Cannot create worker for NONE speculative algorithm." ) + platform_worker_cls = self.get_platform_worker_cls() + if platform_worker_cls is not None: + return platform_worker_cls + if self.is_dflash(): # V2 worker drives both overlap and non-overlap (scheduler runs it # synchronously when overlap is disabled), same as EAGLE. diff --git a/test/registered/unit/spec/test_oot_eagle_hooks.py b/test/registered/unit/spec/test_oot_eagle_hooks.py new file mode 100644 index 0000000000..69c3e4176d --- /dev/null +++ b/test/registered/unit/spec/test_oot_eagle_hooks.py @@ -0,0 +1,146 @@ +import contextlib +import sys +import types +import unittest +from unittest.mock import Mock, patch, sentinel + +import torch + +from sglang.kernels.ops.speculative import spec_tree +from sglang.srt.arg_groups.speculative_hook import _check_oot_speculative_algorithm +from sglang.srt.platforms.interface import SRTPlatform +from sglang.srt.server_args import ServerArgs +from sglang.srt.speculative import eagle_utils +from sglang.srt.speculative.adaptive_spec_params import adaptive_unsupported_reason +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + +HOOK_MODULE = "sglang.srt.arg_groups.speculative_hook" +WORKER_MODULE = "sglang.srt.speculative.eagle_worker_v2" +DEVICE_FLAGS = ("_is_cuda", "_is_hip", "_is_musa", "_is_cpu", "_is_npu", "_is_xpu") + + +class _PlatformWorker: + pass + + +def _platform( + *, out_of_tree: bool = True, supported: bool = True, worker_cls=None +) -> Mock: + platform = Mock(spec=SRTPlatform) + platform.device_name = "custom" + platform.is_out_of_tree.return_value = out_of_tree + platform.supports_speculative_algorithm.return_value = supported + platform.get_speculative_worker_cls.return_value = worker_cls + return platform + + +class TestOOTEagleWorkerSelection(CustomTestCase): + def _create_worker(self, platform: Mock): + args = ServerArgs(model_path="dummy", device="custom") + builtin = types.ModuleType(WORKER_MODULE) + builtin.EAGLEWorkerV2 = sentinel.builtin_worker + with ( + patch("sglang.srt.platforms.current_platform", platform), + patch.dict(sys.modules, {WORKER_MODULE: builtin}), + ): + return SpeculativeAlgorithm.EAGLE.create_worker(args) + + def test_platform_worker_replaces_builtin(self): + platform = _platform(worker_cls=sentinel.platform_worker) + self.assertIs(self._create_worker(platform), sentinel.platform_worker) + platform.get_speculative_worker_cls.assert_called_once_with(algorithm="EAGLE") + + def test_builtin_worker_when_platform_declines(self): + declining = _platform(worker_cls=None) + self.assertIs(self._create_worker(declining), sentinel.builtin_worker) + in_tree = _platform(out_of_tree=False, worker_cls=sentinel.platform_worker) + self.assertIs(self._create_worker(in_tree), sentinel.builtin_worker) + in_tree.get_speculative_worker_cls.assert_not_called() + + +class TestOOTEagleAlgorithmGate(CustomTestCase): + def test_unsupported_out_of_tree_platform_is_rejected(self): + with patch(f"{HOOK_MODULE}.current_platform", _platform(supported=False)): + with self.assertRaisesRegex(ValueError, "EAGLE speculative decoding"): + _check_oot_speculative_algorithm(algorithm="EAGLE", device="custom") + + def test_builtin_platforms_are_not_gated(self): + platform = _platform(out_of_tree=False, supported=False) + with patch(f"{HOOK_MODULE}.current_platform", platform): + _check_oot_speculative_algorithm(algorithm="EAGLE", device="cuda") + platform.supports_speculative_algorithm.assert_not_called() + + +class TestOOTPlatformWorkerAdaptive(CustomTestCase): + def _reason(self, platform: Mock): + args = ServerArgs( + model_path="dummy", device="custom", speculative_algorithm="EAGLE" + ) + with patch("sglang.srt.platforms.current_platform", platform): + return adaptive_unsupported_reason(args) + + def test_platform_worker_disables_adaptive(self): + reason = self._reason(_platform(worker_cls=_PlatformWorker)) + self.assertIn("_PlatformWorker", reason) + + def test_builtin_worker_keeps_adaptive(self): + self.assertIsNone(self._reason(_platform(worker_cls=None))) + self.assertIsNone( + self._reason(_platform(out_of_tree=False, worker_cls=_PlatformWorker)) + ) + + +class TestOOTEagleKernelDispatch(CustomTestCase): + """An OOT device has no tree kernel; it must fail loudly, not compute nothing.""" + + def _run(self, platform: Mock, fn): + with contextlib.ExitStack() as stack: + for flag in DEVICE_FLAGS: + stack.enter_context(patch.object(spec_tree, flag, False)) + stack.enter_context(patch.object(spec_tree, "current_platform", platform)) + return fn() + + def _build_tree(self): + return eagle_utils.build_tree_kernel_efficient( + bonus_tokens=torch.tensor([5]), + parent_list=torch.zeros((1, 2), dtype=torch.long), + top_scores_index=torch.zeros((1, 2), dtype=torch.long), + draft_tokens=torch.tensor([[6, 7]]), + seq_lens=torch.tensor([4]), + seq_lens_sum=4, + topk=1, + spec_steps=2, + num_verify_tokens=3, + tree_mask_mode=spec_tree.TreeMaskMode.QLEN_ONLY, + ) + + def _verify(self): + return spec_tree.verify_tree_greedy_func( + predicts=None, + accept_index=None, + accept_token_num=None, + candidates=None, + retrieve_index=None, + retrieve_next_token=None, + retrieve_next_sibling=None, + target_predict=None, + ) + + def test_out_of_tree_device_is_rejected(self): + for name, fn in (("build_tree", self._build_tree), ("verify", self._verify)): + with self.subTest(op=name): + with self.assertRaisesRegex(NotImplementedError, "no platform seam"): + self._run(_platform(), fn) + + def test_in_tree_verify_fallthrough_is_unchanged(self): + self.assertEqual( + self._run(_platform(out_of_tree=False), self._verify), (None, None, None) + ) + + +if __name__ == "__main__": + unittest.main()