From a7183a82468011b491094a988b17ae99ef4d72f2 Mon Sep 17 00:00:00 2001 From: Yihao Wang <42559837+AgainstEntropy@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:28:56 -0700 Subject: [PATCH] [Feat] spec: Let out-of-tree platforms provide EAGLE draft graph runners (#45145) Co-authored-by: Claude Opus 5.5 --- docs/docs/hardware-platforms/plugin.mdx | 7 +- python/sglang/srt/platforms/interface.py | 16 +++- .../sglang/srt/speculative/eagle_worker_v2.py | 72 +++++++++++---- .../unit/spec/test_oot_eagle_hooks.py | 88 ++++++++++++++++++- 4 files changed, 162 insertions(+), 21 deletions(-) diff --git a/docs/docs/hardware-platforms/plugin.mdx b/docs/docs/hardware-platforms/plugin.mdx index 381b60ce23..36b0f627b0 100644 --- a/docs/docs/hardware-platforms/plugin.mdx +++ b/docs/docs/hardware-platforms/plugin.mdx @@ -608,7 +608,12 @@ field defaults to the conservative answer. get_graph_runner_cls() return None - Graph Runner class + Graph Runner class. Adaptive speculative decoding also builds one target verify runner per step count from it, passing attn_backend, speculative_num_steps and speculative_num_draft_tokens as keyword arguments. + + + get_speculative_graph_runner_cls(algorithm, phase) + return None + EAGLE draft graph runner for "draft_decode" or "draft_extend" on the built-in worker, constructed like the built-in runner for that phase. None runs the phase eagerly; not consulted when capabilities.graph_capture is False. get_speculative_worker_cls(algorithm) diff --git a/python/sglang/srt/platforms/interface.py b/python/sglang/srt/platforms/interface.py index 8ff75c6b0f..4af19e62b4 100644 --- a/python/sglang/srt/platforms/interface.py +++ b/python/sglang/srt/platforms/interface.py @@ -116,7 +116,10 @@ class SRTPlatform(DeviceMixin): """Return the graph runner class, or None for the in-tree default. ``None`` means "no platform opinion": the caller falls back to the - in-tree device-keyed selection. + in-tree device-keyed selection. Adaptive speculative decoding also + builds one target verify runner per step count from this class, passing + ``attn_backend``, ``speculative_num_steps`` and + ``speculative_num_draft_tokens`` as keyword arguments. """ return None @@ -196,6 +199,17 @@ class SRTPlatform(DeviceMixin): """ return None + def get_speculative_graph_runner_cls( + self, algorithm: str, phase: str + ) -> Optional[type]: + """Return the EAGLE draft graph runner for ``phase``; None runs it eagerly. + + ``phase`` is ``"draft_decode"`` or ``"draft_extend"``; the class is + constructed like the built-in runner for that phase. Not consulted when + ``capabilities.graph_capture`` is False. + """ + return None + def get_quantization_config( self, quantization: str ) -> Optional[Type[QuantizationConfig]]: diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py index 2158580fc3..fca04ec8c1 100644 --- a/python/sglang/srt/speculative/eagle_worker_v2.py +++ b/python/sglang/srt/speculative/eagle_worker_v2.py @@ -1,7 +1,7 @@ import contextlib import logging import time -from typing import List, Optional +from typing import Callable, List, Optional import torch @@ -58,6 +58,7 @@ from sglang.srt.model_executor.runner import ( get_batch_sizes_to_capture, ) from sglang.srt.model_loader.weight_utils import get_pp_stage_load_group +from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import ( get_context, get_device, @@ -143,6 +144,19 @@ _is_xpu = is_xpu() logger = logging.getLogger(__name__) +def _select_spec_graph_runner_cls( + *, + builtin_runner_cls: Callable[[], Optional[type]], + oot_runner_cls: Callable[[], Optional[type]], +) -> Optional[type]: + """Each callable runs only on its own kind of platform; None skips capture.""" + if not current_platform.is_out_of_tree(): + return builtin_runner_cls() + if not current_platform.capabilities.graph_capture: + return None + return oot_runner_cls() + + def _qsa_index_share_requested(hf_config) -> bool: """--json-model-override-args writes top-level hf_config attributes, while checkpoint configs carry the flag on the nested text_config; read both.""" @@ -472,7 +486,21 @@ class EagleDraftWorker(EagleDraftWorkerBase): # Capture draft decode_backend = get_exec().graph.cuda_graph_config.decode.backend capture_bs, _ = get_batch_sizes_to_capture(self.draft_runner) - if self.speculative_num_steps > 1: + draft_runner_cls = ( + _select_spec_graph_runner_cls( + builtin_runner_cls=lambda: Device2DraftCudaGraphRunner[ + self.target_worker.device + ], + oot_runner_cls=lambda: ( + current_platform.get_speculative_graph_runner_cls( + algorithm=self.speculative_algorithm.name, phase="draft_decode" + ) + ), + ) + if self.speculative_num_steps > 1 + else None + ) + if draft_runner_cls is not None: tic = time.perf_counter() before_mem = get_available_gpu_memory(self.device, self.gpu_id) log_info_on_rank0( @@ -481,9 +509,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): f"num_tokens_per_req={self.topk}, bs={capture_bs}, " f"avail mem={before_mem:.2f} GB", ) - self.cuda_graph_runner = Device2DraftCudaGraphRunner[ - self.target_worker.device - ](self) + self.cuda_graph_runner = draft_runner_cls(self) after_mem = get_available_gpu_memory(self.device, self.gpu_id) capture_time = time.perf_counter() - tic self._specialized_graph_memory_usage["draft_decode"] = ( @@ -572,17 +598,27 @@ class EagleDraftWorker(EagleDraftWorkerBase): supports_cuda_draft_extend_graph = ( _is_cuda or _is_musa ) and graph_supported_backend + extend_runner_cls = _select_spec_graph_runner_cls( + builtin_runner_cls=lambda: ( + Device2ExtendCudaGraphRunner[self.target_worker.device] + if ( + _is_npu + or _is_xpu + or supports_cuda_draft_extend_graph + or supports_hip_draft_extend_graph + ) + else None + ), + oot_runner_cls=lambda: current_platform.get_speculative_graph_runner_cls( + algorithm=self.speculative_algorithm.name, phase="draft_extend" + ), + ) # Capture extend # TODO: support draft extend cuda graph for more attention backends if ( self.draft_extend_attn_backend and not envs.SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH.get() - and ( - _is_npu - or _is_xpu - or supports_cuda_draft_extend_graph - or supports_hip_draft_extend_graph - ) + and extend_runner_cls is not None ): tic = time.perf_counter() before_mem = get_available_gpu_memory(self.device, self.gpu_id) @@ -592,9 +628,7 @@ class EagleDraftWorker(EagleDraftWorkerBase): f"num_tokens_per_req={self.speculative_num_draft_tokens}, " f"bs={capture_bs}, avail mem={before_mem:.2f} GB", ) - self.cuda_graph_runner_for_draft_extend = Device2ExtendCudaGraphRunner[ - self.target_worker.device - ](self) + self.cuda_graph_runner_for_draft_extend = extend_runner_cls(self) # draft_extend is the step's last shared-buffer-reading phase; its # read-done event is what the scheduler's WAR barrier waits on. after_mem = get_available_gpu_memory(self.device, self.gpu_id) @@ -1754,10 +1788,16 @@ class EAGLEWorkerV2(BaseSpecWorker): target_model_runner.init_new_workspace = backup_init target_graph_runner = None + TargetGraphRunnerCls = None if not check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED): - TargetGraphRunnerCls = ( - NPUGraphRunner if _is_npu else DecodeCudaGraphRunner + # Same runner class as the startup target verify graph on OOT. + TargetGraphRunnerCls = _select_spec_graph_runner_cls( + builtin_runner_cls=lambda: ( + NPUGraphRunner if _is_npu else DecodeCudaGraphRunner + ), + oot_runner_cls=current_platform.get_graph_runner_cls, ) + if TargetGraphRunnerCls is not None: target_graph_before_mem = get_available_gpu_memory( self.device, self.gpu_id ) diff --git a/test/registered/unit/spec/test_oot_eagle_hooks.py b/test/registered/unit/spec/test_oot_eagle_hooks.py index 69c3e4176d..6086e0d66d 100644 --- a/test/registered/unit/spec/test_oot_eagle_hooks.py +++ b/test/registered/unit/spec/test_oot_eagle_hooks.py @@ -2,15 +2,16 @@ import contextlib import sys import types import unittest +from types import SimpleNamespace from unittest.mock import Mock, patch, sentinel import torch from sglang.kernels.ops.speculative import spec_tree from sglang.srt.arg_groups.speculative_hook import _check_oot_speculative_algorithm -from sglang.srt.platforms.interface import SRTPlatform +from sglang.srt.platforms.interface import PlatformCapabilities, SRTPlatform from sglang.srt.server_args import ServerArgs -from sglang.srt.speculative import eagle_utils +from sglang.srt.speculative import eagle_utils, eagle_worker_v2 from sglang.srt.speculative.adaptive_spec_params import adaptive_unsupported_reason from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_cpu_ci @@ -28,9 +29,14 @@ class _PlatformWorker: def _platform( - *, out_of_tree: bool = True, supported: bool = True, worker_cls=None + *, + out_of_tree: bool = True, + supported: bool = True, + worker_cls=None, + graph: bool = True, ) -> Mock: platform = Mock(spec=SRTPlatform) + platform.capabilities = PlatformCapabilities(graph_capture=graph) platform.device_name = "custom" platform.is_out_of_tree.return_value = out_of_tree platform.supports_speculative_algorithm.return_value = supported @@ -142,5 +148,81 @@ class TestOOTEagleKernelDispatch(CustomTestCase): ) +class TestOOTSpecGraphRunnerSelection(CustomTestCase): + def _select(self, platform: Mock, oot_runner_cls): + def builtin(): + raise KeyError("custom") # an OOT device is not a key of the maps + + with patch.object(eagle_worker_v2, "current_platform", platform): + return eagle_worker_v2._select_spec_graph_runner_cls( + builtin_runner_cls=builtin, oot_runner_cls=oot_runner_cls + ) + + def test_no_capture_without_graph_capture(self): + oot = Mock(return_value=sentinel.runner) + self.assertIsNone(self._select(_platform(graph=False), oot)) + oot.assert_not_called() + + def test_builtin_platform_never_consults_the_platform(self): + oot = Mock(return_value=sentinel.runner) + with patch.object( + eagle_worker_v2, "current_platform", _platform(out_of_tree=False) + ): + selected = eagle_worker_v2._select_spec_graph_runner_cls( + builtin_runner_cls=lambda: sentinel.builtin_runner, + oot_runner_cls=oot, + ) + self.assertIs(selected, sentinel.builtin_runner) + oot.assert_not_called() + + def test_out_of_tree_draft_graphs_come_from_the_platform(self): + """An OOT device name used to index the device-keyed runner maps (KeyError).""" + platform = _platform() + runners = { + phase: Mock(name=phase) for phase in ("draft_decode", "draft_extend") + } + platform.get_speculative_graph_runner_cls.side_effect = ( + lambda algorithm, phase: runners[phase] + ) + worker = SimpleNamespace( + speculative_algorithm=SpeculativeAlgorithm.EAGLE, + speculative_num_steps=3, + speculative_num_draft_tokens=4, + topk=1, + target_worker=SimpleNamespace(device="custom"), + draft_runner=None, + device="custom", + gpu_id=0, + draft_attn_backend=object(), + draft_extend_attn_backend=object(), + _specialized_graph_memory_usage={}, + _specialized_graph_time_usage={}, + ) + decode_cfg = SimpleNamespace(backend="custom") + exec_cfg = SimpleNamespace( + graph=SimpleNamespace(cuda_graph_config=SimpleNamespace(decode=decode_cfg)) + ) + with contextlib.ExitStack() as stack: + for flag in DEVICE_FLAGS: + stack.enter_context(patch.object(eagle_worker_v2, flag, False)) + for name, value in ( + ("current_platform", platform), + ("check_cuda_graph_backend", lambda *args: False), + ("get_model", lambda: SimpleNamespace(model_impl="auto")), + ("get_exec", lambda: exec_cfg), + ("get_batch_sizes_to_capture", lambda runner: ([1], None)), + ("get_available_gpu_memory", lambda *args: 0.0), + ): + stack.enter_context(patch.object(eagle_worker_v2, name, value)) + eagle_worker_v2.EagleDraftWorker._capture_cuda_graphs(worker) + for phase, runner in runners.items(): + runner.assert_called_once_with(worker) + self.assertIs(worker.cuda_graph_runner, runners["draft_decode"].return_value) + self.assertIs( + worker.cuda_graph_runner_for_draft_extend, + runners["draft_extend"].return_value, + ) + + if __name__ == "__main__": unittest.main()