diff --git a/docs/docs/hardware-platforms/plugin.mdx b/docs/docs/hardware-platforms/plugin.mdx
index 381b60ce23..36b0f627b0 100644
--- a/docs/docs/hardware-platforms/plugin.mdx
+++ b/docs/docs/hardware-platforms/plugin.mdx
@@ -608,7 +608,12 @@ field defaults to the conservative answer.
get_graph_runner_cls() |
return None |
- Graph Runner class |
+ Graph Runner class. Adaptive speculative decoding also builds one target verify runner per step count from it, passing attn_backend, speculative_num_steps and speculative_num_draft_tokens as keyword arguments. |
+
+
+ get_speculative_graph_runner_cls(algorithm, phase) |
+ return None |
+ EAGLE draft graph runner for "draft_decode" or "draft_extend" on the built-in worker, constructed like the built-in runner for that phase. None runs the phase eagerly; not consulted when capabilities.graph_capture is False. |
get_speculative_worker_cls(algorithm) |
diff --git a/python/sglang/srt/platforms/interface.py b/python/sglang/srt/platforms/interface.py
index 8ff75c6b0f..4af19e62b4 100644
--- a/python/sglang/srt/platforms/interface.py
+++ b/python/sglang/srt/platforms/interface.py
@@ -116,7 +116,10 @@ class SRTPlatform(DeviceMixin):
"""Return the graph runner class, or None for the in-tree default.
``None`` means "no platform opinion": the caller falls back to the
- in-tree device-keyed selection.
+ in-tree device-keyed selection. Adaptive speculative decoding also
+ builds one target verify runner per step count from this class, passing
+ ``attn_backend``, ``speculative_num_steps`` and
+ ``speculative_num_draft_tokens`` as keyword arguments.
"""
return None
@@ -196,6 +199,17 @@ class SRTPlatform(DeviceMixin):
"""
return None
+ def get_speculative_graph_runner_cls(
+ self, algorithm: str, phase: str
+ ) -> Optional[type]:
+ """Return the EAGLE draft graph runner for ``phase``; None runs it eagerly.
+
+ ``phase`` is ``"draft_decode"`` or ``"draft_extend"``; the class is
+ constructed like the built-in runner for that phase. Not consulted when
+ ``capabilities.graph_capture`` is False.
+ """
+ return None
+
def get_quantization_config(
self, quantization: str
) -> Optional[Type[QuantizationConfig]]:
diff --git a/python/sglang/srt/speculative/eagle_worker_v2.py b/python/sglang/srt/speculative/eagle_worker_v2.py
index 2158580fc3..fca04ec8c1 100644
--- a/python/sglang/srt/speculative/eagle_worker_v2.py
+++ b/python/sglang/srt/speculative/eagle_worker_v2.py
@@ -1,7 +1,7 @@
import contextlib
import logging
import time
-from typing import List, Optional
+from typing import Callable, List, Optional
import torch
@@ -58,6 +58,7 @@ from sglang.srt.model_executor.runner import (
get_batch_sizes_to_capture,
)
from sglang.srt.model_loader.weight_utils import get_pp_stage_load_group
+from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import (
get_context,
get_device,
@@ -143,6 +144,19 @@ _is_xpu = is_xpu()
logger = logging.getLogger(__name__)
+def _select_spec_graph_runner_cls(
+ *,
+ builtin_runner_cls: Callable[[], Optional[type]],
+ oot_runner_cls: Callable[[], Optional[type]],
+) -> Optional[type]:
+ """Each callable runs only on its own kind of platform; None skips capture."""
+ if not current_platform.is_out_of_tree():
+ return builtin_runner_cls()
+ if not current_platform.capabilities.graph_capture:
+ return None
+ return oot_runner_cls()
+
+
def _qsa_index_share_requested(hf_config) -> bool:
"""--json-model-override-args writes top-level hf_config attributes, while
checkpoint configs carry the flag on the nested text_config; read both."""
@@ -472,7 +486,21 @@ class EagleDraftWorker(EagleDraftWorkerBase):
# Capture draft
decode_backend = get_exec().graph.cuda_graph_config.decode.backend
capture_bs, _ = get_batch_sizes_to_capture(self.draft_runner)
- if self.speculative_num_steps > 1:
+ draft_runner_cls = (
+ _select_spec_graph_runner_cls(
+ builtin_runner_cls=lambda: Device2DraftCudaGraphRunner[
+ self.target_worker.device
+ ],
+ oot_runner_cls=lambda: (
+ current_platform.get_speculative_graph_runner_cls(
+ algorithm=self.speculative_algorithm.name, phase="draft_decode"
+ )
+ ),
+ )
+ if self.speculative_num_steps > 1
+ else None
+ )
+ if draft_runner_cls is not None:
tic = time.perf_counter()
before_mem = get_available_gpu_memory(self.device, self.gpu_id)
log_info_on_rank0(
@@ -481,9 +509,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
f"num_tokens_per_req={self.topk}, bs={capture_bs}, "
f"avail mem={before_mem:.2f} GB",
)
- self.cuda_graph_runner = Device2DraftCudaGraphRunner[
- self.target_worker.device
- ](self)
+ self.cuda_graph_runner = draft_runner_cls(self)
after_mem = get_available_gpu_memory(self.device, self.gpu_id)
capture_time = time.perf_counter() - tic
self._specialized_graph_memory_usage["draft_decode"] = (
@@ -572,17 +598,27 @@ class EagleDraftWorker(EagleDraftWorkerBase):
supports_cuda_draft_extend_graph = (
_is_cuda or _is_musa
) and graph_supported_backend
+ extend_runner_cls = _select_spec_graph_runner_cls(
+ builtin_runner_cls=lambda: (
+ Device2ExtendCudaGraphRunner[self.target_worker.device]
+ if (
+ _is_npu
+ or _is_xpu
+ or supports_cuda_draft_extend_graph
+ or supports_hip_draft_extend_graph
+ )
+ else None
+ ),
+ oot_runner_cls=lambda: current_platform.get_speculative_graph_runner_cls(
+ algorithm=self.speculative_algorithm.name, phase="draft_extend"
+ ),
+ )
# Capture extend
# TODO: support draft extend cuda graph for more attention backends
if (
self.draft_extend_attn_backend
and not envs.SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH.get()
- and (
- _is_npu
- or _is_xpu
- or supports_cuda_draft_extend_graph
- or supports_hip_draft_extend_graph
- )
+ and extend_runner_cls is not None
):
tic = time.perf_counter()
before_mem = get_available_gpu_memory(self.device, self.gpu_id)
@@ -592,9 +628,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
f"num_tokens_per_req={self.speculative_num_draft_tokens}, "
f"bs={capture_bs}, avail mem={before_mem:.2f} GB",
)
- self.cuda_graph_runner_for_draft_extend = Device2ExtendCudaGraphRunner[
- self.target_worker.device
- ](self)
+ self.cuda_graph_runner_for_draft_extend = extend_runner_cls(self)
# draft_extend is the step's last shared-buffer-reading phase; its
# read-done event is what the scheduler's WAR barrier waits on.
after_mem = get_available_gpu_memory(self.device, self.gpu_id)
@@ -1754,10 +1788,16 @@ class EAGLEWorkerV2(BaseSpecWorker):
target_model_runner.init_new_workspace = backup_init
target_graph_runner = None
+ TargetGraphRunnerCls = None
if not check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED):
- TargetGraphRunnerCls = (
- NPUGraphRunner if _is_npu else DecodeCudaGraphRunner
+ # Same runner class as the startup target verify graph on OOT.
+ TargetGraphRunnerCls = _select_spec_graph_runner_cls(
+ builtin_runner_cls=lambda: (
+ NPUGraphRunner if _is_npu else DecodeCudaGraphRunner
+ ),
+ oot_runner_cls=current_platform.get_graph_runner_cls,
)
+ if TargetGraphRunnerCls is not None:
target_graph_before_mem = get_available_gpu_memory(
self.device, self.gpu_id
)
diff --git a/test/registered/unit/spec/test_oot_eagle_hooks.py b/test/registered/unit/spec/test_oot_eagle_hooks.py
index 69c3e4176d..6086e0d66d 100644
--- a/test/registered/unit/spec/test_oot_eagle_hooks.py
+++ b/test/registered/unit/spec/test_oot_eagle_hooks.py
@@ -2,15 +2,16 @@ import contextlib
import sys
import types
import unittest
+from types import SimpleNamespace
from unittest.mock import Mock, patch, sentinel
import torch
from sglang.kernels.ops.speculative import spec_tree
from sglang.srt.arg_groups.speculative_hook import _check_oot_speculative_algorithm
-from sglang.srt.platforms.interface import SRTPlatform
+from sglang.srt.platforms.interface import PlatformCapabilities, SRTPlatform
from sglang.srt.server_args import ServerArgs
-from sglang.srt.speculative import eagle_utils
+from sglang.srt.speculative import eagle_utils, eagle_worker_v2
from sglang.srt.speculative.adaptive_spec_params import adaptive_unsupported_reason
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci
@@ -28,9 +29,14 @@ class _PlatformWorker:
def _platform(
- *, out_of_tree: bool = True, supported: bool = True, worker_cls=None
+ *,
+ out_of_tree: bool = True,
+ supported: bool = True,
+ worker_cls=None,
+ graph: bool = True,
) -> Mock:
platform = Mock(spec=SRTPlatform)
+ platform.capabilities = PlatformCapabilities(graph_capture=graph)
platform.device_name = "custom"
platform.is_out_of_tree.return_value = out_of_tree
platform.supports_speculative_algorithm.return_value = supported
@@ -142,5 +148,81 @@ class TestOOTEagleKernelDispatch(CustomTestCase):
)
+class TestOOTSpecGraphRunnerSelection(CustomTestCase):
+ def _select(self, platform: Mock, oot_runner_cls):
+ def builtin():
+ raise KeyError("custom") # an OOT device is not a key of the maps
+
+ with patch.object(eagle_worker_v2, "current_platform", platform):
+ return eagle_worker_v2._select_spec_graph_runner_cls(
+ builtin_runner_cls=builtin, oot_runner_cls=oot_runner_cls
+ )
+
+ def test_no_capture_without_graph_capture(self):
+ oot = Mock(return_value=sentinel.runner)
+ self.assertIsNone(self._select(_platform(graph=False), oot))
+ oot.assert_not_called()
+
+ def test_builtin_platform_never_consults_the_platform(self):
+ oot = Mock(return_value=sentinel.runner)
+ with patch.object(
+ eagle_worker_v2, "current_platform", _platform(out_of_tree=False)
+ ):
+ selected = eagle_worker_v2._select_spec_graph_runner_cls(
+ builtin_runner_cls=lambda: sentinel.builtin_runner,
+ oot_runner_cls=oot,
+ )
+ self.assertIs(selected, sentinel.builtin_runner)
+ oot.assert_not_called()
+
+ def test_out_of_tree_draft_graphs_come_from_the_platform(self):
+ """An OOT device name used to index the device-keyed runner maps (KeyError)."""
+ platform = _platform()
+ runners = {
+ phase: Mock(name=phase) for phase in ("draft_decode", "draft_extend")
+ }
+ platform.get_speculative_graph_runner_cls.side_effect = (
+ lambda algorithm, phase: runners[phase]
+ )
+ worker = SimpleNamespace(
+ speculative_algorithm=SpeculativeAlgorithm.EAGLE,
+ speculative_num_steps=3,
+ speculative_num_draft_tokens=4,
+ topk=1,
+ target_worker=SimpleNamespace(device="custom"),
+ draft_runner=None,
+ device="custom",
+ gpu_id=0,
+ draft_attn_backend=object(),
+ draft_extend_attn_backend=object(),
+ _specialized_graph_memory_usage={},
+ _specialized_graph_time_usage={},
+ )
+ decode_cfg = SimpleNamespace(backend="custom")
+ exec_cfg = SimpleNamespace(
+ graph=SimpleNamespace(cuda_graph_config=SimpleNamespace(decode=decode_cfg))
+ )
+ with contextlib.ExitStack() as stack:
+ for flag in DEVICE_FLAGS:
+ stack.enter_context(patch.object(eagle_worker_v2, flag, False))
+ for name, value in (
+ ("current_platform", platform),
+ ("check_cuda_graph_backend", lambda *args: False),
+ ("get_model", lambda: SimpleNamespace(model_impl="auto")),
+ ("get_exec", lambda: exec_cfg),
+ ("get_batch_sizes_to_capture", lambda runner: ([1], None)),
+ ("get_available_gpu_memory", lambda *args: 0.0),
+ ):
+ stack.enter_context(patch.object(eagle_worker_v2, name, value))
+ eagle_worker_v2.EagleDraftWorker._capture_cuda_graphs(worker)
+ for phase, runner in runners.items():
+ runner.assert_called_once_with(worker)
+ self.assertIs(worker.cuda_graph_runner, runners["draft_decode"].return_value)
+ self.assertIs(
+ worker.cuda_graph_runner_for_draft_extend,
+ runners["draft_extend"].return_value,
+ )
+
+
if __name__ == "__main__":
unittest.main()