[Feat] spec: Let out-of-tree platforms provide EAGLE draft graph runners (#45145)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yihao Wang
2026-10-23 21:28:56 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent af3f9e4c8f
commit a7183a8246
4 changed files with 162 additions and 21 deletions
+6 -1
View File
@@ -608,7 +608,12 @@ field defaults to the conservative answer.
<tr> <tr>
<td><code>get_graph_runner_cls()</code></td> <td><code>get_graph_runner_cls()</code></td>
<td><code>return None</code></td> <td><code>return None</code></td>
<td>Graph Runner class</td> <td>Graph Runner class. Adaptive speculative decoding also builds one target verify runner per step count from it, passing <code>attn_backend</code>, <code>speculative_num_steps</code> and <code>speculative_num_draft_tokens</code> as keyword arguments.</td>
</tr>
<tr>
<td><code>get_speculative_graph_runner_cls(algorithm, phase)</code></td>
<td><code>return None</code></td>
<td>EAGLE draft graph runner for <code>"draft_decode"</code> or <code>"draft_extend"</code> on the built-in worker, constructed like the built-in runner for that phase. <code>None</code> runs the phase eagerly; not consulted when <code>capabilities.graph_capture</code> is False.</td>
</tr> </tr>
<tr> <tr>
<td><code>get_speculative_worker_cls(algorithm)</code></td> <td><code>get_speculative_worker_cls(algorithm)</code></td>
+15 -1
View File
@@ -116,7 +116,10 @@ class SRTPlatform(DeviceMixin):
"""Return the graph runner class, or None for the in-tree default. """Return the graph runner class, or None for the in-tree default.
``None`` means "no platform opinion": the caller falls back to the ``None`` means "no platform opinion": the caller falls back to the
in-tree device-keyed selection. in-tree device-keyed selection. Adaptive speculative decoding also
builds one target verify runner per step count from this class, passing
``attn_backend``, ``speculative_num_steps`` and
``speculative_num_draft_tokens`` as keyword arguments.
""" """
return None return None
@@ -196,6 +199,17 @@ class SRTPlatform(DeviceMixin):
""" """
return None return None
def get_speculative_graph_runner_cls(
self, algorithm: str, phase: str
) -> Optional[type]:
"""Return the EAGLE draft graph runner for ``phase``; None runs it eagerly.
``phase`` is ``"draft_decode"`` or ``"draft_extend"``; the class is
constructed like the built-in runner for that phase. Not consulted when
``capabilities.graph_capture`` is False.
"""
return None
def get_quantization_config( def get_quantization_config(
self, quantization: str self, quantization: str
) -> Optional[Type[QuantizationConfig]]: ) -> Optional[Type[QuantizationConfig]]:
@@ -1,7 +1,7 @@
import contextlib import contextlib
import logging import logging
import time import time
from typing import List, Optional from typing import Callable, List, Optional
import torch import torch
@@ -58,6 +58,7 @@ from sglang.srt.model_executor.runner import (
get_batch_sizes_to_capture, get_batch_sizes_to_capture,
) )
from sglang.srt.model_loader.weight_utils import get_pp_stage_load_group from sglang.srt.model_loader.weight_utils import get_pp_stage_load_group
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import ( from sglang.srt.runtime_context import (
get_context, get_context,
get_device, get_device,
@@ -143,6 +144,19 @@ _is_xpu = is_xpu()
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def _select_spec_graph_runner_cls(
*,
builtin_runner_cls: Callable[[], Optional[type]],
oot_runner_cls: Callable[[], Optional[type]],
) -> Optional[type]:
"""Each callable runs only on its own kind of platform; None skips capture."""
if not current_platform.is_out_of_tree():
return builtin_runner_cls()
if not current_platform.capabilities.graph_capture:
return None
return oot_runner_cls()
def _qsa_index_share_requested(hf_config) -> bool: def _qsa_index_share_requested(hf_config) -> bool:
"""--json-model-override-args writes top-level hf_config attributes, while """--json-model-override-args writes top-level hf_config attributes, while
checkpoint configs carry the flag on the nested text_config; read both.""" checkpoint configs carry the flag on the nested text_config; read both."""
@@ -472,7 +486,21 @@ class EagleDraftWorker(EagleDraftWorkerBase):
# Capture draft # Capture draft
decode_backend = get_exec().graph.cuda_graph_config.decode.backend decode_backend = get_exec().graph.cuda_graph_config.decode.backend
capture_bs, _ = get_batch_sizes_to_capture(self.draft_runner) capture_bs, _ = get_batch_sizes_to_capture(self.draft_runner)
if self.speculative_num_steps > 1: draft_runner_cls = (
_select_spec_graph_runner_cls(
builtin_runner_cls=lambda: Device2DraftCudaGraphRunner[
self.target_worker.device
],
oot_runner_cls=lambda: (
current_platform.get_speculative_graph_runner_cls(
algorithm=self.speculative_algorithm.name, phase="draft_decode"
)
),
)
if self.speculative_num_steps > 1
else None
)
if draft_runner_cls is not None:
tic = time.perf_counter() tic = time.perf_counter()
before_mem = get_available_gpu_memory(self.device, self.gpu_id) before_mem = get_available_gpu_memory(self.device, self.gpu_id)
log_info_on_rank0( log_info_on_rank0(
@@ -481,9 +509,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
f"num_tokens_per_req={self.topk}, bs={capture_bs}, " f"num_tokens_per_req={self.topk}, bs={capture_bs}, "
f"avail mem={before_mem:.2f} GB", f"avail mem={before_mem:.2f} GB",
) )
self.cuda_graph_runner = Device2DraftCudaGraphRunner[ self.cuda_graph_runner = draft_runner_cls(self)
self.target_worker.device
](self)
after_mem = get_available_gpu_memory(self.device, self.gpu_id) after_mem = get_available_gpu_memory(self.device, self.gpu_id)
capture_time = time.perf_counter() - tic capture_time = time.perf_counter() - tic
self._specialized_graph_memory_usage["draft_decode"] = ( self._specialized_graph_memory_usage["draft_decode"] = (
@@ -572,17 +598,27 @@ class EagleDraftWorker(EagleDraftWorkerBase):
supports_cuda_draft_extend_graph = ( supports_cuda_draft_extend_graph = (
_is_cuda or _is_musa _is_cuda or _is_musa
) and graph_supported_backend ) and graph_supported_backend
extend_runner_cls = _select_spec_graph_runner_cls(
builtin_runner_cls=lambda: (
Device2ExtendCudaGraphRunner[self.target_worker.device]
if (
_is_npu
or _is_xpu
or supports_cuda_draft_extend_graph
or supports_hip_draft_extend_graph
)
else None
),
oot_runner_cls=lambda: current_platform.get_speculative_graph_runner_cls(
algorithm=self.speculative_algorithm.name, phase="draft_extend"
),
)
# Capture extend # Capture extend
# TODO: support draft extend cuda graph for more attention backends # TODO: support draft extend cuda graph for more attention backends
if ( if (
self.draft_extend_attn_backend self.draft_extend_attn_backend
and not envs.SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH.get() and not envs.SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH.get()
and ( and extend_runner_cls is not None
_is_npu
or _is_xpu
or supports_cuda_draft_extend_graph
or supports_hip_draft_extend_graph
)
): ):
tic = time.perf_counter() tic = time.perf_counter()
before_mem = get_available_gpu_memory(self.device, self.gpu_id) before_mem = get_available_gpu_memory(self.device, self.gpu_id)
@@ -592,9 +628,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
f"num_tokens_per_req={self.speculative_num_draft_tokens}, " f"num_tokens_per_req={self.speculative_num_draft_tokens}, "
f"bs={capture_bs}, avail mem={before_mem:.2f} GB", f"bs={capture_bs}, avail mem={before_mem:.2f} GB",
) )
self.cuda_graph_runner_for_draft_extend = Device2ExtendCudaGraphRunner[ self.cuda_graph_runner_for_draft_extend = extend_runner_cls(self)
self.target_worker.device
](self)
# draft_extend is the step's last shared-buffer-reading phase; its # draft_extend is the step's last shared-buffer-reading phase; its
# read-done event is what the scheduler's WAR barrier waits on. # read-done event is what the scheduler's WAR barrier waits on.
after_mem = get_available_gpu_memory(self.device, self.gpu_id) after_mem = get_available_gpu_memory(self.device, self.gpu_id)
@@ -1754,10 +1788,16 @@ class EAGLEWorkerV2(BaseSpecWorker):
target_model_runner.init_new_workspace = backup_init target_model_runner.init_new_workspace = backup_init
target_graph_runner = None target_graph_runner = None
TargetGraphRunnerCls = None
if not check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED): if not check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED):
TargetGraphRunnerCls = ( # Same runner class as the startup target verify graph on OOT.
NPUGraphRunner if _is_npu else DecodeCudaGraphRunner TargetGraphRunnerCls = _select_spec_graph_runner_cls(
builtin_runner_cls=lambda: (
NPUGraphRunner if _is_npu else DecodeCudaGraphRunner
),
oot_runner_cls=current_platform.get_graph_runner_cls,
) )
if TargetGraphRunnerCls is not None:
target_graph_before_mem = get_available_gpu_memory( target_graph_before_mem = get_available_gpu_memory(
self.device, self.gpu_id self.device, self.gpu_id
) )
@@ -2,15 +2,16 @@ import contextlib
import sys import sys
import types import types
import unittest import unittest
from types import SimpleNamespace
from unittest.mock import Mock, patch, sentinel from unittest.mock import Mock, patch, sentinel
import torch import torch
from sglang.kernels.ops.speculative import spec_tree from sglang.kernels.ops.speculative import spec_tree
from sglang.srt.arg_groups.speculative_hook import _check_oot_speculative_algorithm from sglang.srt.arg_groups.speculative_hook import _check_oot_speculative_algorithm
from sglang.srt.platforms.interface import SRTPlatform from sglang.srt.platforms.interface import PlatformCapabilities, SRTPlatform
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative import eagle_utils from sglang.srt.speculative import eagle_utils, eagle_worker_v2
from sglang.srt.speculative.adaptive_spec_params import adaptive_unsupported_reason from sglang.srt.speculative.adaptive_spec_params import adaptive_unsupported_reason
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.ci.ci_register import register_cpu_ci
@@ -28,9 +29,14 @@ class _PlatformWorker:
def _platform( def _platform(
*, out_of_tree: bool = True, supported: bool = True, worker_cls=None *,
out_of_tree: bool = True,
supported: bool = True,
worker_cls=None,
graph: bool = True,
) -> Mock: ) -> Mock:
platform = Mock(spec=SRTPlatform) platform = Mock(spec=SRTPlatform)
platform.capabilities = PlatformCapabilities(graph_capture=graph)
platform.device_name = "custom" platform.device_name = "custom"
platform.is_out_of_tree.return_value = out_of_tree platform.is_out_of_tree.return_value = out_of_tree
platform.supports_speculative_algorithm.return_value = supported platform.supports_speculative_algorithm.return_value = supported
@@ -142,5 +148,81 @@ class TestOOTEagleKernelDispatch(CustomTestCase):
) )
class TestOOTSpecGraphRunnerSelection(CustomTestCase):
def _select(self, platform: Mock, oot_runner_cls):
def builtin():
raise KeyError("custom") # an OOT device is not a key of the maps
with patch.object(eagle_worker_v2, "current_platform", platform):
return eagle_worker_v2._select_spec_graph_runner_cls(
builtin_runner_cls=builtin, oot_runner_cls=oot_runner_cls
)
def test_no_capture_without_graph_capture(self):
oot = Mock(return_value=sentinel.runner)
self.assertIsNone(self._select(_platform(graph=False), oot))
oot.assert_not_called()
def test_builtin_platform_never_consults_the_platform(self):
oot = Mock(return_value=sentinel.runner)
with patch.object(
eagle_worker_v2, "current_platform", _platform(out_of_tree=False)
):
selected = eagle_worker_v2._select_spec_graph_runner_cls(
builtin_runner_cls=lambda: sentinel.builtin_runner,
oot_runner_cls=oot,
)
self.assertIs(selected, sentinel.builtin_runner)
oot.assert_not_called()
def test_out_of_tree_draft_graphs_come_from_the_platform(self):
"""An OOT device name used to index the device-keyed runner maps (KeyError)."""
platform = _platform()
runners = {
phase: Mock(name=phase) for phase in ("draft_decode", "draft_extend")
}
platform.get_speculative_graph_runner_cls.side_effect = (
lambda algorithm, phase: runners[phase]
)
worker = SimpleNamespace(
speculative_algorithm=SpeculativeAlgorithm.EAGLE,
speculative_num_steps=3,
speculative_num_draft_tokens=4,
topk=1,
target_worker=SimpleNamespace(device="custom"),
draft_runner=None,
device="custom",
gpu_id=0,
draft_attn_backend=object(),
draft_extend_attn_backend=object(),
_specialized_graph_memory_usage={},
_specialized_graph_time_usage={},
)
decode_cfg = SimpleNamespace(backend="custom")
exec_cfg = SimpleNamespace(
graph=SimpleNamespace(cuda_graph_config=SimpleNamespace(decode=decode_cfg))
)
with contextlib.ExitStack() as stack:
for flag in DEVICE_FLAGS:
stack.enter_context(patch.object(eagle_worker_v2, flag, False))
for name, value in (
("current_platform", platform),
("check_cuda_graph_backend", lambda *args: False),
("get_model", lambda: SimpleNamespace(model_impl="auto")),
("get_exec", lambda: exec_cfg),
("get_batch_sizes_to_capture", lambda runner: ([1], None)),
("get_available_gpu_memory", lambda *args: 0.0),
):
stack.enter_context(patch.object(eagle_worker_v2, name, value))
eagle_worker_v2.EagleDraftWorker._capture_cuda_graphs(worker)
for phase, runner in runners.items():
runner.assert_called_once_with(worker)
self.assertIs(worker.cuda_graph_runner, runners["draft_decode"].return_value)
self.assertIs(
worker.cuda_graph_runner_for_draft_extend,
runners["draft_extend"].return_value,
)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()