[Feat] spec: Let out-of-tree platforms provide EAGLE draft graph runners (#45145)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yihao Wang
2026-10-23 21:28:56 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent af3f9e4c8f
commit a7183a8246
4 changed files with 162 additions and 21 deletions
+6 -1
View File
@@ -608,7 +608,12 @@ field defaults to the conservative answer.
<tr>
<td><code>get_graph_runner_cls()</code></td>
<td><code>return None</code></td>
<td>Graph Runner class</td>
<td>Graph Runner class. Adaptive speculative decoding also builds one target verify runner per step count from it, passing <code>attn_backend</code>, <code>speculative_num_steps</code> and <code>speculative_num_draft_tokens</code> as keyword arguments.</td>
</tr>
<tr>
<td><code>get_speculative_graph_runner_cls(algorithm, phase)</code></td>
<td><code>return None</code></td>
<td>EAGLE draft graph runner for <code>"draft_decode"</code> or <code>"draft_extend"</code> on the built-in worker, constructed like the built-in runner for that phase. <code>None</code> runs the phase eagerly; not consulted when <code>capabilities.graph_capture</code> is False.</td>
</tr>
<tr>
<td><code>get_speculative_worker_cls(algorithm)</code></td>
+15 -1
View File
@@ -116,7 +116,10 @@ class SRTPlatform(DeviceMixin):
"""Return the graph runner class, or None for the in-tree default.
``None`` means "no platform opinion": the caller falls back to the
in-tree device-keyed selection.
in-tree device-keyed selection. Adaptive speculative decoding also
builds one target verify runner per step count from this class, passing
``attn_backend``, ``speculative_num_steps`` and
``speculative_num_draft_tokens`` as keyword arguments.
"""
return None
@@ -196,6 +199,17 @@ class SRTPlatform(DeviceMixin):
"""
return None
def get_speculative_graph_runner_cls(
self, algorithm: str, phase: str
) -> Optional[type]:
"""Return the EAGLE draft graph runner for ``phase``; None runs it eagerly.
``phase`` is ``"draft_decode"`` or ``"draft_extend"``; the class is
constructed like the built-in runner for that phase. Not consulted when
``capabilities.graph_capture`` is False.
"""
return None
def get_quantization_config(
self, quantization: str
) -> Optional[Type[QuantizationConfig]]:
@@ -1,7 +1,7 @@
import contextlib
import logging
import time
from typing import List, Optional
from typing import Callable, List, Optional
import torch
@@ -58,6 +58,7 @@ from sglang.srt.model_executor.runner import (
get_batch_sizes_to_capture,
)
from sglang.srt.model_loader.weight_utils import get_pp_stage_load_group
from sglang.srt.platforms import current_platform
from sglang.srt.runtime_context import (
get_context,
get_device,
@@ -143,6 +144,19 @@ _is_xpu = is_xpu()
logger = logging.getLogger(__name__)
def _select_spec_graph_runner_cls(
*,
builtin_runner_cls: Callable[[], Optional[type]],
oot_runner_cls: Callable[[], Optional[type]],
) -> Optional[type]:
"""Each callable runs only on its own kind of platform; None skips capture."""
if not current_platform.is_out_of_tree():
return builtin_runner_cls()
if not current_platform.capabilities.graph_capture:
return None
return oot_runner_cls()
def _qsa_index_share_requested(hf_config) -> bool:
"""--json-model-override-args writes top-level hf_config attributes, while
checkpoint configs carry the flag on the nested text_config; read both."""
@@ -472,7 +486,21 @@ class EagleDraftWorker(EagleDraftWorkerBase):
# Capture draft
decode_backend = get_exec().graph.cuda_graph_config.decode.backend
capture_bs, _ = get_batch_sizes_to_capture(self.draft_runner)
if self.speculative_num_steps > 1:
draft_runner_cls = (
_select_spec_graph_runner_cls(
builtin_runner_cls=lambda: Device2DraftCudaGraphRunner[
self.target_worker.device
],
oot_runner_cls=lambda: (
current_platform.get_speculative_graph_runner_cls(
algorithm=self.speculative_algorithm.name, phase="draft_decode"
)
),
)
if self.speculative_num_steps > 1
else None
)
if draft_runner_cls is not None:
tic = time.perf_counter()
before_mem = get_available_gpu_memory(self.device, self.gpu_id)
log_info_on_rank0(
@@ -481,9 +509,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
f"num_tokens_per_req={self.topk}, bs={capture_bs}, "
f"avail mem={before_mem:.2f} GB",
)
self.cuda_graph_runner = Device2DraftCudaGraphRunner[
self.target_worker.device
](self)
self.cuda_graph_runner = draft_runner_cls(self)
after_mem = get_available_gpu_memory(self.device, self.gpu_id)
capture_time = time.perf_counter() - tic
self._specialized_graph_memory_usage["draft_decode"] = (
@@ -572,17 +598,27 @@ class EagleDraftWorker(EagleDraftWorkerBase):
supports_cuda_draft_extend_graph = (
_is_cuda or _is_musa
) and graph_supported_backend
# Capture extend
# TODO: support draft extend cuda graph for more attention backends
extend_runner_cls = _select_spec_graph_runner_cls(
builtin_runner_cls=lambda: (
Device2ExtendCudaGraphRunner[self.target_worker.device]
if (
self.draft_extend_attn_backend
and not envs.SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH.get()
and (
_is_npu
or _is_xpu
or supports_cuda_draft_extend_graph
or supports_hip_draft_extend_graph
)
else None
),
oot_runner_cls=lambda: current_platform.get_speculative_graph_runner_cls(
algorithm=self.speculative_algorithm.name, phase="draft_extend"
),
)
# Capture extend
# TODO: support draft extend cuda graph for more attention backends
if (
self.draft_extend_attn_backend
and not envs.SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH.get()
and extend_runner_cls is not None
):
tic = time.perf_counter()
before_mem = get_available_gpu_memory(self.device, self.gpu_id)
@@ -592,9 +628,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
f"num_tokens_per_req={self.speculative_num_draft_tokens}, "
f"bs={capture_bs}, avail mem={before_mem:.2f} GB",
)
self.cuda_graph_runner_for_draft_extend = Device2ExtendCudaGraphRunner[
self.target_worker.device
](self)
self.cuda_graph_runner_for_draft_extend = extend_runner_cls(self)
# draft_extend is the step's last shared-buffer-reading phase; its
# read-done event is what the scheduler's WAR barrier waits on.
after_mem = get_available_gpu_memory(self.device, self.gpu_id)
@@ -1754,10 +1788,16 @@ class EAGLEWorkerV2(BaseSpecWorker):
target_model_runner.init_new_workspace = backup_init
target_graph_runner = None
TargetGraphRunnerCls = None
if not check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED):
TargetGraphRunnerCls = (
# Same runner class as the startup target verify graph on OOT.
TargetGraphRunnerCls = _select_spec_graph_runner_cls(
builtin_runner_cls=lambda: (
NPUGraphRunner if _is_npu else DecodeCudaGraphRunner
),
oot_runner_cls=current_platform.get_graph_runner_cls,
)
if TargetGraphRunnerCls is not None:
target_graph_before_mem = get_available_gpu_memory(
self.device, self.gpu_id
)
@@ -2,15 +2,16 @@ import contextlib
import sys
import types
import unittest
from types import SimpleNamespace
from unittest.mock import Mock, patch, sentinel
import torch
from sglang.kernels.ops.speculative import spec_tree
from sglang.srt.arg_groups.speculative_hook import _check_oot_speculative_algorithm
from sglang.srt.platforms.interface import SRTPlatform
from sglang.srt.platforms.interface import PlatformCapabilities, SRTPlatform
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative import eagle_utils
from sglang.srt.speculative import eagle_utils, eagle_worker_v2
from sglang.srt.speculative.adaptive_spec_params import adaptive_unsupported_reason
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci
@@ -28,9 +29,14 @@ class _PlatformWorker:
def _platform(
*, out_of_tree: bool = True, supported: bool = True, worker_cls=None
*,
out_of_tree: bool = True,
supported: bool = True,
worker_cls=None,
graph: bool = True,
) -> Mock:
platform = Mock(spec=SRTPlatform)
platform.capabilities = PlatformCapabilities(graph_capture=graph)
platform.device_name = "custom"
platform.is_out_of_tree.return_value = out_of_tree
platform.supports_speculative_algorithm.return_value = supported
@@ -142,5 +148,81 @@ class TestOOTEagleKernelDispatch(CustomTestCase):
)
class TestOOTSpecGraphRunnerSelection(CustomTestCase):
def _select(self, platform: Mock, oot_runner_cls):
def builtin():
raise KeyError("custom") # an OOT device is not a key of the maps
with patch.object(eagle_worker_v2, "current_platform", platform):
return eagle_worker_v2._select_spec_graph_runner_cls(
builtin_runner_cls=builtin, oot_runner_cls=oot_runner_cls
)
def test_no_capture_without_graph_capture(self):
oot = Mock(return_value=sentinel.runner)
self.assertIsNone(self._select(_platform(graph=False), oot))
oot.assert_not_called()
def test_builtin_platform_never_consults_the_platform(self):
oot = Mock(return_value=sentinel.runner)
with patch.object(
eagle_worker_v2, "current_platform", _platform(out_of_tree=False)
):
selected = eagle_worker_v2._select_spec_graph_runner_cls(
builtin_runner_cls=lambda: sentinel.builtin_runner,
oot_runner_cls=oot,
)
self.assertIs(selected, sentinel.builtin_runner)
oot.assert_not_called()
def test_out_of_tree_draft_graphs_come_from_the_platform(self):
"""An OOT device name used to index the device-keyed runner maps (KeyError)."""
platform = _platform()
runners = {
phase: Mock(name=phase) for phase in ("draft_decode", "draft_extend")
}
platform.get_speculative_graph_runner_cls.side_effect = (
lambda algorithm, phase: runners[phase]
)
worker = SimpleNamespace(
speculative_algorithm=SpeculativeAlgorithm.EAGLE,
speculative_num_steps=3,
speculative_num_draft_tokens=4,
topk=1,
target_worker=SimpleNamespace(device="custom"),
draft_runner=None,
device="custom",
gpu_id=0,
draft_attn_backend=object(),
draft_extend_attn_backend=object(),
_specialized_graph_memory_usage={},
_specialized_graph_time_usage={},
)
decode_cfg = SimpleNamespace(backend="custom")
exec_cfg = SimpleNamespace(
graph=SimpleNamespace(cuda_graph_config=SimpleNamespace(decode=decode_cfg))
)
with contextlib.ExitStack() as stack:
for flag in DEVICE_FLAGS:
stack.enter_context(patch.object(eagle_worker_v2, flag, False))
for name, value in (
("current_platform", platform),
("check_cuda_graph_backend", lambda *args: False),
("get_model", lambda: SimpleNamespace(model_impl="auto")),
("get_exec", lambda: exec_cfg),
("get_batch_sizes_to_capture", lambda runner: ([1], None)),
("get_available_gpu_memory", lambda *args: 0.0),
):
stack.enter_context(patch.object(eagle_worker_v2, name, value))
eagle_worker_v2.EagleDraftWorker._capture_cuda_graphs(worker)
for phase, runner in runners.items():
runner.assert_called_once_with(worker)
self.assertIs(worker.cuda_graph_runner, runners["draft_decode"].return_value)
self.assertIs(
worker.cuda_graph_runner_for_draft_extend,
runners["draft_extend"].return_value,
)
if __name__ == "__main__":
unittest.main()