[Feat] spec: Let out-of-tree platforms provide EAGLE draft graph runners (#45145)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
af3f9e4c8f
commit
a7183a8246
@@ -608,7 +608,12 @@ field defaults to the conservative answer.
|
|||||||
<tr>
|
<tr>
|
||||||
<td><code>get_graph_runner_cls()</code></td>
|
<td><code>get_graph_runner_cls()</code></td>
|
||||||
<td><code>return None</code></td>
|
<td><code>return None</code></td>
|
||||||
<td>Graph Runner class</td>
|
<td>Graph Runner class. Adaptive speculative decoding also builds one target verify runner per step count from it, passing <code>attn_backend</code>, <code>speculative_num_steps</code> and <code>speculative_num_draft_tokens</code> as keyword arguments.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>get_speculative_graph_runner_cls(algorithm, phase)</code></td>
|
||||||
|
<td><code>return None</code></td>
|
||||||
|
<td>EAGLE draft graph runner for <code>"draft_decode"</code> or <code>"draft_extend"</code> on the built-in worker, constructed like the built-in runner for that phase. <code>None</code> runs the phase eagerly; not consulted when <code>capabilities.graph_capture</code> is False.</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>get_speculative_worker_cls(algorithm)</code></td>
|
<td><code>get_speculative_worker_cls(algorithm)</code></td>
|
||||||
|
|||||||
@@ -116,7 +116,10 @@ class SRTPlatform(DeviceMixin):
|
|||||||
"""Return the graph runner class, or None for the in-tree default.
|
"""Return the graph runner class, or None for the in-tree default.
|
||||||
|
|
||||||
``None`` means "no platform opinion": the caller falls back to the
|
``None`` means "no platform opinion": the caller falls back to the
|
||||||
in-tree device-keyed selection.
|
in-tree device-keyed selection. Adaptive speculative decoding also
|
||||||
|
builds one target verify runner per step count from this class, passing
|
||||||
|
``attn_backend``, ``speculative_num_steps`` and
|
||||||
|
``speculative_num_draft_tokens`` as keyword arguments.
|
||||||
"""
|
"""
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -196,6 +199,17 @@ class SRTPlatform(DeviceMixin):
|
|||||||
"""
|
"""
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def get_speculative_graph_runner_cls(
|
||||||
|
self, algorithm: str, phase: str
|
||||||
|
) -> Optional[type]:
|
||||||
|
"""Return the EAGLE draft graph runner for ``phase``; None runs it eagerly.
|
||||||
|
|
||||||
|
``phase`` is ``"draft_decode"`` or ``"draft_extend"``; the class is
|
||||||
|
constructed like the built-in runner for that phase. Not consulted when
|
||||||
|
``capabilities.graph_capture`` is False.
|
||||||
|
"""
|
||||||
|
return None
|
||||||
|
|
||||||
def get_quantization_config(
|
def get_quantization_config(
|
||||||
self, quantization: str
|
self, quantization: str
|
||||||
) -> Optional[Type[QuantizationConfig]]:
|
) -> Optional[Type[QuantizationConfig]]:
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import contextlib
|
import contextlib
|
||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from typing import List, Optional
|
from typing import Callable, List, Optional
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
@@ -58,6 +58,7 @@ from sglang.srt.model_executor.runner import (
|
|||||||
get_batch_sizes_to_capture,
|
get_batch_sizes_to_capture,
|
||||||
)
|
)
|
||||||
from sglang.srt.model_loader.weight_utils import get_pp_stage_load_group
|
from sglang.srt.model_loader.weight_utils import get_pp_stage_load_group
|
||||||
|
from sglang.srt.platforms import current_platform
|
||||||
from sglang.srt.runtime_context import (
|
from sglang.srt.runtime_context import (
|
||||||
get_context,
|
get_context,
|
||||||
get_device,
|
get_device,
|
||||||
@@ -143,6 +144,19 @@ _is_xpu = is_xpu()
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _select_spec_graph_runner_cls(
|
||||||
|
*,
|
||||||
|
builtin_runner_cls: Callable[[], Optional[type]],
|
||||||
|
oot_runner_cls: Callable[[], Optional[type]],
|
||||||
|
) -> Optional[type]:
|
||||||
|
"""Each callable runs only on its own kind of platform; None skips capture."""
|
||||||
|
if not current_platform.is_out_of_tree():
|
||||||
|
return builtin_runner_cls()
|
||||||
|
if not current_platform.capabilities.graph_capture:
|
||||||
|
return None
|
||||||
|
return oot_runner_cls()
|
||||||
|
|
||||||
|
|
||||||
def _qsa_index_share_requested(hf_config) -> bool:
|
def _qsa_index_share_requested(hf_config) -> bool:
|
||||||
"""--json-model-override-args writes top-level hf_config attributes, while
|
"""--json-model-override-args writes top-level hf_config attributes, while
|
||||||
checkpoint configs carry the flag on the nested text_config; read both."""
|
checkpoint configs carry the flag on the nested text_config; read both."""
|
||||||
@@ -472,7 +486,21 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
# Capture draft
|
# Capture draft
|
||||||
decode_backend = get_exec().graph.cuda_graph_config.decode.backend
|
decode_backend = get_exec().graph.cuda_graph_config.decode.backend
|
||||||
capture_bs, _ = get_batch_sizes_to_capture(self.draft_runner)
|
capture_bs, _ = get_batch_sizes_to_capture(self.draft_runner)
|
||||||
if self.speculative_num_steps > 1:
|
draft_runner_cls = (
|
||||||
|
_select_spec_graph_runner_cls(
|
||||||
|
builtin_runner_cls=lambda: Device2DraftCudaGraphRunner[
|
||||||
|
self.target_worker.device
|
||||||
|
],
|
||||||
|
oot_runner_cls=lambda: (
|
||||||
|
current_platform.get_speculative_graph_runner_cls(
|
||||||
|
algorithm=self.speculative_algorithm.name, phase="draft_decode"
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if self.speculative_num_steps > 1
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
if draft_runner_cls is not None:
|
||||||
tic = time.perf_counter()
|
tic = time.perf_counter()
|
||||||
before_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
before_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
||||||
log_info_on_rank0(
|
log_info_on_rank0(
|
||||||
@@ -481,9 +509,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
f"num_tokens_per_req={self.topk}, bs={capture_bs}, "
|
f"num_tokens_per_req={self.topk}, bs={capture_bs}, "
|
||||||
f"avail mem={before_mem:.2f} GB",
|
f"avail mem={before_mem:.2f} GB",
|
||||||
)
|
)
|
||||||
self.cuda_graph_runner = Device2DraftCudaGraphRunner[
|
self.cuda_graph_runner = draft_runner_cls(self)
|
||||||
self.target_worker.device
|
|
||||||
](self)
|
|
||||||
after_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
after_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
||||||
capture_time = time.perf_counter() - tic
|
capture_time = time.perf_counter() - tic
|
||||||
self._specialized_graph_memory_usage["draft_decode"] = (
|
self._specialized_graph_memory_usage["draft_decode"] = (
|
||||||
@@ -572,17 +598,27 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
supports_cuda_draft_extend_graph = (
|
supports_cuda_draft_extend_graph = (
|
||||||
_is_cuda or _is_musa
|
_is_cuda or _is_musa
|
||||||
) and graph_supported_backend
|
) and graph_supported_backend
|
||||||
|
extend_runner_cls = _select_spec_graph_runner_cls(
|
||||||
|
builtin_runner_cls=lambda: (
|
||||||
|
Device2ExtendCudaGraphRunner[self.target_worker.device]
|
||||||
|
if (
|
||||||
|
_is_npu
|
||||||
|
or _is_xpu
|
||||||
|
or supports_cuda_draft_extend_graph
|
||||||
|
or supports_hip_draft_extend_graph
|
||||||
|
)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
oot_runner_cls=lambda: current_platform.get_speculative_graph_runner_cls(
|
||||||
|
algorithm=self.speculative_algorithm.name, phase="draft_extend"
|
||||||
|
),
|
||||||
|
)
|
||||||
# Capture extend
|
# Capture extend
|
||||||
# TODO: support draft extend cuda graph for more attention backends
|
# TODO: support draft extend cuda graph for more attention backends
|
||||||
if (
|
if (
|
||||||
self.draft_extend_attn_backend
|
self.draft_extend_attn_backend
|
||||||
and not envs.SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH.get()
|
and not envs.SGLANG_DISABLE_DRAFT_EXTEND_CUDA_GRAPH.get()
|
||||||
and (
|
and extend_runner_cls is not None
|
||||||
_is_npu
|
|
||||||
or _is_xpu
|
|
||||||
or supports_cuda_draft_extend_graph
|
|
||||||
or supports_hip_draft_extend_graph
|
|
||||||
)
|
|
||||||
):
|
):
|
||||||
tic = time.perf_counter()
|
tic = time.perf_counter()
|
||||||
before_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
before_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
||||||
@@ -592,9 +628,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
|||||||
f"num_tokens_per_req={self.speculative_num_draft_tokens}, "
|
f"num_tokens_per_req={self.speculative_num_draft_tokens}, "
|
||||||
f"bs={capture_bs}, avail mem={before_mem:.2f} GB",
|
f"bs={capture_bs}, avail mem={before_mem:.2f} GB",
|
||||||
)
|
)
|
||||||
self.cuda_graph_runner_for_draft_extend = Device2ExtendCudaGraphRunner[
|
self.cuda_graph_runner_for_draft_extend = extend_runner_cls(self)
|
||||||
self.target_worker.device
|
|
||||||
](self)
|
|
||||||
# draft_extend is the step's last shared-buffer-reading phase; its
|
# draft_extend is the step's last shared-buffer-reading phase; its
|
||||||
# read-done event is what the scheduler's WAR barrier waits on.
|
# read-done event is what the scheduler's WAR barrier waits on.
|
||||||
after_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
after_mem = get_available_gpu_memory(self.device, self.gpu_id)
|
||||||
@@ -1754,10 +1788,16 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
|||||||
target_model_runner.init_new_workspace = backup_init
|
target_model_runner.init_new_workspace = backup_init
|
||||||
|
|
||||||
target_graph_runner = None
|
target_graph_runner = None
|
||||||
|
TargetGraphRunnerCls = None
|
||||||
if not check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED):
|
if not check_cuda_graph_backend(Phase.DECODE, Backend.DISABLED):
|
||||||
TargetGraphRunnerCls = (
|
# Same runner class as the startup target verify graph on OOT.
|
||||||
NPUGraphRunner if _is_npu else DecodeCudaGraphRunner
|
TargetGraphRunnerCls = _select_spec_graph_runner_cls(
|
||||||
|
builtin_runner_cls=lambda: (
|
||||||
|
NPUGraphRunner if _is_npu else DecodeCudaGraphRunner
|
||||||
|
),
|
||||||
|
oot_runner_cls=current_platform.get_graph_runner_cls,
|
||||||
)
|
)
|
||||||
|
if TargetGraphRunnerCls is not None:
|
||||||
target_graph_before_mem = get_available_gpu_memory(
|
target_graph_before_mem = get_available_gpu_memory(
|
||||||
self.device, self.gpu_id
|
self.device, self.gpu_id
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -2,15 +2,16 @@ import contextlib
|
|||||||
import sys
|
import sys
|
||||||
import types
|
import types
|
||||||
import unittest
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
from unittest.mock import Mock, patch, sentinel
|
from unittest.mock import Mock, patch, sentinel
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from sglang.kernels.ops.speculative import spec_tree
|
from sglang.kernels.ops.speculative import spec_tree
|
||||||
from sglang.srt.arg_groups.speculative_hook import _check_oot_speculative_algorithm
|
from sglang.srt.arg_groups.speculative_hook import _check_oot_speculative_algorithm
|
||||||
from sglang.srt.platforms.interface import SRTPlatform
|
from sglang.srt.platforms.interface import PlatformCapabilities, SRTPlatform
|
||||||
from sglang.srt.server_args import ServerArgs
|
from sglang.srt.server_args import ServerArgs
|
||||||
from sglang.srt.speculative import eagle_utils
|
from sglang.srt.speculative import eagle_utils, eagle_worker_v2
|
||||||
from sglang.srt.speculative.adaptive_spec_params import adaptive_unsupported_reason
|
from sglang.srt.speculative.adaptive_spec_params import adaptive_unsupported_reason
|
||||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||||
from sglang.test.ci.ci_register import register_cpu_ci
|
from sglang.test.ci.ci_register import register_cpu_ci
|
||||||
@@ -28,9 +29,14 @@ class _PlatformWorker:
|
|||||||
|
|
||||||
|
|
||||||
def _platform(
|
def _platform(
|
||||||
*, out_of_tree: bool = True, supported: bool = True, worker_cls=None
|
*,
|
||||||
|
out_of_tree: bool = True,
|
||||||
|
supported: bool = True,
|
||||||
|
worker_cls=None,
|
||||||
|
graph: bool = True,
|
||||||
) -> Mock:
|
) -> Mock:
|
||||||
platform = Mock(spec=SRTPlatform)
|
platform = Mock(spec=SRTPlatform)
|
||||||
|
platform.capabilities = PlatformCapabilities(graph_capture=graph)
|
||||||
platform.device_name = "custom"
|
platform.device_name = "custom"
|
||||||
platform.is_out_of_tree.return_value = out_of_tree
|
platform.is_out_of_tree.return_value = out_of_tree
|
||||||
platform.supports_speculative_algorithm.return_value = supported
|
platform.supports_speculative_algorithm.return_value = supported
|
||||||
@@ -142,5 +148,81 @@ class TestOOTEagleKernelDispatch(CustomTestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestOOTSpecGraphRunnerSelection(CustomTestCase):
|
||||||
|
def _select(self, platform: Mock, oot_runner_cls):
|
||||||
|
def builtin():
|
||||||
|
raise KeyError("custom") # an OOT device is not a key of the maps
|
||||||
|
|
||||||
|
with patch.object(eagle_worker_v2, "current_platform", platform):
|
||||||
|
return eagle_worker_v2._select_spec_graph_runner_cls(
|
||||||
|
builtin_runner_cls=builtin, oot_runner_cls=oot_runner_cls
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_no_capture_without_graph_capture(self):
|
||||||
|
oot = Mock(return_value=sentinel.runner)
|
||||||
|
self.assertIsNone(self._select(_platform(graph=False), oot))
|
||||||
|
oot.assert_not_called()
|
||||||
|
|
||||||
|
def test_builtin_platform_never_consults_the_platform(self):
|
||||||
|
oot = Mock(return_value=sentinel.runner)
|
||||||
|
with patch.object(
|
||||||
|
eagle_worker_v2, "current_platform", _platform(out_of_tree=False)
|
||||||
|
):
|
||||||
|
selected = eagle_worker_v2._select_spec_graph_runner_cls(
|
||||||
|
builtin_runner_cls=lambda: sentinel.builtin_runner,
|
||||||
|
oot_runner_cls=oot,
|
||||||
|
)
|
||||||
|
self.assertIs(selected, sentinel.builtin_runner)
|
||||||
|
oot.assert_not_called()
|
||||||
|
|
||||||
|
def test_out_of_tree_draft_graphs_come_from_the_platform(self):
|
||||||
|
"""An OOT device name used to index the device-keyed runner maps (KeyError)."""
|
||||||
|
platform = _platform()
|
||||||
|
runners = {
|
||||||
|
phase: Mock(name=phase) for phase in ("draft_decode", "draft_extend")
|
||||||
|
}
|
||||||
|
platform.get_speculative_graph_runner_cls.side_effect = (
|
||||||
|
lambda algorithm, phase: runners[phase]
|
||||||
|
)
|
||||||
|
worker = SimpleNamespace(
|
||||||
|
speculative_algorithm=SpeculativeAlgorithm.EAGLE,
|
||||||
|
speculative_num_steps=3,
|
||||||
|
speculative_num_draft_tokens=4,
|
||||||
|
topk=1,
|
||||||
|
target_worker=SimpleNamespace(device="custom"),
|
||||||
|
draft_runner=None,
|
||||||
|
device="custom",
|
||||||
|
gpu_id=0,
|
||||||
|
draft_attn_backend=object(),
|
||||||
|
draft_extend_attn_backend=object(),
|
||||||
|
_specialized_graph_memory_usage={},
|
||||||
|
_specialized_graph_time_usage={},
|
||||||
|
)
|
||||||
|
decode_cfg = SimpleNamespace(backend="custom")
|
||||||
|
exec_cfg = SimpleNamespace(
|
||||||
|
graph=SimpleNamespace(cuda_graph_config=SimpleNamespace(decode=decode_cfg))
|
||||||
|
)
|
||||||
|
with contextlib.ExitStack() as stack:
|
||||||
|
for flag in DEVICE_FLAGS:
|
||||||
|
stack.enter_context(patch.object(eagle_worker_v2, flag, False))
|
||||||
|
for name, value in (
|
||||||
|
("current_platform", platform),
|
||||||
|
("check_cuda_graph_backend", lambda *args: False),
|
||||||
|
("get_model", lambda: SimpleNamespace(model_impl="auto")),
|
||||||
|
("get_exec", lambda: exec_cfg),
|
||||||
|
("get_batch_sizes_to_capture", lambda runner: ([1], None)),
|
||||||
|
("get_available_gpu_memory", lambda *args: 0.0),
|
||||||
|
):
|
||||||
|
stack.enter_context(patch.object(eagle_worker_v2, name, value))
|
||||||
|
eagle_worker_v2.EagleDraftWorker._capture_cuda_graphs(worker)
|
||||||
|
for phase, runner in runners.items():
|
||||||
|
runner.assert_called_once_with(worker)
|
||||||
|
self.assertIs(worker.cuda_graph_runner, runners["draft_decode"].return_value)
|
||||||
|
self.assertIs(
|
||||||
|
worker.cuda_graph_runner_for_draft_extend,
|
||||||
|
runners["draft_extend"].return_value,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user