Files
sglang/test/registered/unit/spec/test_oot_eagle_hooks.py
T

229 lines
9.2 KiB
Python

import contextlib
import sys
import types
import unittest
from types import SimpleNamespace
from unittest.mock import Mock, patch, sentinel
import torch
from sglang.kernels.ops.speculative import spec_tree
from sglang.srt.arg_groups.speculative_hook import _check_oot_speculative_algorithm
from sglang.srt.platforms.interface import PlatformCapabilities, SRTPlatform
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative import eagle_utils, eagle_worker_v2
from sglang.srt.speculative.adaptive_spec_params import adaptive_unsupported_reason
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.test.ci.ci_register import register_cpu_ci
from sglang.test.test_utils import CustomTestCase
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
HOOK_MODULE = "sglang.srt.arg_groups.speculative_hook"
WORKER_MODULE = "sglang.srt.speculative.eagle_worker_v2"
DEVICE_FLAGS = ("_is_cuda", "_is_hip", "_is_musa", "_is_cpu", "_is_npu", "_is_xpu")
class _PlatformWorker:
pass
def _platform(
*,
out_of_tree: bool = True,
supported: bool = True,
worker_cls=None,
graph: bool = True,
) -> Mock:
platform = Mock(spec=SRTPlatform)
platform.capabilities = PlatformCapabilities(graph_capture=graph)
platform.device_name = "custom"
platform.is_out_of_tree.return_value = out_of_tree
platform.supports_speculative_algorithm.return_value = supported
platform.get_speculative_worker_cls.return_value = worker_cls
return platform
class TestOOTEagleWorkerSelection(CustomTestCase):
def _create_worker(self, platform: Mock):
args = ServerArgs(model_path="dummy", device="custom")
builtin = types.ModuleType(WORKER_MODULE)
builtin.EAGLEWorkerV2 = sentinel.builtin_worker
with (
patch("sglang.srt.platforms.current_platform", platform),
patch.dict(sys.modules, {WORKER_MODULE: builtin}),
):
return SpeculativeAlgorithm.EAGLE.create_worker(args)
def test_platform_worker_replaces_builtin(self):
platform = _platform(worker_cls=sentinel.platform_worker)
self.assertIs(self._create_worker(platform), sentinel.platform_worker)
platform.get_speculative_worker_cls.assert_called_once_with(algorithm="EAGLE")
def test_builtin_worker_when_platform_declines(self):
declining = _platform(worker_cls=None)
self.assertIs(self._create_worker(declining), sentinel.builtin_worker)
in_tree = _platform(out_of_tree=False, worker_cls=sentinel.platform_worker)
self.assertIs(self._create_worker(in_tree), sentinel.builtin_worker)
in_tree.get_speculative_worker_cls.assert_not_called()
class TestOOTEagleAlgorithmGate(CustomTestCase):
def test_unsupported_out_of_tree_platform_is_rejected(self):
with patch(f"{HOOK_MODULE}.current_platform", _platform(supported=False)):
with self.assertRaisesRegex(ValueError, "EAGLE speculative decoding"):
_check_oot_speculative_algorithm(algorithm="EAGLE", device="custom")
def test_builtin_platforms_are_not_gated(self):
platform = _platform(out_of_tree=False, supported=False)
with patch(f"{HOOK_MODULE}.current_platform", platform):
_check_oot_speculative_algorithm(algorithm="EAGLE", device="cuda")
platform.supports_speculative_algorithm.assert_not_called()
class TestOOTPlatformWorkerAdaptive(CustomTestCase):
def _reason(self, platform: Mock):
args = ServerArgs(
model_path="dummy", device="custom", speculative_algorithm="EAGLE"
)
with patch("sglang.srt.platforms.current_platform", platform):
return adaptive_unsupported_reason(args)
def test_platform_worker_disables_adaptive(self):
reason = self._reason(_platform(worker_cls=_PlatformWorker))
self.assertIn("_PlatformWorker", reason)
def test_builtin_worker_keeps_adaptive(self):
self.assertIsNone(self._reason(_platform(worker_cls=None)))
self.assertIsNone(
self._reason(_platform(out_of_tree=False, worker_cls=_PlatformWorker))
)
class TestOOTEagleKernelDispatch(CustomTestCase):
"""An OOT device has no tree kernel; it must fail loudly, not compute nothing."""
def _run(self, platform: Mock, fn):
with contextlib.ExitStack() as stack:
for flag in DEVICE_FLAGS:
stack.enter_context(patch.object(spec_tree, flag, False))
stack.enter_context(patch.object(spec_tree, "current_platform", platform))
return fn()
def _build_tree(self):
return eagle_utils.build_tree_kernel_efficient(
bonus_tokens=torch.tensor([5]),
parent_list=torch.zeros((1, 2), dtype=torch.long),
top_scores_index=torch.zeros((1, 2), dtype=torch.long),
draft_tokens=torch.tensor([[6, 7]]),
seq_lens=torch.tensor([4]),
seq_lens_sum=4,
topk=1,
spec_steps=2,
num_verify_tokens=3,
tree_mask_mode=spec_tree.TreeMaskMode.QLEN_ONLY,
)
def _verify(self):
return spec_tree.verify_tree_greedy_func(
predicts=None,
accept_index=None,
accept_token_num=None,
candidates=None,
retrieve_index=None,
retrieve_next_token=None,
retrieve_next_sibling=None,
target_predict=None,
)
def test_out_of_tree_device_is_rejected(self):
for name, fn in (("build_tree", self._build_tree), ("verify", self._verify)):
with self.subTest(op=name):
with self.assertRaisesRegex(NotImplementedError, "no platform seam"):
self._run(_platform(), fn)
def test_in_tree_verify_fallthrough_is_unchanged(self):
self.assertEqual(
self._run(_platform(out_of_tree=False), self._verify), (None, None, None)
)
class TestOOTSpecGraphRunnerSelection(CustomTestCase):
def _select(self, platform: Mock, oot_runner_cls):
def builtin():
raise KeyError("custom") # an OOT device is not a key of the maps
with patch.object(eagle_worker_v2, "current_platform", platform):
return eagle_worker_v2._select_spec_graph_runner_cls(
builtin_runner_cls=builtin, oot_runner_cls=oot_runner_cls
)
def test_no_capture_without_graph_capture(self):
oot = Mock(return_value=sentinel.runner)
self.assertIsNone(self._select(_platform(graph=False), oot))
oot.assert_not_called()
def test_builtin_platform_never_consults_the_platform(self):
oot = Mock(return_value=sentinel.runner)
with patch.object(
eagle_worker_v2, "current_platform", _platform(out_of_tree=False)
):
selected = eagle_worker_v2._select_spec_graph_runner_cls(
builtin_runner_cls=lambda: sentinel.builtin_runner,
oot_runner_cls=oot,
)
self.assertIs(selected, sentinel.builtin_runner)
oot.assert_not_called()
def test_out_of_tree_draft_graphs_come_from_the_platform(self):
"""An OOT device name used to index the device-keyed runner maps (KeyError)."""
platform = _platform()
runners = {
phase: Mock(name=phase) for phase in ("draft_decode", "draft_extend")
}
platform.get_speculative_graph_runner_cls.side_effect = (
lambda algorithm, phase: runners[phase]
)
worker = SimpleNamespace(
speculative_algorithm=SpeculativeAlgorithm.EAGLE,
speculative_num_steps=3,
speculative_num_draft_tokens=4,
topk=1,
target_worker=SimpleNamespace(device="custom"),
draft_runner=None,
device="custom",
gpu_id=0,
draft_attn_backend=object(),
draft_extend_attn_backend=object(),
_specialized_graph_memory_usage={},
_specialized_graph_time_usage={},
)
decode_cfg = SimpleNamespace(backend="custom")
exec_cfg = SimpleNamespace(
graph=SimpleNamespace(cuda_graph_config=SimpleNamespace(decode=decode_cfg))
)
with contextlib.ExitStack() as stack:
for flag in DEVICE_FLAGS:
stack.enter_context(patch.object(eagle_worker_v2, flag, False))
for name, value in (
("current_platform", platform),
("check_cuda_graph_backend", lambda *args: False),
("get_model", lambda: SimpleNamespace(model_impl="auto")),
("get_exec", lambda: exec_cfg),
("get_batch_sizes_to_capture", lambda runner: ([1], None)),
("get_available_gpu_memory", lambda *args: 0.0),
):
stack.enter_context(patch.object(eagle_worker_v2, name, value))
eagle_worker_v2.EagleDraftWorker._capture_cuda_graphs(worker)
for phase, runner in runners.items():
runner.assert_called_once_with(worker)
self.assertIs(worker.cuda_graph_runner, runners["draft_decode"].return_value)
self.assertIs(
worker.cuda_graph_runner_for_draft_extend,
runners["draft_extend"].return_value,
)
if __name__ == "__main__":
unittest.main()