diff --git a/python/sglang/srt/utils/custom_op.py b/python/sglang/srt/utils/custom_op.py index aadb857e5c..ece88252e9 100644 --- a/python/sglang/srt/utils/custom_op.py +++ b/python/sglang/srt/utils/custom_op.py @@ -1,12 +1,13 @@ from __future__ import annotations +import functools import inspect from typing import Any, Callable, List, Optional, TypeVar, Union, overload import torch import torch.library -from sglang.kernels.kernel_api_logging import debug_torch_op +from sglang.kernels.kernel_api_logging import _KERNEL_API_LOG_LEVEL, debug_torch_op F = TypeVar("F", bound=Callable) @@ -130,6 +131,45 @@ def register_custom_op( return decorator +_PLAIN_TENSORS = (torch.Tensor, torch.nn.Parameter) +# Bound once: these run on every eager call of every op. +_is_compiling = torch.compiler.is_compiling +_dispatch_modes = torch._C._len_torch_dispatch_stack +_function_modes = torch._C._is_torch_function_mode_enabled +_profiling = torch._C._autograd._profiler_enabled +_grad_enabled = torch.is_grad_enabled + + +def _call_directly_in_eager(op_func: Callable, op: Callable) -> Callable: + """Call ``op_func`` itself in plain eager execution, skipping the dispatcher. + + The op is registered for one device key with no autograd formula, so for plain + CUDA tensors under no_grad the dispatcher only re-enters ``op_func``, at several + microseconds a call. Everything else keeps the registered op: tracing, dispatch + and function modes, the profiler (which records the op), autograd, tensor + subclasses, and tensors on other devices (which the op rejects or fakes). + """ + + @functools.wraps(op_func) + def call(*args, **kwargs): + if ( + _is_compiling() + or _dispatch_modes() + or _function_modes() + or _profiling() + or _grad_enabled() + ): + return op(*args, **kwargs) + for a in (*args, *kwargs.values()): + if isinstance(a, torch.Tensor) and ( + type(a) not in _PLAIN_TENSORS or not a.is_cuda + ): + return op(*args, **kwargs) + return op_func(*args, **kwargs) + + return call + + class CustomOpWrapper: def __init__( self, @@ -150,7 +190,8 @@ class CustomOpWrapper: @property def real_impl(self) -> Callable: if self._impl is None: - if not hasattr(torch.ops.sglang, self.op_name): + registered_here = not hasattr(torch.ops.sglang, self.op_name) + if registered_here: from sglang.srt.utils.common import direct_register_custom_op # NOTE(dark): if torch compile fail here, mark the decorator as eager @@ -161,7 +202,14 @@ class CustomOpWrapper: mutates_args=self.mutates_args, fake_impl=self.fake_impl, ) - self._impl = debug_torch_op(self.op_func, self.op_name) + op = debug_torch_op(self.op_func, self.op_name) + # Logging wraps the registered op, and a name registered elsewhere first + # runs that registration's function; both stay on the dispatcher. + self._impl = ( + _call_directly_in_eager(self.op_func, op) + if registered_here and not _KERNEL_API_LOG_LEVEL + else op + ) assert self._impl is not None return self._impl diff --git a/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py b/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py index 3acf49b51c..f437351f42 100644 --- a/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py +++ b/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py @@ -114,9 +114,9 @@ class TestDualStreamOutputs(CustomTestCase): x, 0, False, replicated ) + # opcheck validates the registered op, not its eager wrapper torch.library.opcheck( - invoke_op - := deepseek_v2.dsv4_flashinfer_moe_dual_stream_outputs, + torch.ops.sglang.dsv4_flashinfer_moe_dual_stream_outputs.default, (value, 0, False, replicated), test_utils=("test_schema", "test_faketensor"), ) diff --git a/test/registered/unit/utils/test_custom_op_eager_direct.py b/test/registered/unit/utils/test_custom_op_eager_direct.py new file mode 100644 index 0000000000..b11c6475f7 --- /dev/null +++ b/test/registered/unit/utils/test_custom_op_eager_direct.py @@ -0,0 +1,134 @@ +"""``register_custom_op`` ops skip the dispatcher in plain eager execution. + +Only plain device tensors under no_grad take the direct call. Tracing, dispatch and +function modes, the profiler, autograd, tensor subclasses, host tensors and a name +registered by another function first all keep the registered op. +""" + +import sys + +import pytest +import torch +from torch.overrides import TorchFunctionMode +from torch.utils._python_dispatch import TorchDispatchMode + +from sglang.srt.utils.custom_op import register_custom_op +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=15, stage="base-b", runner_config="1-gpu-small") + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") + +calls = [] # (function, its caller's frame name) + + +def _fake_add(x, y): + return torch.empty_like(x) + + +@register_custom_op( + op_name="test_eager_direct_add", mutates_args=[], fake_impl=_fake_add +) +def eager_direct_add(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + calls.append(("eager_direct_add", sys._getframe(1).f_code.co_name)) + return x + y + + +def _operands(device="cuda"): + return torch.randn(8, device=device), torch.randn(8, device=device) + + +def _took_direct_call(): + return calls[-1] == ("eager_direct_add", "call") + + +@torch.no_grad() +def test_eager_call_skips_the_dispatcher(): + x, y = _operands() + assert torch.equal(eager_direct_add(x, y), x + y) + assert _took_direct_call() + + +class _SeenOps(TorchDispatchMode): + def __init__(self): + super().__init__() + self.ops = [] + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + self.ops.append(str(func)) + return func(*args, **(kwargs or {})) + + +@torch.no_grad() +def test_dispatch_mode_sees_the_registered_op(): + x, y = _operands() + with _SeenOps() as mode: + assert torch.equal(eager_direct_add(x, y), x + y) + assert "sglang.test_eager_direct_add.default" in mode.ops + + +class _Passthrough(TorchFunctionMode): + def __torch_function__(self, func, types, args=(), kwargs=None): + return func(*args, **(kwargs or {})) + + +class _Sub(torch.Tensor): + pass + + +@pytest.mark.parametrize("case", ["grad", "function_mode", "subclass"]) +def test_other_contexts_use_the_dispatcher(case): + x, y = _operands() + if case == "grad": + out = eager_direct_add(x, y) + else: + with torch.no_grad(): + if case == "subclass": + out = eager_direct_add(x.as_subclass(_Sub), y) + else: + with _Passthrough(): + out = eager_direct_add(x, y) + assert torch.equal(out, x + y) + assert not _took_direct_call() + + +@torch.no_grad() +def test_profiler_records_the_op(): + x, y = _operands() + with torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CPU] + ) as prof: + eager_direct_add(x, y) + assert any("test_eager_direct_add" in event.name for event in prof.events()) + + +@torch.no_grad() +def test_host_tensors_are_still_rejected(): + with pytest.raises(NotImplementedError): + eager_direct_add(*_operands("cpu")) + + +@torch.no_grad() +def test_a_name_registered_first_elsewhere_keeps_that_function(): + def other(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + calls.append(("other", sys._getframe(1).f_code.co_name)) + return x - y + + second = register_custom_op( + op_name="test_eager_direct_add", mutates_args=[], fake_impl=_fake_add + )(other) + x, y = _operands() + assert torch.equal(second(x, y), x + y) + assert calls[-1][0] == "eager_direct_add" + + +def test_compile_traces_the_registered_op(): + x, y = _operands() + compiled = torch.compile( + lambda a, b: eager_direct_add(a, b) * 2, fullgraph=True, backend="eager" + ) + assert torch.equal(compiled(x, y), (x + y) * 2)