[Perf] custom_op: call register_custom_op ops directly in plain eager execution (#44158)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-18 10:10:10 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent d748bede96
commit 14ad64c411
3 changed files with 187 additions and 5 deletions
+51 -3
View File
@@ -1,12 +1,13 @@
from __future__ import annotations
import functools
import inspect
from typing import Any, Callable, List, Optional, TypeVar, Union, overload
import torch
import torch.library
from sglang.kernels.kernel_api_logging import debug_torch_op
from sglang.kernels.kernel_api_logging import _KERNEL_API_LOG_LEVEL, debug_torch_op
F = TypeVar("F", bound=Callable)
@@ -130,6 +131,45 @@ def register_custom_op(
return decorator
_PLAIN_TENSORS = (torch.Tensor, torch.nn.Parameter)
# Bound once: these run on every eager call of every op.
_is_compiling = torch.compiler.is_compiling
_dispatch_modes = torch._C._len_torch_dispatch_stack
_function_modes = torch._C._is_torch_function_mode_enabled
_profiling = torch._C._autograd._profiler_enabled
_grad_enabled = torch.is_grad_enabled
def _call_directly_in_eager(op_func: Callable, op: Callable) -> Callable:
"""Call ``op_func`` itself in plain eager execution, skipping the dispatcher.
The op is registered for one device key with no autograd formula, so for plain
CUDA tensors under no_grad the dispatcher only re-enters ``op_func``, at several
microseconds a call. Everything else keeps the registered op: tracing, dispatch
and function modes, the profiler (which records the op), autograd, tensor
subclasses, and tensors on other devices (which the op rejects or fakes).
"""
@functools.wraps(op_func)
def call(*args, **kwargs):
if (
_is_compiling()
or _dispatch_modes()
or _function_modes()
or _profiling()
or _grad_enabled()
):
return op(*args, **kwargs)
for a in (*args, *kwargs.values()):
if isinstance(a, torch.Tensor) and (
type(a) not in _PLAIN_TENSORS or not a.is_cuda
):
return op(*args, **kwargs)
return op_func(*args, **kwargs)
return call
class CustomOpWrapper:
def __init__(
self,
@@ -150,7 +190,8 @@ class CustomOpWrapper:
@property
def real_impl(self) -> Callable:
if self._impl is None:
if not hasattr(torch.ops.sglang, self.op_name):
registered_here = not hasattr(torch.ops.sglang, self.op_name)
if registered_here:
from sglang.srt.utils.common import direct_register_custom_op
# NOTE(dark): if torch compile fail here, mark the decorator as eager
@@ -161,7 +202,14 @@ class CustomOpWrapper:
mutates_args=self.mutates_args,
fake_impl=self.fake_impl,
)
self._impl = debug_torch_op(self.op_func, self.op_name)
op = debug_torch_op(self.op_func, self.op_name)
# Logging wraps the registered op, and a name registered elsewhere first
# runs that registration's function; both stay on the dispatcher.
self._impl = (
_call_directly_in_eager(self.op_func, op)
if registered_here and not _KERNEL_API_LOG_LEVEL
else op
)
assert self._impl is not None
return self._impl