[Perf] kernels: Enter a JIT kernel's device guard only when the device differs (#44644)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-17 16:43:27 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent f75232cb42
commit 56a20d7e34
5 changed files with 56 additions and 3 deletions
@@ -9,6 +9,7 @@ from sglang.kernels.jit.utils.arch import (
from sglang.kernels.jit.utils.common import (
aligned_new_empty,
cache_once,
device_guard,
empty_sentinel,
get_ci_test_range,
is_hip_runtime,
@@ -30,6 +31,7 @@ __all__ = [
"should_run_full_tests",
"get_ci_test_range",
"cache_once",
"device_guard",
"lazy_register_class",
"is_hip_runtime",
"is_musa_runtime",
+11
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import contextlib
import functools
from typing import Any, Callable, Dict, List, Sequence, TypeVar
@@ -25,6 +26,16 @@ def get_ci_test_range(full_range: List[Any], ci_range: List[Any]) -> List[Any]:
return ci_range if is_in_ci() else full_range
_NO_GUARD = contextlib.nullcontext()
def device_guard(t: torch.Tensor) -> contextlib.AbstractContextManager:
"""``torch.cuda.device(t.device)``, skipped when ``t`` is already on the current device."""
if t.get_device() == torch.cuda.current_device():
return _NO_GUARD
return torch.cuda.device(t.device)
def cache_once(fn: F) -> F:
"""
NOTE: `functools.lru_cache` is not compatible with `torch.compile`
@@ -25,7 +25,7 @@ from typing import TYPE_CHECKING, Optional
import torch
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
from sglang.kernels.jit.utils import cache_once, device_guard, load_jit, make_cpp_args
from sglang.kernels.ops.diffusion.rope.complex_rope_triton import _fuse_real_sin
from sglang.srt.utils.custom_op import register_custom_op
@@ -75,7 +75,7 @@ def qknorm_complex_rope_cuda(
"""``rope(rmsnorm(x))`` for one ``[B, S, H, 128]`` tensor with packed head rows, out of place."""
out = torch.empty(x.shape, dtype=x.dtype, device=x.device)
module = _jit_module(_fuse_real_sin(x.device), False)
with torch.cuda.device(x.device):
with device_guard(x):
module.norm_rope(x, out, weight, _rope_real(rope), float(eps))
return out
@@ -107,7 +107,7 @@ def qknorm_complex_rope_pack_(
token_rows = k_out[:, prefix:]
copy_v = v_src is not None
module = _jit_module(_fuse_real_sin(q.device), copy_v)
with torch.cuda.device(q.device):
with device_guard(q):
module.norm_rope_pack(
q,
q_weight,