[Perf] kernels: Enter a JIT kernel's device guard only when the device differs (#44644)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
f75232cb42
commit
56a20d7e34
@@ -9,6 +9,7 @@ from sglang.kernels.jit.utils.arch import (
|
||||
from sglang.kernels.jit.utils.common import (
|
||||
aligned_new_empty,
|
||||
cache_once,
|
||||
device_guard,
|
||||
empty_sentinel,
|
||||
get_ci_test_range,
|
||||
is_hip_runtime,
|
||||
@@ -30,6 +31,7 @@ __all__ = [
|
||||
"should_run_full_tests",
|
||||
"get_ci_test_range",
|
||||
"cache_once",
|
||||
"device_guard",
|
||||
"lazy_register_class",
|
||||
"is_hip_runtime",
|
||||
"is_musa_runtime",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
from typing import Any, Callable, Dict, List, Sequence, TypeVar
|
||||
|
||||
@@ -25,6 +26,16 @@ def get_ci_test_range(full_range: List[Any], ci_range: List[Any]) -> List[Any]:
|
||||
return ci_range if is_in_ci() else full_range
|
||||
|
||||
|
||||
_NO_GUARD = contextlib.nullcontext()
|
||||
|
||||
|
||||
def device_guard(t: torch.Tensor) -> contextlib.AbstractContextManager:
|
||||
"""``torch.cuda.device(t.device)``, skipped when ``t`` is already on the current device."""
|
||||
if t.get_device() == torch.cuda.current_device():
|
||||
return _NO_GUARD
|
||||
return torch.cuda.device(t.device)
|
||||
|
||||
|
||||
def cache_once(fn: F) -> F:
|
||||
"""
|
||||
NOTE: `functools.lru_cache` is not compatible with `torch.compile`
|
||||
|
||||
@@ -25,7 +25,7 @@ from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args
|
||||
from sglang.kernels.jit.utils import cache_once, device_guard, load_jit, make_cpp_args
|
||||
from sglang.kernels.ops.diffusion.rope.complex_rope_triton import _fuse_real_sin
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
@@ -75,7 +75,7 @@ def qknorm_complex_rope_cuda(
|
||||
"""``rope(rmsnorm(x))`` for one ``[B, S, H, 128]`` tensor with packed head rows, out of place."""
|
||||
out = torch.empty(x.shape, dtype=x.dtype, device=x.device)
|
||||
module = _jit_module(_fuse_real_sin(x.device), False)
|
||||
with torch.cuda.device(x.device):
|
||||
with device_guard(x):
|
||||
module.norm_rope(x, out, weight, _rope_real(rope), float(eps))
|
||||
return out
|
||||
|
||||
@@ -107,7 +107,7 @@ def qknorm_complex_rope_pack_(
|
||||
token_rows = k_out[:, prefix:]
|
||||
copy_v = v_src is not None
|
||||
module = _jit_module(_fuse_real_sin(q.device), copy_v)
|
||||
with torch.cuda.device(q.device):
|
||||
with device_guard(q):
|
||||
module.norm_rope_pack(
|
||||
q,
|
||||
q_weight,
|
||||
|
||||
Reference in New Issue
Block a user