From 56a20d7e3436005278be612fd1623ac9d488f124 Mon Sep 17 00:00:00 2001 From: Mick Date: Sat, 17 Oct 2026 16:43:27 +0800 Subject: [PATCH] [Perf] kernels: Enter a JIT kernel's device guard only when the device differs (#44644) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- python/sglang/kernels/jit/utils/__init__.py | 2 + python/sglang/kernels/jit/utils/common.py | 11 ++++++ .../diffusion/rope/qknorm_complex_rope_jit.py | 6 +-- .../test_diffusion_stage_profiler_sync.py | 3 ++ .../unit/kernels/test_device_guard.py | 37 +++++++++++++++++++ 5 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 test/registered/unit/kernels/test_device_guard.py diff --git a/python/sglang/kernels/jit/utils/__init__.py b/python/sglang/kernels/jit/utils/__init__.py index 2469116a82..06c7fe8ed8 100644 --- a/python/sglang/kernels/jit/utils/__init__.py +++ b/python/sglang/kernels/jit/utils/__init__.py @@ -9,6 +9,7 @@ from sglang.kernels.jit.utils.arch import ( from sglang.kernels.jit.utils.common import ( aligned_new_empty, cache_once, + device_guard, empty_sentinel, get_ci_test_range, is_hip_runtime, @@ -30,6 +31,7 @@ __all__ = [ "should_run_full_tests", "get_ci_test_range", "cache_once", + "device_guard", "lazy_register_class", "is_hip_runtime", "is_musa_runtime", diff --git a/python/sglang/kernels/jit/utils/common.py b/python/sglang/kernels/jit/utils/common.py index dc34560560..e78317c971 100644 --- a/python/sglang/kernels/jit/utils/common.py +++ b/python/sglang/kernels/jit/utils/common.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import functools from typing import Any, Callable, Dict, List, Sequence, TypeVar @@ -25,6 +26,16 @@ def get_ci_test_range(full_range: List[Any], ci_range: List[Any]) -> List[Any]: return ci_range if is_in_ci() else full_range +_NO_GUARD = contextlib.nullcontext() + + +def device_guard(t: torch.Tensor) -> contextlib.AbstractContextManager: + """``torch.cuda.device(t.device)``, skipped when ``t`` is already on the current device.""" + if t.get_device() == torch.cuda.current_device(): + return _NO_GUARD + return torch.cuda.device(t.device) + + def cache_once(fn: F) -> F: """ NOTE: `functools.lru_cache` is not compatible with `torch.compile` diff --git a/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py b/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py index 12747bae49..ea3b824071 100644 --- a/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py +++ b/python/sglang/kernels/ops/diffusion/rope/qknorm_complex_rope_jit.py @@ -25,7 +25,7 @@ from typing import TYPE_CHECKING, Optional import torch -from sglang.kernels.jit.utils import cache_once, load_jit, make_cpp_args +from sglang.kernels.jit.utils import cache_once, device_guard, load_jit, make_cpp_args from sglang.kernels.ops.diffusion.rope.complex_rope_triton import _fuse_real_sin from sglang.srt.utils.custom_op import register_custom_op @@ -75,7 +75,7 @@ def qknorm_complex_rope_cuda( """``rope(rmsnorm(x))`` for one ``[B, S, H, 128]`` tensor with packed head rows, out of place.""" out = torch.empty(x.shape, dtype=x.dtype, device=x.device) module = _jit_module(_fuse_real_sin(x.device), False) - with torch.cuda.device(x.device): + with device_guard(x): module.norm_rope(x, out, weight, _rope_real(rope), float(eps)) return out @@ -107,7 +107,7 @@ def qknorm_complex_rope_pack_( token_rows = k_out[:, prefix:] copy_v = v_src is not None module = _jit_module(_fuse_real_sin(q.device), copy_v) - with torch.cuda.device(q.device): + with device_guard(q): module.norm_rope_pack( q, q_weight, diff --git a/test/registered/profiling/test_diffusion_stage_profiler_sync.py b/test/registered/profiling/test_diffusion_stage_profiler_sync.py index 82ba6fe290..0101830551 100644 --- a/test/registered/profiling/test_diffusion_stage_profiler_sync.py +++ b/test/registered/profiling/test_diffusion_stage_profiler_sync.py @@ -25,6 +25,9 @@ def test_stage_entry_sync_excludes_previous_stage_tail(monkeypatch): logger = init_logger(__name__) metrics = RequestMetrics("stage-sync-test") + # warm the consumer kernels before measuring stage synchronization + torch.ones(8, device="cuda").sum().cpu() + # Calibrate ~0.5 s of queued GPU work. torch.cuda.synchronize() t0 = time.perf_counter() diff --git a/test/registered/unit/kernels/test_device_guard.py b/test/registered/unit/kernels/test_device_guard.py new file mode 100644 index 0000000000..9419438bfb --- /dev/null +++ b/test/registered/unit/kernels/test_device_guard.py @@ -0,0 +1,37 @@ +"""``device_guard`` enters a tensor's CUDA device only when it is not the current one.""" + +import sys + +import pytest +import torch + +from sglang.kernels.jit.utils import device_guard +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=5, stage="base-b", runner_config="1-gpu-small") + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required") + + +def test_current_device_enters_no_guard(monkeypatch): + x = torch.empty(1, device=f"cuda:{torch.cuda.current_device()}") + + def entered(device): + raise AssertionError(f"guard entered for {device}") + + monkeypatch.setattr(torch.cuda, "device", entered) + with device_guard(x): + assert torch.cuda.current_device() == x.get_device() + + +@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="needs two GPUs") +def test_other_device_is_entered_and_restored(): + torch.cuda.set_device(0) + x = torch.empty(1, device="cuda:1") + with device_guard(x): + assert torch.cuda.current_device() == 1 + assert torch.cuda.current_device() == 0 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))