diff --git a/docs/docs/hardware-platforms/plugin.mdx b/docs/docs/hardware-platforms/plugin.mdx index 0727b1b525..9d88ce6d09 100644 --- a/docs/docs/hardware-platforms/plugin.mdx +++ b/docs/docs/hardware-platforms/plugin.mdx @@ -576,6 +576,11 @@ field defaults to the conservative answer. False The sgl_kernel HiCache transfer / write-back kernels are available; otherwise host pools skip their staging buffers. + + supports_pynccl + False + Model-parallel groups create a PyNccl communicator when the caller does not choose. Set it to True only if the device runs NCCL (or an NCCL-compatible library PyNccl loads). + @@ -630,9 +635,16 @@ field defaults to the conservative answer. "native" BaseFusedOp (fused-op) dispatch key name + + get_group_coordinator_device(local_rank) + torch.device("cpu") + Device each GroupCoordinator binds to on this rank (e.g. torch.device(f"mydev:{local_rank}")). Without it, device collectives on an out-of-tree platform run against a CPU device. + +SGLang custom ops (e.g. the inplace_all_reduce collective) register their implementation under the PrivateUse1 dispatch key on an out-of-tree platform, the key torch reserves for out-of-tree backends. + #### Lifecycle Hooks (from SRTPlatform) diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index 4524b21c3d..2656bfc2e3 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -331,19 +331,7 @@ class GroupCoordinator: # Scale cohorts create these groups without the serving ranks. use_local_synchronization = rank_offset > 0 and not recovered_rank - if is_cuda_alike(): - device_id = ( - 0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank - ) - self.device = torch.device(f"cuda:{device_id}") - elif _is_npu: - self.device = torch.device(f"npu:{local_rank}") - elif _is_xpu: - self.device = torch.device(f"xpu:{local_rank}") - elif _is_musa: - self.device = torch.device(f"musa:{local_rank}") - else: - self.device = torch.device("cpu") + self.device = _group_coordinator_device(local_rank) self.device_module = torch.get_device_module(self.device) for ranks in group_ranks: @@ -2122,6 +2110,24 @@ class GroupCoordinator: self.mq_broadcaster = None +def _group_coordinator_device(local_rank: int) -> torch.device: + if is_cuda_alike(): + device_id = ( + 0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank + ) + return torch.device(f"cuda:{device_id}") + if _is_npu: + return torch.device(f"npu:{local_rank}") + if _is_xpu: + return torch.device(f"xpu:{local_rank}") + if _is_musa: + return torch.device(f"musa:{local_rank}") + platform = platforms.current_platform + if platform.is_out_of_tree(): + return platform.get_group_coordinator_device(local_rank) + return torch.device("cpu") + + _WORLD: Optional[GroupCoordinator] = None @@ -2173,15 +2179,17 @@ def init_model_parallel_group( use_mscclpp = _ENABLE_MSCCLPP if use_torch_symm_mem_allreduce is None: use_torch_symm_mem_allreduce = _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE + if use_pynccl is None: + platform = platforms.current_platform + if platform.is_out_of_tree() and not platform.capabilities.supports_pynccl: + use_pynccl = False + else: + use_pynccl = not (_is_npu or _is_xpu or backend == "mooncake") return GroupCoordinator( group_ranks=group_ranks, local_rank=local_rank, torch_distributed_backend=backend, - use_pynccl=( - not (_is_npu or _is_xpu or backend == "mooncake") - if use_pynccl is None - else use_pynccl - ), + use_pynccl=use_pynccl, use_mscclpp=use_mscclpp, use_custom_allreduce=use_custom_allreduce, use_torch_symm_mem_all_reduce=use_torch_symm_mem_allreduce, diff --git a/python/sglang/srt/platforms/cuda.py b/python/sglang/srt/platforms/cuda.py index cb62bf7846..37cb541866 100644 --- a/python/sglang/srt/platforms/cuda.py +++ b/python/sglang/srt/platforms/cuda.py @@ -108,4 +108,5 @@ class CudaSRTPlatform(CudaDeviceMixin, SRTPlatform): graph_capture=True, piecewise_graph=True, hicache_device_kernels=True, + supports_pynccl=True, ) diff --git a/python/sglang/srt/platforms/interface.py b/python/sglang/srt/platforms/interface.py index 70fc60d1ef..843ff70903 100644 --- a/python/sglang/srt/platforms/interface.py +++ b/python/sglang/srt/platforms/interface.py @@ -61,12 +61,16 @@ class PlatformCapabilities(msgspec.Struct, frozen=True, kw_only=True): ``hicache_device_kernels``: the sgl_kernel HiCache transfer / write-back kernels are available; otherwise the host pools skip their staging buffers. + ``supports_pynccl``: model-parallel groups create a PyNccl communicator + when the caller does not choose. Read for out-of-tree platforms; in-tree + devices decide in ``parallel_state``. """ supports_triton: bool = False graph_capture: bool = False piecewise_graph: bool = False hicache_device_kernels: bool = False + supports_pynccl: bool = False class SRTPlatform(DeviceMixin): @@ -208,6 +212,21 @@ class SRTPlatform(DeviceMixin): """Whether this out-of-tree platform supports Mamba cache snapshots.""" return False + # ------------------------------------------------------------------ + # Distributed + # ------------------------------------------------------------------ + + def get_group_coordinator_device(self, local_rank: int) -> torch.device: + """Return the device a ``GroupCoordinator`` binds to on this rank. + + Consulted only for out-of-tree platforms; in-tree devices are resolved + in ``parallel_state``. The default is CPU, which is what an unknown + device got before this hook existed. + """ + import torch + + return torch.device("cpu") + # ------------------------------------------------------------------ # Initialization # ------------------------------------------------------------------ diff --git a/python/sglang/srt/platforms/musa.py b/python/sglang/srt/platforms/musa.py index 508213a240..131b55fd96 100644 --- a/python/sglang/srt/platforms/musa.py +++ b/python/sglang/srt/platforms/musa.py @@ -22,4 +22,6 @@ class MusaDeviceMixin(CudaDeviceMixin): class MusaSRTPlatform(MusaDeviceMixin, SRTPlatform): """Default in-tree MUSA SRT platform.""" - capabilities = PlatformCapabilities(supports_triton=True, graph_capture=True) + capabilities = PlatformCapabilities( + supports_triton=True, graph_capture=True, supports_pynccl=True + ) diff --git a/python/sglang/srt/platforms/rocm.py b/python/sglang/srt/platforms/rocm.py index c242436f00..a556fbaa69 100644 --- a/python/sglang/srt/platforms/rocm.py +++ b/python/sglang/srt/platforms/rocm.py @@ -28,4 +28,5 @@ class RocmSRTPlatform(RocmDeviceMixin, SRTPlatform): graph_capture=True, piecewise_graph=True, hicache_device_kernels=True, + supports_pynccl=True, ) diff --git a/python/sglang/srt/platforms/sail.py b/python/sglang/srt/platforms/sail.py index 882a97f163..63077bc3c4 100644 --- a/python/sglang/srt/platforms/sail.py +++ b/python/sglang/srt/platforms/sail.py @@ -26,4 +26,5 @@ class SailSRTPlatform(SailDeviceMixin, SRTPlatform): supports_triton=True, graph_capture=True, piecewise_graph=True, + supports_pynccl=True, ) diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index 463b8df2dd..752a6f9693 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -2988,8 +2988,9 @@ def direct_register_custom_op( try: my_lib.define(op_name + schema_str) - if is_npu(): + if is_npu() or current_platform.is_out_of_tree(): # https://github.com/sgl-project/sglang/pull/12287/files#r2499583982 + # Out-of-tree devices are torch PrivateUse1 backends too. my_lib.impl(op_name, op_func, "PrivateUse1") elif is_xpu(): my_lib.impl(op_name, op_func, "XPU") diff --git a/test/registered/unit/distributed/test_oot_group_coordinator_hooks.py b/test/registered/unit/distributed/test_oot_group_coordinator_hooks.py new file mode 100644 index 0000000000..bab66ca07e --- /dev/null +++ b/test/registered/unit/distributed/test_oot_group_coordinator_hooks.py @@ -0,0 +1,114 @@ +import sys +from unittest.mock import Mock, patch + +import pytest +import torch + +from sglang.srt.platforms.interface import PlatformCapabilities, SRTPlatform +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + +parallel_state = pytest.importorskip("sglang.srt.distributed.parallel_state") + + +def _platform(out_of_tree: bool, device: str = "cpu", pynccl: bool = False) -> Mock: + platform = Mock(spec=SRTPlatform) + platform.is_out_of_tree.return_value = out_of_tree + platform.get_group_coordinator_device.side_effect = lambda local_rank: torch.device( + device + ) + platform.capabilities = PlatformCapabilities(supports_pynccl=pynccl) + return platform + + +def _no_in_tree_device(): + return ( + patch.object(parallel_state, "is_cuda_alike", return_value=False), + patch.object(parallel_state, "_is_npu", False), + patch.object(parallel_state, "_is_xpu", False), + patch.object(parallel_state, "_is_musa", False), + ) + + +def _with_platform(platform): + return patch.object( + parallel_state.platforms, "current_platform", platform, create=True + ) + + +def test_out_of_tree_platform_chooses_group_device(): + platform = _platform(out_of_tree=True, device="meta") + p1, p2, p3, p4 = _no_in_tree_device() + with p1, p2, p3, p4, _with_platform(platform): + device = parallel_state._group_coordinator_device(3) + assert device == torch.device("meta") + platform.get_group_coordinator_device.assert_called_once_with(3) + + +def test_in_tree_unknown_device_falls_back_to_cpu(): + platform = _platform(out_of_tree=False) + p1, p2, p3, p4 = _no_in_tree_device() + with p1, p2, p3, p4, _with_platform(platform): + device = parallel_state._group_coordinator_device(3) + assert device == torch.device("cpu") + platform.get_group_coordinator_device.assert_not_called() + + +def test_base_platform_defaults(): + platform = SRTPlatform() + assert platform.get_group_coordinator_device(0) == torch.device("cpu") + assert platform.capabilities.supports_pynccl is False + + +@pytest.mark.parametrize( + "out_of_tree, platform_pynccl, explicit, expected", + [ + (True, False, None, False), + (True, True, None, True), + (False, False, None, True), + (True, False, True, True), + (False, True, False, False), + ], +) +def test_pynccl_default(out_of_tree, platform_pynccl, explicit, expected): + platform = _platform(out_of_tree=out_of_tree, pynccl=platform_pynccl) + with ( + _with_platform(platform), + patch.object(parallel_state, "_is_npu", False), + patch.object(parallel_state, "_is_xpu", False), + patch.object(parallel_state, "GroupCoordinator") as coordinator, + ): + parallel_state.init_model_parallel_group( + group_ranks=[[0, 1]], + local_rank=0, + backend="gloo", + use_pynccl=explicit, + ) + assert coordinator.call_args.kwargs["use_pynccl"] is expected + + +@pytest.mark.parametrize("out_of_tree, key", [(True, "PrivateUse1"), (False, "CUDA")]) +def test_custom_op_dispatch_key(out_of_tree, key): + from torch.library import Library + + from sglang.srt.utils import common + + lib_name = f"sglang_oot_test_{int(out_of_tree)}" + lib = Library(lib_name, "FRAGMENT") + + def op(x: torch.Tensor) -> torch.Tensor: + return x.clone() + + with ( + patch.object(common, "current_platform", _platform(out_of_tree=out_of_tree)), + patch.object(common, "is_npu", return_value=False), + patch.object(common, "is_xpu", return_value=False), + patch.object(common, "is_musa", return_value=False), + ): + common.direct_register_custom_op("op", op, mutates_args=[], target_lib=lib) + assert torch._C._dispatch_has_kernel_for_dispatch_key(f"{lib_name}::op", key) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"]))