diff --git a/docs/docs/hardware-platforms/plugin.mdx b/docs/docs/hardware-platforms/plugin.mdx
index 0727b1b525..9d88ce6d09 100644
--- a/docs/docs/hardware-platforms/plugin.mdx
+++ b/docs/docs/hardware-platforms/plugin.mdx
@@ -576,6 +576,11 @@ field defaults to the conservative answer.
False |
The sgl_kernel HiCache transfer / write-back kernels are available; otherwise host pools skip their staging buffers. |
+
+ supports_pynccl |
+ False |
+ Model-parallel groups create a PyNccl communicator when the caller does not choose. Set it to True only if the device runs NCCL (or an NCCL-compatible library PyNccl loads). |
+
@@ -630,9 +635,16 @@ field defaults to the conservative answer.
"native" |
BaseFusedOp (fused-op) dispatch key name |
+
+ get_group_coordinator_device(local_rank) |
+ torch.device("cpu") |
+ Device each GroupCoordinator binds to on this rank (e.g. torch.device(f"mydev:{local_rank}")). Without it, device collectives on an out-of-tree platform run against a CPU device. |
+
+SGLang custom ops (e.g. the inplace_all_reduce collective) register their implementation under the PrivateUse1 dispatch key on an out-of-tree platform, the key torch reserves for out-of-tree backends.
+
#### Lifecycle Hooks (from SRTPlatform)
diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py
index 4524b21c3d..2656bfc2e3 100644
--- a/python/sglang/srt/distributed/parallel_state.py
+++ b/python/sglang/srt/distributed/parallel_state.py
@@ -331,19 +331,7 @@ class GroupCoordinator:
# Scale cohorts create these groups without the serving ranks.
use_local_synchronization = rank_offset > 0 and not recovered_rank
- if is_cuda_alike():
- device_id = (
- 0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank
- )
- self.device = torch.device(f"cuda:{device_id}")
- elif _is_npu:
- self.device = torch.device(f"npu:{local_rank}")
- elif _is_xpu:
- self.device = torch.device(f"xpu:{local_rank}")
- elif _is_musa:
- self.device = torch.device(f"musa:{local_rank}")
- else:
- self.device = torch.device("cpu")
+ self.device = _group_coordinator_device(local_rank)
self.device_module = torch.get_device_module(self.device)
for ranks in group_ranks:
@@ -2122,6 +2110,24 @@ class GroupCoordinator:
self.mq_broadcaster = None
+def _group_coordinator_device(local_rank: int) -> torch.device:
+ if is_cuda_alike():
+ device_id = (
+ 0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank
+ )
+ return torch.device(f"cuda:{device_id}")
+ if _is_npu:
+ return torch.device(f"npu:{local_rank}")
+ if _is_xpu:
+ return torch.device(f"xpu:{local_rank}")
+ if _is_musa:
+ return torch.device(f"musa:{local_rank}")
+ platform = platforms.current_platform
+ if platform.is_out_of_tree():
+ return platform.get_group_coordinator_device(local_rank)
+ return torch.device("cpu")
+
+
_WORLD: Optional[GroupCoordinator] = None
@@ -2173,15 +2179,17 @@ def init_model_parallel_group(
use_mscclpp = _ENABLE_MSCCLPP
if use_torch_symm_mem_allreduce is None:
use_torch_symm_mem_allreduce = _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE
+ if use_pynccl is None:
+ platform = platforms.current_platform
+ if platform.is_out_of_tree() and not platform.capabilities.supports_pynccl:
+ use_pynccl = False
+ else:
+ use_pynccl = not (_is_npu or _is_xpu or backend == "mooncake")
return GroupCoordinator(
group_ranks=group_ranks,
local_rank=local_rank,
torch_distributed_backend=backend,
- use_pynccl=(
- not (_is_npu or _is_xpu or backend == "mooncake")
- if use_pynccl is None
- else use_pynccl
- ),
+ use_pynccl=use_pynccl,
use_mscclpp=use_mscclpp,
use_custom_allreduce=use_custom_allreduce,
use_torch_symm_mem_all_reduce=use_torch_symm_mem_allreduce,
diff --git a/python/sglang/srt/platforms/cuda.py b/python/sglang/srt/platforms/cuda.py
index cb62bf7846..37cb541866 100644
--- a/python/sglang/srt/platforms/cuda.py
+++ b/python/sglang/srt/platforms/cuda.py
@@ -108,4 +108,5 @@ class CudaSRTPlatform(CudaDeviceMixin, SRTPlatform):
graph_capture=True,
piecewise_graph=True,
hicache_device_kernels=True,
+ supports_pynccl=True,
)
diff --git a/python/sglang/srt/platforms/interface.py b/python/sglang/srt/platforms/interface.py
index 70fc60d1ef..843ff70903 100644
--- a/python/sglang/srt/platforms/interface.py
+++ b/python/sglang/srt/platforms/interface.py
@@ -61,12 +61,16 @@ class PlatformCapabilities(msgspec.Struct, frozen=True, kw_only=True):
``hicache_device_kernels``: the sgl_kernel HiCache transfer / write-back
kernels are available; otherwise the host pools skip their staging
buffers.
+ ``supports_pynccl``: model-parallel groups create a PyNccl communicator
+ when the caller does not choose. Read for out-of-tree platforms; in-tree
+ devices decide in ``parallel_state``.
"""
supports_triton: bool = False
graph_capture: bool = False
piecewise_graph: bool = False
hicache_device_kernels: bool = False
+ supports_pynccl: bool = False
class SRTPlatform(DeviceMixin):
@@ -208,6 +212,21 @@ class SRTPlatform(DeviceMixin):
"""Whether this out-of-tree platform supports Mamba cache snapshots."""
return False
+ # ------------------------------------------------------------------
+ # Distributed
+ # ------------------------------------------------------------------
+
+ def get_group_coordinator_device(self, local_rank: int) -> torch.device:
+ """Return the device a ``GroupCoordinator`` binds to on this rank.
+
+ Consulted only for out-of-tree platforms; in-tree devices are resolved
+ in ``parallel_state``. The default is CPU, which is what an unknown
+ device got before this hook existed.
+ """
+ import torch
+
+ return torch.device("cpu")
+
# ------------------------------------------------------------------
# Initialization
# ------------------------------------------------------------------
diff --git a/python/sglang/srt/platforms/musa.py b/python/sglang/srt/platforms/musa.py
index 508213a240..131b55fd96 100644
--- a/python/sglang/srt/platforms/musa.py
+++ b/python/sglang/srt/platforms/musa.py
@@ -22,4 +22,6 @@ class MusaDeviceMixin(CudaDeviceMixin):
class MusaSRTPlatform(MusaDeviceMixin, SRTPlatform):
"""Default in-tree MUSA SRT platform."""
- capabilities = PlatformCapabilities(supports_triton=True, graph_capture=True)
+ capabilities = PlatformCapabilities(
+ supports_triton=True, graph_capture=True, supports_pynccl=True
+ )
diff --git a/python/sglang/srt/platforms/rocm.py b/python/sglang/srt/platforms/rocm.py
index c242436f00..a556fbaa69 100644
--- a/python/sglang/srt/platforms/rocm.py
+++ b/python/sglang/srt/platforms/rocm.py
@@ -28,4 +28,5 @@ class RocmSRTPlatform(RocmDeviceMixin, SRTPlatform):
graph_capture=True,
piecewise_graph=True,
hicache_device_kernels=True,
+ supports_pynccl=True,
)
diff --git a/python/sglang/srt/platforms/sail.py b/python/sglang/srt/platforms/sail.py
index 882a97f163..63077bc3c4 100644
--- a/python/sglang/srt/platforms/sail.py
+++ b/python/sglang/srt/platforms/sail.py
@@ -26,4 +26,5 @@ class SailSRTPlatform(SailDeviceMixin, SRTPlatform):
supports_triton=True,
graph_capture=True,
piecewise_graph=True,
+ supports_pynccl=True,
)
diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py
index 463b8df2dd..752a6f9693 100644
--- a/python/sglang/srt/utils/common.py
+++ b/python/sglang/srt/utils/common.py
@@ -2988,8 +2988,9 @@ def direct_register_custom_op(
try:
my_lib.define(op_name + schema_str)
- if is_npu():
+ if is_npu() or current_platform.is_out_of_tree():
# https://github.com/sgl-project/sglang/pull/12287/files#r2499583982
+ # Out-of-tree devices are torch PrivateUse1 backends too.
my_lib.impl(op_name, op_func, "PrivateUse1")
elif is_xpu():
my_lib.impl(op_name, op_func, "XPU")
diff --git a/test/registered/unit/distributed/test_oot_group_coordinator_hooks.py b/test/registered/unit/distributed/test_oot_group_coordinator_hooks.py
new file mode 100644
index 0000000000..bab66ca07e
--- /dev/null
+++ b/test/registered/unit/distributed/test_oot_group_coordinator_hooks.py
@@ -0,0 +1,114 @@
+import sys
+from unittest.mock import Mock, patch
+
+import pytest
+import torch
+
+from sglang.srt.platforms.interface import PlatformCapabilities, SRTPlatform
+from sglang.test.ci.ci_register import register_cpu_ci
+
+register_cpu_ci(est_time=2, suite="base-a-test-cpu")
+
+parallel_state = pytest.importorskip("sglang.srt.distributed.parallel_state")
+
+
+def _platform(out_of_tree: bool, device: str = "cpu", pynccl: bool = False) -> Mock:
+ platform = Mock(spec=SRTPlatform)
+ platform.is_out_of_tree.return_value = out_of_tree
+ platform.get_group_coordinator_device.side_effect = lambda local_rank: torch.device(
+ device
+ )
+ platform.capabilities = PlatformCapabilities(supports_pynccl=pynccl)
+ return platform
+
+
+def _no_in_tree_device():
+ return (
+ patch.object(parallel_state, "is_cuda_alike", return_value=False),
+ patch.object(parallel_state, "_is_npu", False),
+ patch.object(parallel_state, "_is_xpu", False),
+ patch.object(parallel_state, "_is_musa", False),
+ )
+
+
+def _with_platform(platform):
+ return patch.object(
+ parallel_state.platforms, "current_platform", platform, create=True
+ )
+
+
+def test_out_of_tree_platform_chooses_group_device():
+ platform = _platform(out_of_tree=True, device="meta")
+ p1, p2, p3, p4 = _no_in_tree_device()
+ with p1, p2, p3, p4, _with_platform(platform):
+ device = parallel_state._group_coordinator_device(3)
+ assert device == torch.device("meta")
+ platform.get_group_coordinator_device.assert_called_once_with(3)
+
+
+def test_in_tree_unknown_device_falls_back_to_cpu():
+ platform = _platform(out_of_tree=False)
+ p1, p2, p3, p4 = _no_in_tree_device()
+ with p1, p2, p3, p4, _with_platform(platform):
+ device = parallel_state._group_coordinator_device(3)
+ assert device == torch.device("cpu")
+ platform.get_group_coordinator_device.assert_not_called()
+
+
+def test_base_platform_defaults():
+ platform = SRTPlatform()
+ assert platform.get_group_coordinator_device(0) == torch.device("cpu")
+ assert platform.capabilities.supports_pynccl is False
+
+
+@pytest.mark.parametrize(
+ "out_of_tree, platform_pynccl, explicit, expected",
+ [
+ (True, False, None, False),
+ (True, True, None, True),
+ (False, False, None, True),
+ (True, False, True, True),
+ (False, True, False, False),
+ ],
+)
+def test_pynccl_default(out_of_tree, platform_pynccl, explicit, expected):
+ platform = _platform(out_of_tree=out_of_tree, pynccl=platform_pynccl)
+ with (
+ _with_platform(platform),
+ patch.object(parallel_state, "_is_npu", False),
+ patch.object(parallel_state, "_is_xpu", False),
+ patch.object(parallel_state, "GroupCoordinator") as coordinator,
+ ):
+ parallel_state.init_model_parallel_group(
+ group_ranks=[[0, 1]],
+ local_rank=0,
+ backend="gloo",
+ use_pynccl=explicit,
+ )
+ assert coordinator.call_args.kwargs["use_pynccl"] is expected
+
+
+@pytest.mark.parametrize("out_of_tree, key", [(True, "PrivateUse1"), (False, "CUDA")])
+def test_custom_op_dispatch_key(out_of_tree, key):
+ from torch.library import Library
+
+ from sglang.srt.utils import common
+
+ lib_name = f"sglang_oot_test_{int(out_of_tree)}"
+ lib = Library(lib_name, "FRAGMENT")
+
+ def op(x: torch.Tensor) -> torch.Tensor:
+ return x.clone()
+
+ with (
+ patch.object(common, "current_platform", _platform(out_of_tree=out_of_tree)),
+ patch.object(common, "is_npu", return_value=False),
+ patch.object(common, "is_xpu", return_value=False),
+ patch.object(common, "is_musa", return_value=False),
+ ):
+ common.direct_register_custom_op("op", op, mutates_args=[], target_lib=lib)
+ assert torch._C._dispatch_has_kernel_for_dispatch_key(f"{lib_name}::op", key)
+
+
+if __name__ == "__main__":
+ sys.exit(pytest.main([__file__, "-v"]))