[Feat] platforms: Let out-of-tree platforms bind group devices and custom ops (#45897)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alex Nails
2026-10-23 12:02:42 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 1e894eb92d
commit baf80c61b7
9 changed files with 179 additions and 20 deletions
+12
View File
@@ -576,6 +576,11 @@ field defaults to the conservative answer.
<td><code>False</code></td>
<td>The sgl_kernel HiCache transfer / write-back kernels are available; otherwise host pools skip their staging buffers.</td>
</tr>
<tr>
<td><code>supports_pynccl</code></td>
<td><code>False</code></td>
<td>Model-parallel groups create a PyNccl communicator when the caller does not choose. Set it to True only if the device runs NCCL (or an NCCL-compatible library PyNccl loads).</td>
</tr>
</tbody>
</table>
@@ -630,9 +635,16 @@ field defaults to the conservative answer.
<td><code>"native"</code></td>
<td>BaseFusedOp (fused-op) dispatch key name</td>
</tr>
<tr>
<td><code>get_group_coordinator_device(local_rank)</code></td>
<td><code>torch.device("cpu")</code></td>
<td>Device each <code>GroupCoordinator</code> binds to on this rank (e.g. <code>torch.device(f"mydev:{local_rank}")</code>). Without it, device collectives on an out-of-tree platform run against a CPU device.</td>
</tr>
</tbody>
</table>
SGLang custom ops (e.g. the <code>inplace_all_reduce</code> collective) register their implementation under the <code>PrivateUse1</code> dispatch key on an out-of-tree platform, the key torch reserves for out-of-tree backends.
#### Lifecycle Hooks (from SRTPlatform)
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
+26 -18
View File
@@ -331,19 +331,7 @@ class GroupCoordinator:
# Scale cohorts create these groups without the serving ranks.
use_local_synchronization = rank_offset > 0 and not recovered_rank
if is_cuda_alike():
device_id = (
0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank
)
self.device = torch.device(f"cuda:{device_id}")
elif _is_npu:
self.device = torch.device(f"npu:{local_rank}")
elif _is_xpu:
self.device = torch.device(f"xpu:{local_rank}")
elif _is_musa:
self.device = torch.device(f"musa:{local_rank}")
else:
self.device = torch.device("cpu")
self.device = _group_coordinator_device(local_rank)
self.device_module = torch.get_device_module(self.device)
for ranks in group_ranks:
@@ -2122,6 +2110,24 @@ class GroupCoordinator:
self.mq_broadcaster = None
def _group_coordinator_device(local_rank: int) -> torch.device:
if is_cuda_alike():
device_id = (
0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank
)
return torch.device(f"cuda:{device_id}")
if _is_npu:
return torch.device(f"npu:{local_rank}")
if _is_xpu:
return torch.device(f"xpu:{local_rank}")
if _is_musa:
return torch.device(f"musa:{local_rank}")
platform = platforms.current_platform
if platform.is_out_of_tree():
return platform.get_group_coordinator_device(local_rank)
return torch.device("cpu")
_WORLD: Optional[GroupCoordinator] = None
@@ -2173,15 +2179,17 @@ def init_model_parallel_group(
use_mscclpp = _ENABLE_MSCCLPP
if use_torch_symm_mem_allreduce is None:
use_torch_symm_mem_allreduce = _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE
if use_pynccl is None:
platform = platforms.current_platform
if platform.is_out_of_tree() and not platform.capabilities.supports_pynccl:
use_pynccl = False
else:
use_pynccl = not (_is_npu or _is_xpu or backend == "mooncake")
return GroupCoordinator(
group_ranks=group_ranks,
local_rank=local_rank,
torch_distributed_backend=backend,
use_pynccl=(
not (_is_npu or _is_xpu or backend == "mooncake")
if use_pynccl is None
else use_pynccl
),
use_pynccl=use_pynccl,
use_mscclpp=use_mscclpp,
use_custom_allreduce=use_custom_allreduce,
use_torch_symm_mem_all_reduce=use_torch_symm_mem_allreduce,
+1
View File
@@ -108,4 +108,5 @@ class CudaSRTPlatform(CudaDeviceMixin, SRTPlatform):
graph_capture=True,
piecewise_graph=True,
hicache_device_kernels=True,
supports_pynccl=True,
)
+19
View File
@@ -61,12 +61,16 @@ class PlatformCapabilities(msgspec.Struct, frozen=True, kw_only=True):
``hicache_device_kernels``: the sgl_kernel HiCache transfer / write-back
kernels are available; otherwise the host pools skip their staging
buffers.
``supports_pynccl``: model-parallel groups create a PyNccl communicator
when the caller does not choose. Read for out-of-tree platforms; in-tree
devices decide in ``parallel_state``.
"""
supports_triton: bool = False
graph_capture: bool = False
piecewise_graph: bool = False
hicache_device_kernels: bool = False
supports_pynccl: bool = False
class SRTPlatform(DeviceMixin):
@@ -208,6 +212,21 @@ class SRTPlatform(DeviceMixin):
"""Whether this out-of-tree platform supports Mamba cache snapshots."""
return False
# ------------------------------------------------------------------
# Distributed
# ------------------------------------------------------------------
def get_group_coordinator_device(self, local_rank: int) -> torch.device:
"""Return the device a ``GroupCoordinator`` binds to on this rank.
Consulted only for out-of-tree platforms; in-tree devices are resolved
in ``parallel_state``. The default is CPU, which is what an unknown
device got before this hook existed.
"""
import torch
return torch.device("cpu")
# ------------------------------------------------------------------
# Initialization
# ------------------------------------------------------------------
+3 -1
View File
@@ -22,4 +22,6 @@ class MusaDeviceMixin(CudaDeviceMixin):
class MusaSRTPlatform(MusaDeviceMixin, SRTPlatform):
"""Default in-tree MUSA SRT platform."""
capabilities = PlatformCapabilities(supports_triton=True, graph_capture=True)
capabilities = PlatformCapabilities(
supports_triton=True, graph_capture=True, supports_pynccl=True
)
+1
View File
@@ -28,4 +28,5 @@ class RocmSRTPlatform(RocmDeviceMixin, SRTPlatform):
graph_capture=True,
piecewise_graph=True,
hicache_device_kernels=True,
supports_pynccl=True,
)
+1
View File
@@ -26,4 +26,5 @@ class SailSRTPlatform(SailDeviceMixin, SRTPlatform):
supports_triton=True,
graph_capture=True,
piecewise_graph=True,
supports_pynccl=True,
)
+2 -1
View File
@@ -2988,8 +2988,9 @@ def direct_register_custom_op(
try:
my_lib.define(op_name + schema_str)
if is_npu():
if is_npu() or current_platform.is_out_of_tree():
# https://github.com/sgl-project/sglang/pull/12287/files#r2499583982
# Out-of-tree devices are torch PrivateUse1 backends too.
my_lib.impl(op_name, op_func, "PrivateUse1")
elif is_xpu():
my_lib.impl(op_name, op_func, "XPU")
@@ -0,0 +1,114 @@
import sys
from unittest.mock import Mock, patch
import pytest
import torch
from sglang.srt.platforms.interface import PlatformCapabilities, SRTPlatform
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
parallel_state = pytest.importorskip("sglang.srt.distributed.parallel_state")
def _platform(out_of_tree: bool, device: str = "cpu", pynccl: bool = False) -> Mock:
platform = Mock(spec=SRTPlatform)
platform.is_out_of_tree.return_value = out_of_tree
platform.get_group_coordinator_device.side_effect = lambda local_rank: torch.device(
device
)
platform.capabilities = PlatformCapabilities(supports_pynccl=pynccl)
return platform
def _no_in_tree_device():
return (
patch.object(parallel_state, "is_cuda_alike", return_value=False),
patch.object(parallel_state, "_is_npu", False),
patch.object(parallel_state, "_is_xpu", False),
patch.object(parallel_state, "_is_musa", False),
)
def _with_platform(platform):
return patch.object(
parallel_state.platforms, "current_platform", platform, create=True
)
def test_out_of_tree_platform_chooses_group_device():
platform = _platform(out_of_tree=True, device="meta")
p1, p2, p3, p4 = _no_in_tree_device()
with p1, p2, p3, p4, _with_platform(platform):
device = parallel_state._group_coordinator_device(3)
assert device == torch.device("meta")
platform.get_group_coordinator_device.assert_called_once_with(3)
def test_in_tree_unknown_device_falls_back_to_cpu():
platform = _platform(out_of_tree=False)
p1, p2, p3, p4 = _no_in_tree_device()
with p1, p2, p3, p4, _with_platform(platform):
device = parallel_state._group_coordinator_device(3)
assert device == torch.device("cpu")
platform.get_group_coordinator_device.assert_not_called()
def test_base_platform_defaults():
platform = SRTPlatform()
assert platform.get_group_coordinator_device(0) == torch.device("cpu")
assert platform.capabilities.supports_pynccl is False
@pytest.mark.parametrize(
"out_of_tree, platform_pynccl, explicit, expected",
[
(True, False, None, False),
(True, True, None, True),
(False, False, None, True),
(True, False, True, True),
(False, True, False, False),
],
)
def test_pynccl_default(out_of_tree, platform_pynccl, explicit, expected):
platform = _platform(out_of_tree=out_of_tree, pynccl=platform_pynccl)
with (
_with_platform(platform),
patch.object(parallel_state, "_is_npu", False),
patch.object(parallel_state, "_is_xpu", False),
patch.object(parallel_state, "GroupCoordinator") as coordinator,
):
parallel_state.init_model_parallel_group(
group_ranks=[[0, 1]],
local_rank=0,
backend="gloo",
use_pynccl=explicit,
)
assert coordinator.call_args.kwargs["use_pynccl"] is expected
@pytest.mark.parametrize("out_of_tree, key", [(True, "PrivateUse1"), (False, "CUDA")])
def test_custom_op_dispatch_key(out_of_tree, key):
from torch.library import Library
from sglang.srt.utils import common
lib_name = f"sglang_oot_test_{int(out_of_tree)}"
lib = Library(lib_name, "FRAGMENT")
def op(x: torch.Tensor) -> torch.Tensor:
return x.clone()
with (
patch.object(common, "current_platform", _platform(out_of_tree=out_of_tree)),
patch.object(common, "is_npu", return_value=False),
patch.object(common, "is_xpu", return_value=False),
patch.object(common, "is_musa", return_value=False),
):
common.direct_register_custom_op("op", op, mutates_args=[], target_lib=lib)
assert torch._C._dispatch_has_kernel_for_dispatch_key(f"{lib_name}::op", key)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-v"]))