[Feat] platforms: Let out-of-tree platforms bind group devices and custom ops (#45897)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alex Nails
2026-10-23 12:02:42 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 1e894eb92d
commit baf80c61b7
9 changed files with 179 additions and 20 deletions
+26 -18
View File
@@ -331,19 +331,7 @@ class GroupCoordinator:
# Scale cohorts create these groups without the serving ranks.
use_local_synchronization = rank_offset > 0 and not recovered_rank
if is_cuda_alike():
device_id = (
0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank
)
self.device = torch.device(f"cuda:{device_id}")
elif _is_npu:
self.device = torch.device(f"npu:{local_rank}")
elif _is_xpu:
self.device = torch.device(f"xpu:{local_rank}")
elif _is_musa:
self.device = torch.device(f"musa:{local_rank}")
else:
self.device = torch.device("cpu")
self.device = _group_coordinator_device(local_rank)
self.device_module = torch.get_device_module(self.device)
for ranks in group_ranks:
@@ -2122,6 +2110,24 @@ class GroupCoordinator:
self.mq_broadcaster = None
def _group_coordinator_device(local_rank: int) -> torch.device:
if is_cuda_alike():
device_id = (
0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank
)
return torch.device(f"cuda:{device_id}")
if _is_npu:
return torch.device(f"npu:{local_rank}")
if _is_xpu:
return torch.device(f"xpu:{local_rank}")
if _is_musa:
return torch.device(f"musa:{local_rank}")
platform = platforms.current_platform
if platform.is_out_of_tree():
return platform.get_group_coordinator_device(local_rank)
return torch.device("cpu")
_WORLD: Optional[GroupCoordinator] = None
@@ -2173,15 +2179,17 @@ def init_model_parallel_group(
use_mscclpp = _ENABLE_MSCCLPP
if use_torch_symm_mem_allreduce is None:
use_torch_symm_mem_allreduce = _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE
if use_pynccl is None:
platform = platforms.current_platform
if platform.is_out_of_tree() and not platform.capabilities.supports_pynccl:
use_pynccl = False
else:
use_pynccl = not (_is_npu or _is_xpu or backend == "mooncake")
return GroupCoordinator(
group_ranks=group_ranks,
local_rank=local_rank,
torch_distributed_backend=backend,
use_pynccl=(
not (_is_npu or _is_xpu or backend == "mooncake")
if use_pynccl is None
else use_pynccl
),
use_pynccl=use_pynccl,
use_mscclpp=use_mscclpp,
use_custom_allreduce=use_custom_allreduce,
use_torch_symm_mem_all_reduce=use_torch_symm_mem_allreduce,
+1
View File
@@ -108,4 +108,5 @@ class CudaSRTPlatform(CudaDeviceMixin, SRTPlatform):
graph_capture=True,
piecewise_graph=True,
hicache_device_kernels=True,
supports_pynccl=True,
)
+19
View File
@@ -61,12 +61,16 @@ class PlatformCapabilities(msgspec.Struct, frozen=True, kw_only=True):
``hicache_device_kernels``: the sgl_kernel HiCache transfer / write-back
kernels are available; otherwise the host pools skip their staging
buffers.
``supports_pynccl``: model-parallel groups create a PyNccl communicator
when the caller does not choose. Read for out-of-tree platforms; in-tree
devices decide in ``parallel_state``.
"""
supports_triton: bool = False
graph_capture: bool = False
piecewise_graph: bool = False
hicache_device_kernels: bool = False
supports_pynccl: bool = False
class SRTPlatform(DeviceMixin):
@@ -208,6 +212,21 @@ class SRTPlatform(DeviceMixin):
"""Whether this out-of-tree platform supports Mamba cache snapshots."""
return False
# ------------------------------------------------------------------
# Distributed
# ------------------------------------------------------------------
def get_group_coordinator_device(self, local_rank: int) -> torch.device:
"""Return the device a ``GroupCoordinator`` binds to on this rank.
Consulted only for out-of-tree platforms; in-tree devices are resolved
in ``parallel_state``. The default is CPU, which is what an unknown
device got before this hook existed.
"""
import torch
return torch.device("cpu")
# ------------------------------------------------------------------
# Initialization
# ------------------------------------------------------------------
+3 -1
View File
@@ -22,4 +22,6 @@ class MusaDeviceMixin(CudaDeviceMixin):
class MusaSRTPlatform(MusaDeviceMixin, SRTPlatform):
"""Default in-tree MUSA SRT platform."""
capabilities = PlatformCapabilities(supports_triton=True, graph_capture=True)
capabilities = PlatformCapabilities(
supports_triton=True, graph_capture=True, supports_pynccl=True
)
+1
View File
@@ -28,4 +28,5 @@ class RocmSRTPlatform(RocmDeviceMixin, SRTPlatform):
graph_capture=True,
piecewise_graph=True,
hicache_device_kernels=True,
supports_pynccl=True,
)
+1
View File
@@ -26,4 +26,5 @@ class SailSRTPlatform(SailDeviceMixin, SRTPlatform):
supports_triton=True,
graph_capture=True,
piecewise_graph=True,
supports_pynccl=True,
)
+2 -1
View File
@@ -2988,8 +2988,9 @@ def direct_register_custom_op(
try:
my_lib.define(op_name + schema_str)
if is_npu():
if is_npu() or current_platform.is_out_of_tree():
# https://github.com/sgl-project/sglang/pull/12287/files#r2499583982
# Out-of-tree devices are torch PrivateUse1 backends too.
my_lib.impl(op_name, op_func, "PrivateUse1")
elif is_xpu():
my_lib.impl(op_name, op_func, "XPU")