[Feat] platforms: Let out-of-tree platforms bind group devices and custom ops (#45897)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1e894eb92d
commit
baf80c61b7
@@ -331,19 +331,7 @@ class GroupCoordinator:
|
||||
# Scale cohorts create these groups without the serving ranks.
|
||||
use_local_synchronization = rank_offset > 0 and not recovered_rank
|
||||
|
||||
if is_cuda_alike():
|
||||
device_id = (
|
||||
0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank
|
||||
)
|
||||
self.device = torch.device(f"cuda:{device_id}")
|
||||
elif _is_npu:
|
||||
self.device = torch.device(f"npu:{local_rank}")
|
||||
elif _is_xpu:
|
||||
self.device = torch.device(f"xpu:{local_rank}")
|
||||
elif _is_musa:
|
||||
self.device = torch.device(f"musa:{local_rank}")
|
||||
else:
|
||||
self.device = torch.device("cpu")
|
||||
self.device = _group_coordinator_device(local_rank)
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
|
||||
for ranks in group_ranks:
|
||||
@@ -2122,6 +2110,24 @@ class GroupCoordinator:
|
||||
self.mq_broadcaster = None
|
||||
|
||||
|
||||
def _group_coordinator_device(local_rank: int) -> torch.device:
|
||||
if is_cuda_alike():
|
||||
device_id = (
|
||||
0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank
|
||||
)
|
||||
return torch.device(f"cuda:{device_id}")
|
||||
if _is_npu:
|
||||
return torch.device(f"npu:{local_rank}")
|
||||
if _is_xpu:
|
||||
return torch.device(f"xpu:{local_rank}")
|
||||
if _is_musa:
|
||||
return torch.device(f"musa:{local_rank}")
|
||||
platform = platforms.current_platform
|
||||
if platform.is_out_of_tree():
|
||||
return platform.get_group_coordinator_device(local_rank)
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
_WORLD: Optional[GroupCoordinator] = None
|
||||
|
||||
|
||||
@@ -2173,15 +2179,17 @@ def init_model_parallel_group(
|
||||
use_mscclpp = _ENABLE_MSCCLPP
|
||||
if use_torch_symm_mem_allreduce is None:
|
||||
use_torch_symm_mem_allreduce = _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE
|
||||
if use_pynccl is None:
|
||||
platform = platforms.current_platform
|
||||
if platform.is_out_of_tree() and not platform.capabilities.supports_pynccl:
|
||||
use_pynccl = False
|
||||
else:
|
||||
use_pynccl = not (_is_npu or _is_xpu or backend == "mooncake")
|
||||
return GroupCoordinator(
|
||||
group_ranks=group_ranks,
|
||||
local_rank=local_rank,
|
||||
torch_distributed_backend=backend,
|
||||
use_pynccl=(
|
||||
not (_is_npu or _is_xpu or backend == "mooncake")
|
||||
if use_pynccl is None
|
||||
else use_pynccl
|
||||
),
|
||||
use_pynccl=use_pynccl,
|
||||
use_mscclpp=use_mscclpp,
|
||||
use_custom_allreduce=use_custom_allreduce,
|
||||
use_torch_symm_mem_all_reduce=use_torch_symm_mem_allreduce,
|
||||
|
||||
@@ -108,4 +108,5 @@ class CudaSRTPlatform(CudaDeviceMixin, SRTPlatform):
|
||||
graph_capture=True,
|
||||
piecewise_graph=True,
|
||||
hicache_device_kernels=True,
|
||||
supports_pynccl=True,
|
||||
)
|
||||
|
||||
@@ -61,12 +61,16 @@ class PlatformCapabilities(msgspec.Struct, frozen=True, kw_only=True):
|
||||
``hicache_device_kernels``: the sgl_kernel HiCache transfer / write-back
|
||||
kernels are available; otherwise the host pools skip their staging
|
||||
buffers.
|
||||
``supports_pynccl``: model-parallel groups create a PyNccl communicator
|
||||
when the caller does not choose. Read for out-of-tree platforms; in-tree
|
||||
devices decide in ``parallel_state``.
|
||||
"""
|
||||
|
||||
supports_triton: bool = False
|
||||
graph_capture: bool = False
|
||||
piecewise_graph: bool = False
|
||||
hicache_device_kernels: bool = False
|
||||
supports_pynccl: bool = False
|
||||
|
||||
|
||||
class SRTPlatform(DeviceMixin):
|
||||
@@ -208,6 +212,21 @@ class SRTPlatform(DeviceMixin):
|
||||
"""Whether this out-of-tree platform supports Mamba cache snapshots."""
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Distributed
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_group_coordinator_device(self, local_rank: int) -> torch.device:
|
||||
"""Return the device a ``GroupCoordinator`` binds to on this rank.
|
||||
|
||||
Consulted only for out-of-tree platforms; in-tree devices are resolved
|
||||
in ``parallel_state``. The default is CPU, which is what an unknown
|
||||
device got before this hook existed.
|
||||
"""
|
||||
import torch
|
||||
|
||||
return torch.device("cpu")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Initialization
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -22,4 +22,6 @@ class MusaDeviceMixin(CudaDeviceMixin):
|
||||
class MusaSRTPlatform(MusaDeviceMixin, SRTPlatform):
|
||||
"""Default in-tree MUSA SRT platform."""
|
||||
|
||||
capabilities = PlatformCapabilities(supports_triton=True, graph_capture=True)
|
||||
capabilities = PlatformCapabilities(
|
||||
supports_triton=True, graph_capture=True, supports_pynccl=True
|
||||
)
|
||||
|
||||
@@ -28,4 +28,5 @@ class RocmSRTPlatform(RocmDeviceMixin, SRTPlatform):
|
||||
graph_capture=True,
|
||||
piecewise_graph=True,
|
||||
hicache_device_kernels=True,
|
||||
supports_pynccl=True,
|
||||
)
|
||||
|
||||
@@ -26,4 +26,5 @@ class SailSRTPlatform(SailDeviceMixin, SRTPlatform):
|
||||
supports_triton=True,
|
||||
graph_capture=True,
|
||||
piecewise_graph=True,
|
||||
supports_pynccl=True,
|
||||
)
|
||||
|
||||
@@ -2988,8 +2988,9 @@ def direct_register_custom_op(
|
||||
|
||||
try:
|
||||
my_lib.define(op_name + schema_str)
|
||||
if is_npu():
|
||||
if is_npu() or current_platform.is_out_of_tree():
|
||||
# https://github.com/sgl-project/sglang/pull/12287/files#r2499583982
|
||||
# Out-of-tree devices are torch PrivateUse1 backends too.
|
||||
my_lib.impl(op_name, op_func, "PrivateUse1")
|
||||
elif is_xpu():
|
||||
my_lib.impl(op_name, op_func, "XPU")
|
||||
|
||||
Reference in New Issue
Block a user