[Feat] platforms: Let out-of-tree platforms bind group devices and custom ops (#45897)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
1e894eb92d
commit
baf80c61b7
@@ -576,6 +576,11 @@ field defaults to the conservative answer.
|
||||
<td><code>False</code></td>
|
||||
<td>The sgl_kernel HiCache transfer / write-back kernels are available; otherwise host pools skip their staging buffers.</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>supports_pynccl</code></td>
|
||||
<td><code>False</code></td>
|
||||
<td>Model-parallel groups create a PyNccl communicator when the caller does not choose. Set it to True only if the device runs NCCL (or an NCCL-compatible library PyNccl loads).</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
@@ -630,9 +635,16 @@ field defaults to the conservative answer.
|
||||
<td><code>"native"</code></td>
|
||||
<td>BaseFusedOp (fused-op) dispatch key name</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><code>get_group_coordinator_device(local_rank)</code></td>
|
||||
<td><code>torch.device("cpu")</code></td>
|
||||
<td>Device each <code>GroupCoordinator</code> binds to on this rank (e.g. <code>torch.device(f"mydev:{local_rank}")</code>). Without it, device collectives on an out-of-tree platform run against a CPU device.</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
SGLang custom ops (e.g. the <code>inplace_all_reduce</code> collective) register their implementation under the <code>PrivateUse1</code> dispatch key on an out-of-tree platform, the key torch reserves for out-of-tree backends.
|
||||
|
||||
#### Lifecycle Hooks (from SRTPlatform)
|
||||
|
||||
<table style={{width: "100%", borderCollapse: "collapse", tableLayout: "fixed"}}>
|
||||
|
||||
@@ -331,19 +331,7 @@ class GroupCoordinator:
|
||||
# Scale cohorts create these groups without the serving ranks.
|
||||
use_local_synchronization = rank_offset > 0 and not recovered_rank
|
||||
|
||||
if is_cuda_alike():
|
||||
device_id = (
|
||||
0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank
|
||||
)
|
||||
self.device = torch.device(f"cuda:{device_id}")
|
||||
elif _is_npu:
|
||||
self.device = torch.device(f"npu:{local_rank}")
|
||||
elif _is_xpu:
|
||||
self.device = torch.device(f"xpu:{local_rank}")
|
||||
elif _is_musa:
|
||||
self.device = torch.device(f"musa:{local_rank}")
|
||||
else:
|
||||
self.device = torch.device("cpu")
|
||||
self.device = _group_coordinator_device(local_rank)
|
||||
self.device_module = torch.get_device_module(self.device)
|
||||
|
||||
for ranks in group_ranks:
|
||||
@@ -2122,6 +2110,24 @@ class GroupCoordinator:
|
||||
self.mq_broadcaster = None
|
||||
|
||||
|
||||
def _group_coordinator_device(local_rank: int) -> torch.device:
|
||||
if is_cuda_alike():
|
||||
device_id = (
|
||||
0 if envs.SGLANG_ONE_VISIBLE_DEVICE_PER_PROCESS.get() else local_rank
|
||||
)
|
||||
return torch.device(f"cuda:{device_id}")
|
||||
if _is_npu:
|
||||
return torch.device(f"npu:{local_rank}")
|
||||
if _is_xpu:
|
||||
return torch.device(f"xpu:{local_rank}")
|
||||
if _is_musa:
|
||||
return torch.device(f"musa:{local_rank}")
|
||||
platform = platforms.current_platform
|
||||
if platform.is_out_of_tree():
|
||||
return platform.get_group_coordinator_device(local_rank)
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
_WORLD: Optional[GroupCoordinator] = None
|
||||
|
||||
|
||||
@@ -2173,15 +2179,17 @@ def init_model_parallel_group(
|
||||
use_mscclpp = _ENABLE_MSCCLPP
|
||||
if use_torch_symm_mem_allreduce is None:
|
||||
use_torch_symm_mem_allreduce = _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE
|
||||
if use_pynccl is None:
|
||||
platform = platforms.current_platform
|
||||
if platform.is_out_of_tree() and not platform.capabilities.supports_pynccl:
|
||||
use_pynccl = False
|
||||
else:
|
||||
use_pynccl = not (_is_npu or _is_xpu or backend == "mooncake")
|
||||
return GroupCoordinator(
|
||||
group_ranks=group_ranks,
|
||||
local_rank=local_rank,
|
||||
torch_distributed_backend=backend,
|
||||
use_pynccl=(
|
||||
not (_is_npu or _is_xpu or backend == "mooncake")
|
||||
if use_pynccl is None
|
||||
else use_pynccl
|
||||
),
|
||||
use_pynccl=use_pynccl,
|
||||
use_mscclpp=use_mscclpp,
|
||||
use_custom_allreduce=use_custom_allreduce,
|
||||
use_torch_symm_mem_all_reduce=use_torch_symm_mem_allreduce,
|
||||
|
||||
@@ -108,4 +108,5 @@ class CudaSRTPlatform(CudaDeviceMixin, SRTPlatform):
|
||||
graph_capture=True,
|
||||
piecewise_graph=True,
|
||||
hicache_device_kernels=True,
|
||||
supports_pynccl=True,
|
||||
)
|
||||
|
||||
@@ -61,12 +61,16 @@ class PlatformCapabilities(msgspec.Struct, frozen=True, kw_only=True):
|
||||
``hicache_device_kernels``: the sgl_kernel HiCache transfer / write-back
|
||||
kernels are available; otherwise the host pools skip their staging
|
||||
buffers.
|
||||
``supports_pynccl``: model-parallel groups create a PyNccl communicator
|
||||
when the caller does not choose. Read for out-of-tree platforms; in-tree
|
||||
devices decide in ``parallel_state``.
|
||||
"""
|
||||
|
||||
supports_triton: bool = False
|
||||
graph_capture: bool = False
|
||||
piecewise_graph: bool = False
|
||||
hicache_device_kernels: bool = False
|
||||
supports_pynccl: bool = False
|
||||
|
||||
|
||||
class SRTPlatform(DeviceMixin):
|
||||
@@ -208,6 +212,21 @@ class SRTPlatform(DeviceMixin):
|
||||
"""Whether this out-of-tree platform supports Mamba cache snapshots."""
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Distributed
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def get_group_coordinator_device(self, local_rank: int) -> torch.device:
|
||||
"""Return the device a ``GroupCoordinator`` binds to on this rank.
|
||||
|
||||
Consulted only for out-of-tree platforms; in-tree devices are resolved
|
||||
in ``parallel_state``. The default is CPU, which is what an unknown
|
||||
device got before this hook existed.
|
||||
"""
|
||||
import torch
|
||||
|
||||
return torch.device("cpu")
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Initialization
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@@ -22,4 +22,6 @@ class MusaDeviceMixin(CudaDeviceMixin):
|
||||
class MusaSRTPlatform(MusaDeviceMixin, SRTPlatform):
|
||||
"""Default in-tree MUSA SRT platform."""
|
||||
|
||||
capabilities = PlatformCapabilities(supports_triton=True, graph_capture=True)
|
||||
capabilities = PlatformCapabilities(
|
||||
supports_triton=True, graph_capture=True, supports_pynccl=True
|
||||
)
|
||||
|
||||
@@ -28,4 +28,5 @@ class RocmSRTPlatform(RocmDeviceMixin, SRTPlatform):
|
||||
graph_capture=True,
|
||||
piecewise_graph=True,
|
||||
hicache_device_kernels=True,
|
||||
supports_pynccl=True,
|
||||
)
|
||||
|
||||
@@ -26,4 +26,5 @@ class SailSRTPlatform(SailDeviceMixin, SRTPlatform):
|
||||
supports_triton=True,
|
||||
graph_capture=True,
|
||||
piecewise_graph=True,
|
||||
supports_pynccl=True,
|
||||
)
|
||||
|
||||
@@ -2988,8 +2988,9 @@ def direct_register_custom_op(
|
||||
|
||||
try:
|
||||
my_lib.define(op_name + schema_str)
|
||||
if is_npu():
|
||||
if is_npu() or current_platform.is_out_of_tree():
|
||||
# https://github.com/sgl-project/sglang/pull/12287/files#r2499583982
|
||||
# Out-of-tree devices are torch PrivateUse1 backends too.
|
||||
my_lib.impl(op_name, op_func, "PrivateUse1")
|
||||
elif is_xpu():
|
||||
my_lib.impl(op_name, op_func, "XPU")
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import sys
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.srt.platforms.interface import PlatformCapabilities, SRTPlatform
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
||||
|
||||
parallel_state = pytest.importorskip("sglang.srt.distributed.parallel_state")
|
||||
|
||||
|
||||
def _platform(out_of_tree: bool, device: str = "cpu", pynccl: bool = False) -> Mock:
|
||||
platform = Mock(spec=SRTPlatform)
|
||||
platform.is_out_of_tree.return_value = out_of_tree
|
||||
platform.get_group_coordinator_device.side_effect = lambda local_rank: torch.device(
|
||||
device
|
||||
)
|
||||
platform.capabilities = PlatformCapabilities(supports_pynccl=pynccl)
|
||||
return platform
|
||||
|
||||
|
||||
def _no_in_tree_device():
|
||||
return (
|
||||
patch.object(parallel_state, "is_cuda_alike", return_value=False),
|
||||
patch.object(parallel_state, "_is_npu", False),
|
||||
patch.object(parallel_state, "_is_xpu", False),
|
||||
patch.object(parallel_state, "_is_musa", False),
|
||||
)
|
||||
|
||||
|
||||
def _with_platform(platform):
|
||||
return patch.object(
|
||||
parallel_state.platforms, "current_platform", platform, create=True
|
||||
)
|
||||
|
||||
|
||||
def test_out_of_tree_platform_chooses_group_device():
|
||||
platform = _platform(out_of_tree=True, device="meta")
|
||||
p1, p2, p3, p4 = _no_in_tree_device()
|
||||
with p1, p2, p3, p4, _with_platform(platform):
|
||||
device = parallel_state._group_coordinator_device(3)
|
||||
assert device == torch.device("meta")
|
||||
platform.get_group_coordinator_device.assert_called_once_with(3)
|
||||
|
||||
|
||||
def test_in_tree_unknown_device_falls_back_to_cpu():
|
||||
platform = _platform(out_of_tree=False)
|
||||
p1, p2, p3, p4 = _no_in_tree_device()
|
||||
with p1, p2, p3, p4, _with_platform(platform):
|
||||
device = parallel_state._group_coordinator_device(3)
|
||||
assert device == torch.device("cpu")
|
||||
platform.get_group_coordinator_device.assert_not_called()
|
||||
|
||||
|
||||
def test_base_platform_defaults():
|
||||
platform = SRTPlatform()
|
||||
assert platform.get_group_coordinator_device(0) == torch.device("cpu")
|
||||
assert platform.capabilities.supports_pynccl is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"out_of_tree, platform_pynccl, explicit, expected",
|
||||
[
|
||||
(True, False, None, False),
|
||||
(True, True, None, True),
|
||||
(False, False, None, True),
|
||||
(True, False, True, True),
|
||||
(False, True, False, False),
|
||||
],
|
||||
)
|
||||
def test_pynccl_default(out_of_tree, platform_pynccl, explicit, expected):
|
||||
platform = _platform(out_of_tree=out_of_tree, pynccl=platform_pynccl)
|
||||
with (
|
||||
_with_platform(platform),
|
||||
patch.object(parallel_state, "_is_npu", False),
|
||||
patch.object(parallel_state, "_is_xpu", False),
|
||||
patch.object(parallel_state, "GroupCoordinator") as coordinator,
|
||||
):
|
||||
parallel_state.init_model_parallel_group(
|
||||
group_ranks=[[0, 1]],
|
||||
local_rank=0,
|
||||
backend="gloo",
|
||||
use_pynccl=explicit,
|
||||
)
|
||||
assert coordinator.call_args.kwargs["use_pynccl"] is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("out_of_tree, key", [(True, "PrivateUse1"), (False, "CUDA")])
|
||||
def test_custom_op_dispatch_key(out_of_tree, key):
|
||||
from torch.library import Library
|
||||
|
||||
from sglang.srt.utils import common
|
||||
|
||||
lib_name = f"sglang_oot_test_{int(out_of_tree)}"
|
||||
lib = Library(lib_name, "FRAGMENT")
|
||||
|
||||
def op(x: torch.Tensor) -> torch.Tensor:
|
||||
return x.clone()
|
||||
|
||||
with (
|
||||
patch.object(common, "current_platform", _platform(out_of_tree=out_of_tree)),
|
||||
patch.object(common, "is_npu", return_value=False),
|
||||
patch.object(common, "is_xpu", return_value=False),
|
||||
patch.object(common, "is_musa", return_value=False),
|
||||
):
|
||||
common.direct_register_custom_op("op", op, mutates_args=[], target_lib=lib)
|
||||
assert torch._C._dispatch_has_kernel_for_dispatch_key(f"{lib_name}::op", key)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, "-v"]))
|
||||
Reference in New Issue
Block a user