Add 8-node AllReduce/AllGather and MNVLS algorithm support to MSCCL++ (#37442)

Co-authored-by: Caio Rocha <caiorocha@microsft.com>
This commit is contained in:
Caio Rocha
2026-09-24 15:51:29 -07:00
committed by GitHub
co-authored by Caio Rocha
parent 3ec86307f2
commit e047e50d4e
8 changed files with 1451 additions and 243 deletions
+1 -1
View File
@@ -606,7 +606,7 @@ class ExecComm(msgspec.Struct):
] = False
enable_mscclpp: A[
bool,
"Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL.",
"Enable MSCCL++ for tuned AllReduce and AllGather messages, with NCCL fallback.",
] = False
enable_torch_symm_mem: A[
bool,
+2 -2
View File
@@ -20,7 +20,7 @@ from sglang.srt.distributed.parallel_state import (
initialize_model_parallel,
set_custom_all_reduce,
set_flashinfer_allreduce_only,
set_mscclpp_all_reduce,
set_mscclpp,
set_torch_symm_mem_all_reduce,
)
from sglang.srt.environ import envs
@@ -234,7 +234,7 @@ def _resolve_dist_init_method(*, dist_port: int) -> str:
def _set_all_reduce_flags() -> None:
set_custom_all_reduce(not get_exec().comm.disable_custom_all_reduce)
set_mscclpp_all_reduce(get_exec().comm.enable_mscclpp)
set_mscclpp(get_exec().comm.enable_mscclpp)
set_torch_symm_mem_all_reduce(get_exec().comm.enable_torch_symm_mem)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
File diff suppressed because it is too large Load Diff
+21 -13
View File
@@ -267,7 +267,7 @@ class GroupCoordinator:
cpu_group: ProcessGroup # group for CPU communication
device_group: ProcessGroup # group for device communication
use_pynccl: bool # a hint of whether to use PyNccl
use_pymscclpp: bool # a hint of whether to use PyMsccl
use_mscclpp: bool # a hint of whether to use MSCCL++
use_custom_allreduce: bool # a hint of whether to use CustomAllreduce
use_torch_symm_mem_all_reduce: (
bool # a hint of whether to use TorchSymmMemAllReduce
@@ -287,7 +287,7 @@ class GroupCoordinator:
local_rank: int,
torch_distributed_backend: Union[str, Backend],
use_pynccl: bool,
use_pymscclpp: bool,
use_mscclpp: bool,
use_custom_allreduce: bool,
use_torch_symm_mem_all_reduce: bool,
use_hpu_communicator: bool,
@@ -424,7 +424,7 @@ class GroupCoordinator:
# Import communicators
self.use_pynccl = use_pynccl
self.use_pymscclpp = use_pymscclpp
self.use_mscclpp = use_mscclpp
self.use_custom_allreduce = use_custom_allreduce
self.use_torch_symm_mem_all_reduce = use_torch_symm_mem_all_reduce
self.use_hpu_communicator = use_hpu_communicator
@@ -471,10 +471,11 @@ class GroupCoordinator:
)
self.pymscclpp_comm: Optional[PyMscclppCommunicator] = None
if use_pymscclpp and self.world_size > 1:
if use_mscclpp and self.world_size > 1:
self.pymscclpp_comm = PyMscclppCommunicator(
group=self.cpu_group,
device=self.device,
group_name=group_name,
)
self.ca_comm: Optional[Any] = None
@@ -1285,6 +1286,13 @@ class GroupCoordinator:
ca_comm.all_gather_unreg(input, out=output, dim=0)
return
pymscclpp_comm = self.pymscclpp_comm
if pymscclpp_comm is not None and pymscclpp_comm.should_mscclpp_allgather(
output, input
):
pymscclpp_comm.all_gather(output, input)
return
pynccl_comm = self.pynccl_comm
if pynccl_comm is not None and (
not pynccl_comm.disabled or self.is_symmetric_memory_enabled()
@@ -2087,7 +2095,7 @@ def init_world_group(
local_rank=local_rank,
torch_distributed_backend=backend,
use_pynccl=False,
use_pymscclpp=False,
use_mscclpp=False,
use_custom_allreduce=False,
use_torch_symm_mem_all_reduce=False,
use_hpu_communicator=False,
@@ -2107,7 +2115,7 @@ def init_model_parallel_group(
use_custom_allreduce: Optional[bool] = None,
use_message_queue_broadcaster: bool = False,
group_name: Optional[str] = None,
use_mscclpp_allreduce: Optional[bool] = None,
use_mscclpp: Optional[bool] = None,
use_torch_symm_mem_allreduce: Optional[bool] = None,
recovered_rank: bool = False,
rank_offset: int = 0,
@@ -2115,8 +2123,8 @@ def init_model_parallel_group(
) -> GroupCoordinator:
if use_custom_allreduce is None:
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
if use_mscclpp_allreduce is None:
use_mscclpp_allreduce = _ENABLE_MSCCLPP_ALL_REDUCE
if use_mscclpp is None:
use_mscclpp = _ENABLE_MSCCLPP
if use_torch_symm_mem_allreduce is None:
use_torch_symm_mem_allreduce = _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE
return GroupCoordinator(
@@ -2128,7 +2136,7 @@ def init_model_parallel_group(
if use_pynccl is None
else use_pynccl
),
use_pymscclpp=use_mscclpp_allreduce,
use_mscclpp=use_mscclpp,
use_custom_allreduce=use_custom_allreduce,
use_torch_symm_mem_all_reduce=use_torch_symm_mem_allreduce,
use_hpu_communicator=True,
@@ -2270,7 +2278,7 @@ def graph_capture(stream=None):
logger = logging.getLogger(__name__)
_ENABLE_CUSTOM_ALL_REDUCE = True
_ENABLE_MSCCLPP_ALL_REDUCE = False
_ENABLE_MSCCLPP = False
_ENABLE_TORCH_SYMM_MEM_ALL_REDUCE = False
_ENABLE_FLASHINFER_ALLREDUCE_ONLY = False
@@ -2280,9 +2288,9 @@ def set_custom_all_reduce(enable: bool):
_ENABLE_CUSTOM_ALL_REDUCE = enable
def set_mscclpp_all_reduce(enable: bool):
global _ENABLE_MSCCLPP_ALL_REDUCE
_ENABLE_MSCCLPP_ALL_REDUCE = enable
def set_mscclpp(enable: bool):
global _ENABLE_MSCCLPP
_ENABLE_MSCCLPP = enable
def set_torch_symm_mem_all_reduce(enable: bool):