Add 8-node AllReduce/AllGather and MNVLS algorithm support to MSCCL++ (#37442)
Co-authored-by: Caio Rocha <caiorocha@microsft.com>
This commit is contained in:
@@ -606,7 +606,7 @@ class ExecComm(msgspec.Struct):
|
||||
] = False
|
||||
enable_mscclpp: A[
|
||||
bool,
|
||||
"Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL.",
|
||||
"Enable MSCCL++ for tuned AllReduce and AllGather messages, with NCCL fallback.",
|
||||
] = False
|
||||
enable_torch_symm_mem: A[
|
||||
bool,
|
||||
|
||||
@@ -20,7 +20,7 @@ from sglang.srt.distributed.parallel_state import (
|
||||
initialize_model_parallel,
|
||||
set_custom_all_reduce,
|
||||
set_flashinfer_allreduce_only,
|
||||
set_mscclpp_all_reduce,
|
||||
set_mscclpp,
|
||||
set_torch_symm_mem_all_reduce,
|
||||
)
|
||||
from sglang.srt.environ import envs
|
||||
@@ -234,7 +234,7 @@ def _resolve_dist_init_method(*, dist_port: int) -> str:
|
||||
|
||||
def _set_all_reduce_flags() -> None:
|
||||
set_custom_all_reduce(not get_exec().comm.disable_custom_all_reduce)
|
||||
set_mscclpp_all_reduce(get_exec().comm.enable_mscclpp)
|
||||
set_mscclpp(get_exec().comm.enable_mscclpp)
|
||||
set_torch_symm_mem_all_reduce(get_exec().comm.enable_torch_symm_mem)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -267,7 +267,7 @@ class GroupCoordinator:
|
||||
cpu_group: ProcessGroup # group for CPU communication
|
||||
device_group: ProcessGroup # group for device communication
|
||||
use_pynccl: bool # a hint of whether to use PyNccl
|
||||
use_pymscclpp: bool # a hint of whether to use PyMsccl
|
||||
use_mscclpp: bool # a hint of whether to use MSCCL++
|
||||
use_custom_allreduce: bool # a hint of whether to use CustomAllreduce
|
||||
use_torch_symm_mem_all_reduce: (
|
||||
bool # a hint of whether to use TorchSymmMemAllReduce
|
||||
@@ -287,7 +287,7 @@ class GroupCoordinator:
|
||||
local_rank: int,
|
||||
torch_distributed_backend: Union[str, Backend],
|
||||
use_pynccl: bool,
|
||||
use_pymscclpp: bool,
|
||||
use_mscclpp: bool,
|
||||
use_custom_allreduce: bool,
|
||||
use_torch_symm_mem_all_reduce: bool,
|
||||
use_hpu_communicator: bool,
|
||||
@@ -424,7 +424,7 @@ class GroupCoordinator:
|
||||
|
||||
# Import communicators
|
||||
self.use_pynccl = use_pynccl
|
||||
self.use_pymscclpp = use_pymscclpp
|
||||
self.use_mscclpp = use_mscclpp
|
||||
self.use_custom_allreduce = use_custom_allreduce
|
||||
self.use_torch_symm_mem_all_reduce = use_torch_symm_mem_all_reduce
|
||||
self.use_hpu_communicator = use_hpu_communicator
|
||||
@@ -471,10 +471,11 @@ class GroupCoordinator:
|
||||
)
|
||||
|
||||
self.pymscclpp_comm: Optional[PyMscclppCommunicator] = None
|
||||
if use_pymscclpp and self.world_size > 1:
|
||||
if use_mscclpp and self.world_size > 1:
|
||||
self.pymscclpp_comm = PyMscclppCommunicator(
|
||||
group=self.cpu_group,
|
||||
device=self.device,
|
||||
group_name=group_name,
|
||||
)
|
||||
|
||||
self.ca_comm: Optional[Any] = None
|
||||
@@ -1285,6 +1286,13 @@ class GroupCoordinator:
|
||||
ca_comm.all_gather_unreg(input, out=output, dim=0)
|
||||
return
|
||||
|
||||
pymscclpp_comm = self.pymscclpp_comm
|
||||
if pymscclpp_comm is not None and pymscclpp_comm.should_mscclpp_allgather(
|
||||
output, input
|
||||
):
|
||||
pymscclpp_comm.all_gather(output, input)
|
||||
return
|
||||
|
||||
pynccl_comm = self.pynccl_comm
|
||||
if pynccl_comm is not None and (
|
||||
not pynccl_comm.disabled or self.is_symmetric_memory_enabled()
|
||||
@@ -2087,7 +2095,7 @@ def init_world_group(
|
||||
local_rank=local_rank,
|
||||
torch_distributed_backend=backend,
|
||||
use_pynccl=False,
|
||||
use_pymscclpp=False,
|
||||
use_mscclpp=False,
|
||||
use_custom_allreduce=False,
|
||||
use_torch_symm_mem_all_reduce=False,
|
||||
use_hpu_communicator=False,
|
||||
@@ -2107,7 +2115,7 @@ def init_model_parallel_group(
|
||||
use_custom_allreduce: Optional[bool] = None,
|
||||
use_message_queue_broadcaster: bool = False,
|
||||
group_name: Optional[str] = None,
|
||||
use_mscclpp_allreduce: Optional[bool] = None,
|
||||
use_mscclpp: Optional[bool] = None,
|
||||
use_torch_symm_mem_allreduce: Optional[bool] = None,
|
||||
recovered_rank: bool = False,
|
||||
rank_offset: int = 0,
|
||||
@@ -2115,8 +2123,8 @@ def init_model_parallel_group(
|
||||
) -> GroupCoordinator:
|
||||
if use_custom_allreduce is None:
|
||||
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
|
||||
if use_mscclpp_allreduce is None:
|
||||
use_mscclpp_allreduce = _ENABLE_MSCCLPP_ALL_REDUCE
|
||||
if use_mscclpp is None:
|
||||
use_mscclpp = _ENABLE_MSCCLPP
|
||||
if use_torch_symm_mem_allreduce is None:
|
||||
use_torch_symm_mem_allreduce = _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE
|
||||
return GroupCoordinator(
|
||||
@@ -2128,7 +2136,7 @@ def init_model_parallel_group(
|
||||
if use_pynccl is None
|
||||
else use_pynccl
|
||||
),
|
||||
use_pymscclpp=use_mscclpp_allreduce,
|
||||
use_mscclpp=use_mscclpp,
|
||||
use_custom_allreduce=use_custom_allreduce,
|
||||
use_torch_symm_mem_all_reduce=use_torch_symm_mem_allreduce,
|
||||
use_hpu_communicator=True,
|
||||
@@ -2270,7 +2278,7 @@ def graph_capture(stream=None):
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ENABLE_CUSTOM_ALL_REDUCE = True
|
||||
_ENABLE_MSCCLPP_ALL_REDUCE = False
|
||||
_ENABLE_MSCCLPP = False
|
||||
_ENABLE_TORCH_SYMM_MEM_ALL_REDUCE = False
|
||||
_ENABLE_FLASHINFER_ALLREDUCE_ONLY = False
|
||||
|
||||
@@ -2280,9 +2288,9 @@ def set_custom_all_reduce(enable: bool):
|
||||
_ENABLE_CUSTOM_ALL_REDUCE = enable
|
||||
|
||||
|
||||
def set_mscclpp_all_reduce(enable: bool):
|
||||
global _ENABLE_MSCCLPP_ALL_REDUCE
|
||||
_ENABLE_MSCCLPP_ALL_REDUCE = enable
|
||||
def set_mscclpp(enable: bool):
|
||||
global _ENABLE_MSCCLPP
|
||||
_ENABLE_MSCCLPP = enable
|
||||
|
||||
|
||||
def set_torch_symm_mem_all_reduce(enable: bool):
|
||||
|
||||
Reference in New Issue
Block a user