diff --git a/benchmark/kernels/all_gather/benchmark_mscclpp.py b/benchmark/kernels/all_gather/benchmark_mscclpp.py new file mode 100644 index 0000000000..1e902ca0d9 --- /dev/null +++ b/benchmark/kernels/all_gather/benchmark_mscclpp.py @@ -0,0 +1,284 @@ +"""Benchmark multi-node MSCCL++ AllGather against NCCL.""" + +import argparse +import os + +import torch +import torch.distributed as dist + +from sglang.srt.distributed import init_distributed_environment +from sglang.srt.distributed.parallel_state import ( + cleanup_dist_env_and_memory, + get_tensor_model_parallel_group, + initialize_model_parallel, + set_mscclpp, +) +from sglang.test.test_utils import publish_build_topology + + +def _max_across_ranks(value: float, group) -> float: + tensor = torch.tensor([value], dtype=torch.float64) + dist.all_reduce(tensor, op=dist.ReduceOp.MAX, group=group) + return tensor.item() + + +def _time_eager(func, group, warmup: int = 2, repeat: int = 10) -> float: + for _ in range(warmup): + func() + torch.cuda.synchronize() + dist.barrier(group=group) + + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record() + for _ in range(repeat): + func() + end.record() + end.synchronize() + return _max_across_ranks(start.elapsed_time(end) * 1000 / repeat, group) + + +def _time_cuda_graph( + func, + group, + warmup: int = 100, + graph_loop: int = 10, + repeat: int = 100, +) -> float: + current_stream = torch.cuda.current_stream() + capture_stream = torch.cuda.Stream() + capture_stream.wait_stream(current_stream) + with torch.cuda.stream(capture_stream): + func(capture_stream) + capture_stream.synchronize() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=capture_stream): + for _ in range(graph_loop): + func(capture_stream) + + with torch.cuda.stream(capture_stream): + for _ in range(warmup): + graph.replay() + capture_stream.synchronize() + + dist.barrier(group=group) + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + start.record(capture_stream) + with torch.cuda.stream(capture_stream): + for _ in range(repeat): + graph.replay() + end.record(capture_stream) + end.synchronize() + torch.cuda.current_stream().wait_stream(capture_stream) + return _max_across_ranks( + start.elapsed_time(end) * 1000 / (repeat * graph_loop), + group, + ) + + +def _validate_cuda_graph_updates( + func, + reference_func, + input_tensor: torch.Tensor, + actual: torch.Tensor, + expected: torch.Tensor, + group, +) -> None: + current_stream = torch.cuda.current_stream() + capture_stream = torch.cuda.Stream() + capture_stream.wait_stream(current_stream) + with torch.cuda.stream(capture_stream): + func(capture_stream) + capture_stream.synchronize() + + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=capture_stream): + func(capture_stream) + + input_tensor.neg_() + reference_func() + current_stream.synchronize() + capture_stream.wait_stream(current_stream) + with torch.cuda.stream(capture_stream): + graph.replay() + capture_stream.synchronize() + + if not torch.equal(actual, expected): + raise AssertionError("MSCCL++ CUDA graph replay used stale AllGather data") + dist.barrier(group=group) + + +def _message_sizes(min_bytes: int, max_bytes: int): + size = 1 << (min_bytes - 1).bit_length() + while size <= max_bytes: + yield size + size <<= 1 + + +def human_readable_size(size, decimal_places=1): + for unit in ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]: + if size < 1024.0 or unit == "PiB": + break + size /= 1024.0 + return f"{size:.{decimal_places}f} {unit}" + + +try: + from tabulate import tabulate +except ImportError: + print("tabulate not installed, skipping table printing") + tabulate = None + + +def print_markdown_table(data): + if tabulate is not None: + print(tabulate(data, headers="keys", tablefmt="github")) + return + headers = data[0].keys() + header_row = "| " + " | ".join(headers) + " |" + separator = "| " + " | ".join(["---"] * len(headers)) + " |" + rows = [] + for item in data: + row = "| " + " | ".join(str(item[key]) for key in headers) + " |" + rows.append(row) + markdown_table = "\n".join([header_row, separator] + rows) + print(markdown_table) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--min-bytes", type=int, default=1 << 10) + parser.add_argument("--max-bytes", type=int, default=8 << 20) + args = parser.parse_args() + if not (1 << 10) <= args.min_bytes <= args.max_bytes <= (8 << 20): + parser.error("Require 1 KiB <= min-bytes <= max-bytes <= 8 MiB") + + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + if not dist.is_initialized(): + dist.init_process_group( + backend="nccl", + device_id=torch.device("cuda", local_rank), + ) + world_size = dist.get_world_size() + rank = dist.get_rank() + device = torch.cuda.current_device() + set_mscclpp(True) + init_distributed_environment( + world_size=world_size, + rank=rank, + local_rank=local_rank, + ) + publish_build_topology(world_rank=rank, tp_size=world_size) + initialize_model_parallel() + tp_group = get_tensor_model_parallel_group() + gpu_group = tp_group.device_group + cpu_group = tp_group.cpu_group + pynccl_comm = tp_group.pynccl_comm + mscclpp_comm = tp_group.pymscclpp_comm + assert pynccl_comm is not None + assert mscclpp_comm is not None + if args.min_bytes % world_size != 0 or args.max_bytes % world_size != 0: + parser.error("Total message sizes must be divisible by WORLD_SIZE") + + if not mscclpp_comm._best_configs["allgather"]: + raise RuntimeError("No tuned MSCCL++ AllGather configuration is available") + + result = [] + + for total_bytes in _message_sizes(args.min_bytes, args.max_bytes): + input_bytes = total_bytes // world_size + if input_bytes % torch.bfloat16.itemsize != 0: + continue + elements = input_bytes // torch.bfloat16.itemsize + input_tensor = ( + torch.arange( + elements, + dtype=torch.long, + device=device, + ) + % 193 + + rank + ).to(torch.bfloat16) + nccl_output = torch.empty( + elements * world_size, + dtype=input_tensor.dtype, + device=device, + ) + mscclpp_output = torch.empty_like(nccl_output) + + def torch_nccl(): + dist.all_gather_into_tensor( + nccl_output, + input_tensor, + group=gpu_group, + ) + + def py_nccl(): + pynccl_comm.all_gather(nccl_output, input_tensor) + + def mscclpp(): + mscclpp_comm.all_gather(mscclpp_output, input_tensor) + + def mscclpp_on_stream(stream): + mscclpp_comm.all_gather( + mscclpp_output, + input_tensor, + stream=stream, + ) + + def py_nccl_on_stream(_stream): + with pynccl_comm.change_state(enable=True): + pynccl_comm.all_gather(nccl_output, input_tensor) + + torch_nccl() + mscclpp() + py_nccl() + torch.cuda.synchronize() + if not torch.equal(mscclpp_output, nccl_output): + raise AssertionError( + f"MSCCL++ AllGather mismatch at {input_bytes} input bytes" + ) + _validate_cuda_graph_updates( + mscclpp_on_stream, + torch_nccl, + input_tensor, + mscclpp_output, + nccl_output, + cpu_group, + ) + + nccl_eager_us = _time_eager(torch_nccl, cpu_group) + mscclpp_eager_us = _time_eager(mscclpp, cpu_group) + mscclpp_graph_us = _time_cuda_graph(mscclpp_on_stream, cpu_group) + pynccl_graph_us = _time_cuda_graph(py_nccl_on_stream, cpu_group) + + result.append( + { + "msg_size": human_readable_size(total_bytes), + "torch eager time": nccl_eager_us, + "msccl eager time": mscclpp_eager_us, + "msccl graph time": mscclpp_graph_us, + "pynccl graph time": pynccl_graph_us, + } + ) + if rank == 0: + print( + f"sz={total_bytes // input_tensor.element_size()}, " + f"dtype={input_tensor.dtype}: correctness check PASS!" + ) + + torch.cuda.empty_cache() + + if rank == 0: + print_markdown_table(result) + + mscclpp_comm.destroy() + dist.barrier() + cleanup_dist_env_and_memory() + + +if __name__ == "__main__": + main() diff --git a/benchmark/kernels/all_reduce/benchmark_mscclpp.py b/benchmark/kernels/all_reduce/benchmark_mscclpp.py index 433d404e50..5c131619a2 100644 --- a/benchmark/kernels/all_reduce/benchmark_mscclpp.py +++ b/benchmark/kernels/all_reduce/benchmark_mscclpp.py @@ -12,9 +12,9 @@ torchrun --nproc_per_node gpu \ --master_port $MASTER_PORT benchmark/kernels/all_reduce/benchmark_mscclpp.py """ +import argparse import os from contextlib import nullcontext -from typing import List import torch import torch.distributed as dist @@ -23,11 +23,14 @@ from torch.distributed import ProcessGroup from sglang.srt.distributed import init_distributed_environment from sglang.srt.distributed.device_communicators.pymscclpp import PyMscclppCommunicator from sglang.srt.distributed.device_communicators.pynccl import PyNcclCommunicator +from sglang.srt.distributed.device_communicators.pynccl_allocator import ( + use_symmetric_memory, +) from sglang.srt.distributed.parallel_state import ( cleanup_dist_env_and_memory, graph_capture, initialize_model_parallel, - set_mscclpp_all_reduce, + set_mscclpp, ) from sglang.srt.runtime_context import get_parallel from sglang.test.test_utils import publish_build_topology @@ -51,14 +54,21 @@ def pynccl_allreduce( return msccl_input -def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=10): - graph_input = inp_randn.clone() - graph_input_snapshot = inp_randn.clone() +def _bench_graph_time( + func, + inp_randn, + tp_group, + warmup_loop=2, + graph_loop=10, + test_loop=10, +): + with use_symmetric_memory(tp_group): + graph_input = inp_randn.clone() with graph_capture() as graph_capture_context: graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph, stream=graph_capture_context.stream): for _ in range(graph_loop): - graph_input.copy_(graph_input_snapshot) + graph_input.copy_(inp_randn) graph_out = func(graph_input) graph.replay() @@ -71,7 +81,7 @@ def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=1 start_event = torch.cuda.Event(enable_timing=True) end_event = torch.cuda.Event(enable_timing=True) - latencies: List[float] = [] + latencies = [] for _ in range(test_loop): torch.cuda.synchronize() dist.barrier() @@ -85,8 +95,9 @@ def _bench_graph_time(func, inp_randn, warmup_loop=2, graph_loop=10, test_loop=1 return func_output, func_cost_us -def _bench_eager_time(func, inp_randn, warmup_loop=2, test_loop=10): - eager_input = inp_randn.clone() +def _bench_eager_time(func, inp_randn, tp_group, warmup_loop=2, test_loop=10): + with use_symmetric_memory(tp_group): + eager_input = inp_randn.clone() eager_output = func(eager_input) func_output = eager_output.clone() @@ -156,57 +167,77 @@ def print_markdown_table(data): if __name__ == "__main__": import logging + parser = argparse.ArgumentParser() + parser.add_argument( + "--enable-symm-mem", + action="store_true", + help="Allocate collective inputs from NCCL symmetric memory.", + ) + args = parser.parse_args() + logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S", force=True, ) + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + device = torch.cuda.current_device() if not dist.is_initialized(): - dist.init_process_group(backend="nccl") + dist.init_process_group( + backend="nccl", device_id=torch.device("cuda", local_rank) + ) world, world_size = dist.group.WORLD, dist.get_world_size() rank = dist.get_rank() - torch.cuda.set_device(rank % 8) - device = torch.cuda.current_device() - set_mscclpp_all_reduce(True) + set_mscclpp(True) init_distributed_environment( world_size=world_size, rank=rank, - local_rank=rank % 8, + local_rank=local_rank, ) - publish_build_topology(world_rank=rank, tp_size=world_size) - initialize_model_parallel() - group = get_parallel().tp_group.device_group - cpu_group = get_parallel().tp_group.cpu_group - pynccl_comm = get_parallel().tp_group.pynccl_comm - pymscclpp_comm = get_parallel().tp_group.pymscclpp_comm + publish_build_topology( + world_rank=rank, + tp_size=world_size, + enable_symm_mem=args.enable_symm_mem, + ) + initialize_model_parallel(enable_symm_mem=args.enable_symm_mem) + tp_group = get_parallel().tp_group + group = tp_group.device_group + pynccl_comm = tp_group.pynccl_comm + pymscclpp_comm = tp_group.pymscclpp_comm dist.barrier() profile = False dtype = torch.bfloat16 ctx = get_torch_prof_ctx(profile) result = [] + max_message_bytes = 1 << 20 with ctx: for i in range(10, 20): sz = 2**i - if sz * dtype.itemsize > 2**20: + if sz * dtype.itemsize > max_message_bytes: break - inp_randn = torch.randint(1, 16, (sz,), dtype=dtype, device=device) + inp_randn = torch.randint(0, 2, (sz,), dtype=dtype, device=device) - memory = torch.empty_like(inp_randn) - memory_out = torch.empty_like(memory) torch_eager_output, torch_eager_time = _bench_eager_time( - lambda inp: torch_allreduce(inp, group), inp_randn + lambda inp: torch_allreduce(inp, group), inp_randn, tp_group ) msccl_eager_output, msccl_eager_time = _bench_eager_time( - lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn + lambda inp: msccl_allreduce(inp, pymscclpp_comm), + inp_randn, + tp_group, ) msccl_graph_output, msccl_graph_time = _bench_graph_time( - lambda inp: msccl_allreduce(inp, pymscclpp_comm), inp_randn + lambda inp: msccl_allreduce(inp, pymscclpp_comm), + inp_randn, + tp_group, ) # since pynccl is inplace op, this return result is not correct if graph loop > 1 _, pynccl_graph_time = _bench_graph_time( - lambda inp: pynccl_allreduce(inp, pynccl_comm), inp_randn + lambda inp: pynccl_allreduce(inp, pynccl_comm), + inp_randn, + tp_group, ) torch.testing.assert_close(torch_eager_output, msccl_graph_output) torch.testing.assert_close(torch_eager_output, msccl_eager_output) diff --git a/docker/Dockerfile b/docker/Dockerfile index a9712bc1d9..8d6ed1b7b1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,7 +15,7 @@ ARG GITHUB_ARTIFACTORY=github.com ARG INSTALL_FLASHINFER_JIT_CACHE=0 ARG FLASHINFER_VERSION=0.6.18 ARG MOONCAKE_VERSION=0.3.13 -ARG MSCCLPP_VERSION=sglang-v0.9.1 +ARG MSCCLPP_VERSION=sglang-v0.10.1 ENV DEBIAN_FRONTEND=noninteractive \ CUDA_HOME=/usr/local/cuda \ @@ -767,12 +767,13 @@ COPY --from=framework_final /root/.cache/huggingface /root/.cache/huggingface COPY --from=framework_final /root/.cache/sglang /root/.cache/sglang # Copy GDRCopy runtime libraries (but not the build artifacts) -COPY --from=framework_final /usr/lib/libgdrapi.so* /usr/lib/ +COPY --from=framework_final /usr/lib/*-linux-gnu/libgdrapi.so* /usr/lib/ COPY --from=framework_final /usr/bin/gdrcopy_* /usr/bin/ COPY --from=framework_final /usr/src/gdrdrv-2.5.1 /usr/src/gdrdrv-2.5.1 # Fix DeepEP IBGDA symlink in runtime -RUN ln -sf /usr/lib/$(uname -m)-linux-gnu/libmlx5.so.1 /usr/lib/$(uname -m)-linux-gnu/libmlx5.so +RUN ln -sf /usr/lib/$(uname -m)-linux-gnu/libmlx5.so.1 /usr/lib/$(uname -m)-linux-gnu/libmlx5.so \ + && ldconfig WORKDIR /sgl-workspace/sglang diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 95705ead56..bc3edcde58 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -606,7 +606,7 @@ class ExecComm(msgspec.Struct): ] = False enable_mscclpp: A[ bool, - "Enable using mscclpp for small messages for all-reduce kernel and fall back to NCCL.", + "Enable MSCCL++ for tuned AllReduce and AllGather messages, with NCCL fallback.", ] = False enable_torch_symm_mem: A[ bool, diff --git a/python/sglang/srt/distributed/bootstrap.py b/python/sglang/srt/distributed/bootstrap.py index dce7f01009..61e0539695 100644 --- a/python/sglang/srt/distributed/bootstrap.py +++ b/python/sglang/srt/distributed/bootstrap.py @@ -20,7 +20,7 @@ from sglang.srt.distributed.parallel_state import ( initialize_model_parallel, set_custom_all_reduce, set_flashinfer_allreduce_only, - set_mscclpp_all_reduce, + set_mscclpp, set_torch_symm_mem_all_reduce, ) from sglang.srt.environ import envs @@ -234,7 +234,7 @@ def _resolve_dist_init_method(*, dist_port: int) -> str: def _set_all_reduce_flags() -> None: set_custom_all_reduce(not get_exec().comm.disable_custom_all_reduce) - set_mscclpp_all_reduce(get_exec().comm.enable_mscclpp) + set_mscclpp(get_exec().comm.enable_mscclpp) set_torch_symm_mem_all_reduce(get_exec().comm.enable_torch_symm_mem) from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion diff --git a/python/sglang/srt/distributed/device_communicators/pymscclpp.py b/python/sglang/srt/distributed/device_communicators/pymscclpp.py index 1d5c2cfaa5..2442c9fc5b 100644 --- a/python/sglang/srt/distributed/device_communicators/pymscclpp.py +++ b/python/sglang/srt/distributed/device_communicators/pymscclpp.py @@ -1,8 +1,11 @@ import importlib import logging from contextlib import contextmanager -from typing import Optional, Union +from dataclasses import replace as dataclass_replace +from itertools import product +from typing import Any, Callable, ClassVar, Optional, Union +import msgspec import torch import torch.distributed as dist from torch.distributed import ProcessGroup, ReduceOp @@ -18,11 +21,535 @@ from sglang.srt.runtime_context import get_exec logger = logging.getLogger(__name__) +_SUPPORTED_COLLECTIVES = {"allreduce", "allgather", "reducescatter"} +_DEFAULT_SUPPORTED_DTYPES = (torch.float, torch.float16, torch.bfloat16) +_DEFAULT_GPU_BUFFER_SIZE = 1 << 27 +_TUNING_MESSAGE_SIZES = tuple(1 << exponent for exponent in range(9, 24)) +_ParameterAdapter = Callable[..., "_AlgorithmConfig"] + + +class _MessageSizeRange(msgspec.Struct, frozen=True): + minimum: int + maximum: int + + def __post_init__(self): + if self.minimum < 0 or self.maximum < self.minimum: + raise ValueError(f"Invalid message size range: {self}") + + def contains(self, size: int) -> bool: + return self.minimum <= size <= self.maximum + + +class _AlgorithmConfig(msgspec.Struct, frozen=True, kw_only=True): + implementation: ClassVar[str] + name: str + collective: str + world_sizes: tuple[int, ...] + ipc_domain_counts: tuple[int, ...] + threads_per_block: tuple[int, ...] + message_size_range: _MessageSizeRange + reduce_op: str + parameter_adapter: Optional[_ParameterAdapter] = None + supported_dtypes: tuple[torch.dtype, ...] = _DEFAULT_SUPPORTED_DTYPES + in_place: bool = True + requires_nvls: bool = False + requires_symmetric_memory: bool = False + algorithm: Any = None + + def __post_init__(self): + if self.collective not in _SUPPORTED_COLLECTIVES: + raise ValueError(f"Unsupported collective: {self.collective}") + if self.reduce_op not in {"SUM", "NOP"}: + raise ValueError(f"Unsupported reduction operation: {self.reduce_op}") + if not self.world_sizes or not self.ipc_domain_counts: + raise ValueError("Algorithm topology constraints cannot be empty") + if not self.threads_per_block: + raise ValueError("Algorithm topology constraints cannot be empty") + if not self.supported_dtypes: + raise ValueError("Algorithm supported dtypes cannot be empty") + + def supports_topology(self, world_size: int, ipc_domain_count: int) -> bool: + return ( + world_size in self.world_sizes + and ipc_domain_count in self.ipc_domain_counts + ) + + def requirements_satisfied( + self, + world_size: int, + ipc_domain_count: int, + nvls_supported: bool, + symmetric_memory: bool, + ) -> bool: + return ( + self.supports_topology(world_size, ipc_domain_count) + and self.support_nvls(nvls_supported) + and self.support_symmetric_memory(symmetric_memory) + ) + + def support_nvls(self, nvls_supported: bool) -> bool: + return not self.requires_nvls or nvls_supported + + def support_symmetric_memory(self, symmetric_memory: bool) -> bool: + return not self.requires_symmetric_memory or symmetric_memory + + def supports_message_size(self, message_size: int) -> bool: + return self.message_size_range.contains(message_size) + + def supports_dtype(self, dtype: torch.dtype) -> bool: + return dtype in self.supported_dtypes + + def resolve_reduce_op(self, reduce_ops): + return getattr(reduce_ops, self.reduce_op) + + def adapt_to_topology( + self, world_size: int, nranks_per_ipc_domain: int + ) -> "_AlgorithmConfig": + if self.parameter_adapter is None: + return self + return self.parameter_adapter( + self, + world_size=world_size, + nranks_per_ipc_domain=nranks_per_ipc_domain, + ) + + def input_buffer_size(self, message_size: int, world_size: int) -> int: + if self.collective == "allgather": + if message_size % world_size != 0: + raise ValueError( + f"All-gather output size {message_size} must be divisible " + f"by world size {world_size}" + ) + return message_size // world_size + return message_size + + def output_buffer_size(self, message_size: int, world_size: int) -> int: + if self.collective == "reducescatter": + if message_size % world_size != 0: + raise ValueError( + f"Reduce-scatter input size {message_size} must be divisible " + f"by world size {world_size}" + ) + return message_size // world_size + return message_size + + def tuning_launches(self) -> tuple[tuple[int, int], ...]: + raise NotImplementedError + + def bind(self, algorithm) -> "_AlgorithmConfig": + return msgspec.structs.replace(self, algorithm=algorithm) + + def select(self, nblocks: int, threads_per_block: int) -> "_AlgorithmConfig": + if self.algorithm is None: + raise RuntimeError(f"Algorithm {self.name} has not been bound") + if (nblocks, threads_per_block) not in self.tuning_launches(): + raise ValueError(f"Invalid launch selection for algorithm {self.name}") + return self + + def selected_launch(self) -> tuple[int, int]: + if self.algorithm is None: + raise RuntimeError(f"Algorithm {self.name} has not been bound") + return self.tuning_launches()[0] + + def reset(self): + if self.algorithm is None: + raise RuntimeError(f"Algorithm {self.name} has not been bound") + self.algorithm.reset() + + @staticmethod + def compose_rsag( + reduce_scatter: "_AlgorithmConfig", + allgather: "_AlgorithmConfig", + *, + rank: int, + world_size: int, + ipc_domain_count: int, + reduce_ops: Any, + ) -> "_CompositeAlgorithmConfig": + if ( + reduce_scatter.collective != "reducescatter" + or allgather.collective != "allgather" + ): + raise ValueError("RSAG requires reduce-scatter and all-gather algorithms") + if reduce_scatter.algorithm is None or allgather.algorithm is None: + raise RuntimeError("RSAG composition requires bound algorithms") + message_size_range = _MessageSizeRange( + minimum=max( + reduce_scatter.message_size_range.minimum, + allgather.message_size_range.minimum, + ), + maximum=min( + reduce_scatter.message_size_range.maximum, + allgather.message_size_range.maximum, + ), + ) + algorithm = _TwoKernelAllReduce( + name=( + f"allreduce_rsag_{reduce_scatter.algorithm.name}_" + f"{allgather.algorithm.name}" + ), + reduce_scatter=reduce_scatter.algorithm, + allgather=allgather.algorithm, + allgather_op=allgather.resolve_reduce_op(reduce_ops), + rank=rank, + world_size=world_size, + ) + return _CompositeAlgorithmConfig( + name=algorithm.name, + collective="allreduce", + world_sizes=(world_size,), + ipc_domain_counts=(ipc_domain_count,), + threads_per_block=(0,), + message_size_range=message_size_range, + reduce_op="SUM", + supported_dtypes=tuple( + dtype + for dtype in reduce_scatter.supported_dtypes + if dtype in allgather.supported_dtypes + ), + algorithm=algorithm, + ) + + +class _DslAlgorithmConfig(_AlgorithmConfig): + implementation: ClassVar[str] = "dsl" + algo_spec: Any + algorithm_kwargs: dict[str, tuple[Any, ...]] = msgspec.field(default_factory=dict) + + def __post_init__(self): + super().__post_init__() + if not self.threads_per_block: + raise ValueError("DSL compile thread candidates cannot be empty") + if any(threads <= 0 for threads in self.threads_per_block): + raise ValueError("DSL compile thread candidates must be positive") + if any(not values for values in self.algorithm_kwargs.values()): + raise ValueError("DSL algorithm kwarg candidates cannot be empty") + if self.algorithm is None: + if self.algo_spec.world_size != 0 or self.algo_spec.nranks_per_node != 0: + raise ValueError("DSL AlgoSpec templates require zero topology values") + if self.algo_spec.name != self.name: + raise ValueError("DSL AlgoSpec and algorithm names must match") + elif self.algo_spec.world_size <= 0 or self.algo_spec.nranks_per_node <= 0: + raise ValueError("Bound DSL algorithms require a materialized AlgoSpec") + if self.algo_spec.collective.name != self.collective: + raise ValueError("DSL AlgoSpec and algorithm collectives must match") + if self.algo_spec.in_place != self.in_place: + raise ValueError("DSL AlgoSpec and algorithm buffer modes must match") + + def tuning_launches(self) -> tuple[tuple[int, int], ...]: + return ((0, 0),) + + def adapt_to_topology( + self, + world_size: int, + nranks_per_ipc_domain: int, + algorithm_kwargs: Optional[dict[str, Any]] = None, + ) -> "_AlgorithmConfig": + if self.parameter_adapter is None: + return self + return self.parameter_adapter( + self, + world_size=world_size, + nranks_per_ipc_domain=nranks_per_ipc_domain, + instances=self.algo_spec.instances, + algorithm_kwargs=algorithm_kwargs or {}, + ) + + def bind(self, algorithm, algo_spec) -> "_AlgorithmConfig": + return msgspec.structs.replace(self, algorithm=algorithm, algo_spec=algo_spec) + + def dsl_name( + self, + ipc_domain_count: int, + threads_per_block: int, + algorithm_kwargs: dict[str, Any], + ) -> str: + variant_parts = [ + f"{key}_{value}" for key, value in sorted(algorithm_kwargs.items()) + ] + variant = f"{'_'.join(variant_parts)}_" if variant_parts else "" + return f"{self.name}_{ipc_domain_count}node_{variant}{threads_per_block}TPB" + + +class _NativeAlgorithmConfig(_AlgorithmConfig): + implementation: ClassVar[str] = "native" + nblocks: tuple[int, ...] + selected_launch_parameters: Optional[tuple[int, int]] = None + + def __post_init__(self): + super().__post_init__() + if not self.nblocks: + raise ValueError("Native launch candidates cannot be empty") + if self.selected_launch_parameters is not None: + if self.algorithm is None: + raise ValueError("Only bound native algorithms can select a launch") + if self.selected_launch_parameters not in self.tuning_launches(): + raise ValueError("Invalid native launch selection") + + def tuning_launches(self) -> tuple[tuple[int, int], ...]: + return tuple( + (nblocks, threads_per_block) + for nblocks in self.nblocks + for threads_per_block in self.threads_per_block + ) + + def select(self, nblocks: int, threads_per_block: int) -> "_AlgorithmConfig": + if self.algorithm is None: + raise RuntimeError(f"Algorithm {self.name} has not been bound") + launch = (nblocks, threads_per_block) + if launch not in self.tuning_launches(): + raise ValueError(f"Invalid launch selection for algorithm {self.name}") + return msgspec.structs.replace(self, selected_launch_parameters=launch) + + def selected_launch(self) -> tuple[int, int]: + if self.selected_launch_parameters is None: + raise RuntimeError(f"Algorithm {self.name} has not been tuned") + return self.selected_launch_parameters + + +class _CompositeAlgorithmConfig(_AlgorithmConfig): + implementation: ClassVar[str] = "composite" + + def __post_init__(self): + super().__post_init__() + if self.algorithm is None: + raise ValueError("Composite algorithms must be bound when constructed") + + def tuning_launches(self) -> tuple[tuple[int, int], ...]: + return ((0, 0),) + + +_DEFAULT_THREADS_PER_BLOCK = (256, 512, 768, 1024) +_DEFAULT_THREAD_BLOCK_GROUP_SIZES = (1, 2, 4, 8) +_SUPPORTED_WORLD_SIZES = (4, 8, 16, 32, 64) +_NATIVE_SUPPORTED_WORLD_SIZES = (4, 8, 16, 32) +_NATIVE_IPC_DOMAIN_COUNTS = (1,) +_MULTI_NODE_IPC_DOMAIN_COUNTS = (2, 4, 8) + + +def _adapt_allreduce_packet( + config: _AlgorithmConfig, + *, + world_size: int, + nranks_per_ipc_domain: int, +) -> _AlgorithmConfig: + if not isinstance(config, _NativeAlgorithmConfig): + raise TypeError("AllReduce packet adaptation requires a native config") + if world_size != nranks_per_ipc_domain: + return config + min_blocks = nranks_per_ipc_domain - 1 + nblocks = tuple(nblocks for nblocks in config.nblocks if nblocks >= min_blocks) + if not nblocks: + return config + return msgspec.structs.replace(config, nblocks=nblocks) + + +def _adapt_dsl_message_size_range( + config: _AlgorithmConfig, + *, + world_size: int, + nranks_per_ipc_domain: int, + algorithm_kwargs: dict[str, Any], + **_: Any, +) -> _AlgorithmConfig: + if not isinstance(config, _DslAlgorithmConfig): + raise TypeError("DSL message size adaptation requires a DSL config") + ipc_domain_count = world_size // nranks_per_ipc_domain + thread_block_group_size = algorithm_kwargs.get("thread_block_group_size", 1) + return msgspec.structs.replace( + config, + message_size_range=_MessageSizeRange( + minimum=config.message_size_range.minimum * thread_block_group_size, + maximum=config.message_size_range.maximum * ipc_domain_count, + ), + ) + + +def _create_native_algorithm_configs() -> tuple[_NativeAlgorithmConfig, ...]: + return ( + _NativeAlgorithmConfig( + name="default_allreduce_nvls_packet", + collective="allreduce", + world_sizes=_NATIVE_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(0, 512 << 10), + nblocks=(4, 8, 12, 16), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + requires_nvls=True, + ), + _NativeAlgorithmConfig( + name="default_allreduce_packet", + collective="allreduce", + world_sizes=_NATIVE_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(0, 2 << 20), + nblocks=(14, 21, 28, 42, 56), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + parameter_adapter=_adapt_allreduce_packet, + ), + _NativeAlgorithmConfig( + name="default_allreduce_rsag_zero_copy", + collective="allreduce", + world_sizes=_NATIVE_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(512 << 10, 4 << 30), + nblocks=(32, 48, 64, 128), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + ), + _NativeAlgorithmConfig( + name="default_allreduce_nvls_zero_copy", + collective="allreduce", + world_sizes=_NATIVE_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_NATIVE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(1 << 10, 4 << 30), + nblocks=(4, 8, 12, 16, 32), + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + reduce_op="SUM", + requires_nvls=True, + requires_symmetric_memory=True, + ), + ) + + +def _create_algorithm_configs(language) -> tuple[_AlgorithmConfig, ...]: + default_spec = language.AlgoSpec( + name="allreduce_multi_nodes", + collective=language.collectives.AllReduce(0, 1, True), + nranks_per_node=0, + world_size=0, + in_place=True, + instances=1, + protocol="LL", + instr_fusion=True, + auto_sync=False, + replication_policy=language.ReplicationPolicy.interleaved, + reuse_resources=True, + use_double_scratch_buffer=True, + buffer_alignment=16, + ) + allgather_spec = dataclass_replace( + default_spec, + name="allgather_multi_nodes", + collective=language.collectives.AllGather(0, 1, False), + in_place=False, + ) + reduce_scatter_spec = dataclass_replace( + default_spec, + name="reducescatter_multi_nodes", + collective=language.collectives.ReduceScatter(0, 1, True), + instr_fusion=False, + ) + dsl_configs = ( + _DslAlgorithmConfig( + name="allreduce_multi_nodes", + collective="allreduce", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(1 << 10, 1 << 20), + reduce_op="SUM", + algo_spec=default_spec, + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + algorithm_kwargs={ + "thread_block_group_size": _DEFAULT_THREAD_BLOCK_GROUP_SIZES + }, + parameter_adapter=_adapt_dsl_message_size_range, + ), + _DslAlgorithmConfig( + name="allgather_multi_nodes", + collective="allgather", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(1 << 10, 1 << 20), + reduce_op="NOP", + in_place=False, + algo_spec=allgather_spec, + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + parameter_adapter=_adapt_dsl_message_size_range, + ), + _DslAlgorithmConfig( + name="reducescatter_multi_nodes", + collective="reducescatter", + world_sizes=_SUPPORTED_WORLD_SIZES, + ipc_domain_counts=_MULTI_NODE_IPC_DOMAIN_COUNTS, + message_size_range=_MessageSizeRange(1 << 10, 1 << 20), + reduce_op="SUM", + algo_spec=reduce_scatter_spec, + threads_per_block=_DEFAULT_THREADS_PER_BLOCK, + algorithm_kwargs={ + "thread_block_group_size": _DEFAULT_THREAD_BLOCK_GROUP_SIZES + }, + parameter_adapter=_adapt_dsl_message_size_range, + ), + ) + return (*dsl_configs, *_create_native_algorithm_configs()) + + +class _TwoKernelAllReduce(msgspec.Struct, frozen=True): + name: str + reduce_scatter: Any + allgather: Any + allgather_op: Any + rank: int + world_size: int + + def execute( + self, + *, + comm, + executor, + input_buffer, + output_buffer, + input_size, + output_size, + dtype, + op, + stream, + nblocks, + nthreads_per_block, + symmetric_memory, + ): + shard_size = input_size // self.world_size + + result = self.reduce_scatter.execute( + comm=comm, + executor=executor, + input_buffer=input_buffer, + output_buffer=output_buffer, + input_size=input_size, + output_size=shard_size, + dtype=dtype, + op=op, + stream=stream, + nblocks=nblocks, + nthreads_per_block=nthreads_per_block, + symmetric_memory=symmetric_memory, + ) + if result != 0: + return result + return self.allgather.execute( + comm=comm, + executor=executor, + input_buffer=output_buffer + self.rank * shard_size, + output_buffer=output_buffer, + input_size=shard_size, + output_size=output_size, + dtype=dtype, + op=self.allgather_op, + stream=stream, + nblocks=nblocks, + nthreads_per_block=nthreads_per_block, + symmetric_memory=symmetric_memory, + ) + + def reset(self): + self.reduce_scatter.reset() + self.allgather.reset() + class PyMscclppCommunicator: - _SUPPORTED_WORLD_SIZES = [8, 16, 32] - _SUPPORTED_DTYPE = [torch.float, torch.float16, torch.bfloat16] - def _is_symm_mem_enabled(self) -> bool: try: return get_exec().comm.enable_symm_mem @@ -35,262 +562,529 @@ class PyMscclppCommunicator: == inp.numel() * inp.element_size() ) - def _get_tuned_config(self, size): - if size <= 512: - target_size = 512 - elif size > 256 * 1024 * 1024: - target_size = 256 * 1024 * 1024 - else: - target_size = 1 << (size - 1).bit_length() - return self.best_configs.get(target_size) + def _get_tuned_config(self, collective: str, message_size: int): + if message_size <= 0: + return None + configs = self._best_configs[collective] + target_size = 1 << (message_size - 1).bit_length() + config = configs.get(target_size) + if config is not None and config.supports_message_size(message_size): + return config - def _create_dsl_algorithms(self): - dsl_algos_config = [] - n_nodes = self.world_size // self.nranks_per_node - if n_nodes == 2 or n_nodes == 4: - for tbg in [1, 2, 4, 8]: - for num_threads_per_block in [256, 512, 768, 1024]: - spec = self.mscclpp.language.AlgoSpec( - name=f"allreduce_{n_nodes}node_{tbg}TBG_{num_threads_per_block}TPB", - collective=self.mscclpp.language.collectives.AllReduce( - self.world_size, 1, True - ), - nranks_per_node=self.nranks_per_node, - world_size=self.world_size, - in_place=True, - instances=1, - protocol="LL", - auto_sync=False, - num_threads_per_block=num_threads_per_block, - reuse_resources=True, - use_double_scratch_buffer=True, - min_message_size=tbg * (1 << 10), - max_message_size=8 << 20, - tags={"default": 1}, - ) - algo = self.mscclpp.compile( - self.def_algo.allreduce_multi_nodes, - spec, - self.rank, - thread_block_group_size=tbg, - ) - dsl_algos_config.append((algo, [0], [0])) - return dsl_algos_config + if not configs: + return None + minimum_tuned_size = min(configs) + minimum_config = configs[minimum_tuned_size] + if target_size < minimum_tuned_size and minimum_config.supports_message_size( + message_size + ): + return minimum_config + return None - def _create_native_algorithms(self): - navitve_algorithms_config = [] - dlpack = self.mscclpp.RawGpuBuffer(1 << 27).to_dlpack( + def _get_allreduce_tuned_config(self, message_size): + return self._get_tuned_config("allreduce", message_size) + + def _get_allgather_tuned_config(self, message_size): + return self._get_tuned_config("allgather", message_size) + + def _algorithm_configs(self, ipc_domain_count: int): + return [ + config + for config in self._registered_algorithm_configs + if config.requirements_satisfied( + self.world_size, + ipc_domain_count, + nvls_supported=self.mscclpp.is_nvls_supported(), + symmetric_memory=self.symm_mem_enabled, + ) + ] + + def _compile_dsl_algorithm( + self, + config: _DslAlgorithmConfig, + ipc_domain_count: int, + ) -> list[_DslAlgorithmConfig]: + algorithm_builder = getattr(self.def_algo, config.name, None) + if algorithm_builder is None: + raise RuntimeError( + "The installed MSCCL++ package does not provide " + f"default_algos.{config.name}" + ) + template_collective = config.algo_spec.collective + collective = type(template_collective)( + self.world_size, + template_collective.chunk_factor, + config.in_place, + ) + algorithms = [] + algorithm_kwarg_names = tuple(config.algorithm_kwargs) + for values in product(*config.algorithm_kwargs.values()): + algorithm_kwargs = dict(zip(algorithm_kwarg_names, values)) + for threads_per_block in config.threads_per_block: + spec = dataclass_replace( + config.algo_spec, + name=config.dsl_name( + ipc_domain_count, + threads_per_block, + algorithm_kwargs, + ), + collective=collective, + nranks_per_node=self.nranks_per_ipc_domain, + world_size=self.world_size, + num_threads_per_block=threads_per_block, + ) + candidate = config.adapt_to_topology( + self.world_size, + self.nranks_per_ipc_domain, + algorithm_kwargs, + ) + algorithm = self.mscclpp.compile( + algorithm_builder, + spec, + self.rank, + **algorithm_kwargs, + ) + algorithms.append(candidate.bind(algorithm, spec)) + return algorithms + + def _create_dsl_algorithms( + self, + configs: list[_DslAlgorithmConfig], + ipc_domain_count: int, + ) -> list[_DslAlgorithmConfig]: + algorithms = [] + for config in configs: + algorithms.extend(self._compile_dsl_algorithm(config, ipc_domain_count)) + return algorithms + + def _create_native_algorithms( + self, configs: list[_NativeAlgorithmConfig] + ) -> list[_NativeAlgorithmConfig]: + dlpack = self.mscclpp.RawGpuBuffer(_DEFAULT_GPU_BUFFER_SIZE).to_dlpack( data_type=str(torch.float16) ) self.scratch_buffer = torch.utils.dlpack.from_dlpack(dlpack) - self.flag_buffer = torch.ones(128, dtype=torch.uint32, device="cuda") algos = self.mscclpp_ext.AlgorithmCollectionBuilder().build_default_algorithms( scratch_buffer=self.scratch_buffer.data_ptr(), scratch_buffer_size=self.scratch_buffer.nbytes, rank=self.rank, ) + configs_by_name = {config.name: config for config in configs} + algorithms = [] for algo in algos: - if algo.name == "default_allreduce_nvls_packet": - algo.set_message_size_range(0, 512 << 10) - navitve_algorithms_config.append( - (algo, [4, 8, 12, 16], [256, 512, 768, 1024]) - ) - if algo.name == "default_allreduce_packet": - algo.set_message_size_range(0, 2 << 20) - navitve_algorithms_config.append( - (algo, [14, 21, 28, 42, 56], [256, 512, 768, 1024]) - ) - if algo.name == "default_allreduce_rsag_zero_copy": - algo.set_message_size_range(512 << 10, 4 << 30) - navitve_algorithms_config.append( - (algo, [32, 48, 64, 128], [256, 512, 768, 1024]) - ) - if ( - self.symm_mem_enabled - and algo.name == "default_allreduce_nvls_zero_copy" - ): - algo.set_message_size_range(512 << 10, 4 << 30) - navitve_algorithms_config.append( - (algo, [4, 8, 12, 16, 32], [256, 512, 768, 1024]) - ) + config = configs_by_name.get(algo.name) + if config is None: + continue + message_range = config.message_size_range + algo.set_message_size_range(message_range.minimum, message_range.maximum) + config = config.adapt_to_topology( + self.world_size, self.nranks_per_ipc_domain + ) + algorithms.append(config.bind(algo)) - return navitve_algorithms_config + return algorithms + + def _create_algorithm_candidates( + self, + configs: list[_AlgorithmConfig], + ipc_domain_count: int, + ) -> list[_AlgorithmConfig]: + native_configs = [ + config for config in configs if isinstance(config, _NativeAlgorithmConfig) + ] + dsl_configs = [ + config for config in configs if isinstance(config, _DslAlgorithmConfig) + ] + algorithms = [] + if native_configs: + algorithms.extend(self._create_native_algorithms(native_configs)) + if dsl_configs: + algorithms.extend( + self._create_dsl_algorithms(dsl_configs, ipc_domain_count) + ) + return algorithms def _create_algorithms(self): - if self.world_size == 8: - self.algos_config = self._create_native_algorithms() - self._tune(5, 10, 20, self.algos_config) - elif self.world_size == 16 or self.world_size == 32: - self.dsl_algos_config = self._create_dsl_algorithms() - self._tune(5, 10, 20, self.dsl_algos_config) + ipc_domain_count = self.world_size // self.nranks_per_ipc_domain + algorithms = self._create_algorithm_candidates( + self._algorithm_configs(ipc_domain_count), + ipc_domain_count, + ) + algorithms_by_collective = { + collective: [ + algorithm + for algorithm in algorithms + if algorithm.collective == collective + ] + for collective in ("allreduce", "allgather", "reducescatter") + } + allreduce_algorithms = algorithms_by_collective["allreduce"] + allgather_algorithms = algorithms_by_collective["allgather"] + reduce_scatter_algorithms = algorithms_by_collective["reducescatter"] + need_rsag = ( + "allreduce" in self.collectives + and bool(allgather_algorithms) + and bool(reduce_scatter_algorithms) + ) + if "allgather" in self.collectives or need_rsag: + if allgather_algorithms: + self._tune(allgather_algorithms) + + if "allreduce" in self.collectives and allreduce_algorithms: + self._tune(allreduce_algorithms) + if need_rsag: + self._tune(reduce_scatter_algorithms) + self._merge_rsag_configs() + + def _merge_rsag_configs(self): + allreduce_configs = self._best_configs["allreduce"] + allgather_configs = self._best_configs["allgather"] + reduce_scatter_configs = self._best_configs["reducescatter"] + allreduce_times = self._best_config_times["allreduce"] + allgather_times = self._best_config_times["allgather"] + reduce_scatter_times = self._best_config_times["reducescatter"] + for message_size, reduce_scatter_config in sorted( + reduce_scatter_configs.items() + ): + allreduce_config = allreduce_configs.get(message_size) + allgather_config = allgather_configs.get(message_size) + if allreduce_config is None or allgather_config is None: + continue + + composite = _AlgorithmConfig.compose_rsag( + reduce_scatter_config, + allgather_config, + rank=self.rank, + world_size=self.world_size, + ipc_domain_count=(self.world_size // self.nranks_per_ipc_domain), + reduce_ops=self.mscclpp.ReduceOp, + ) + try: + composite_time = ( + reduce_scatter_times[message_size] + allgather_times[message_size] + ) + allreduce_time = allreduce_times[message_size] + except KeyError as exc: + raise RuntimeError( + "RSAG merge requires timings for all tuned algorithms" + ) from exc + selected = composite_time < allreduce_time + if selected: + allreduce_configs[message_size] = composite + allreduce_times[message_size] = composite_time + if self.rank == 0: + logger.info( + "MSCCL++ allreduce RSAG merge: input_bytes=%d " + "reduce_scatter=%s allgather=%s estimated_time_ms=%.4f " + "selected=%s", + message_size, + reduce_scatter_config.algorithm.name, + allgather_config.algorithm.name, + composite_time, + selected, + ) def _get_time( self, - algo, - tune_tensor, - size, - nb, - nt, + algorithm_config, + input_tensor, + output_tensor, + message_size, + nblocks, + nthreads, n_warmup, n_graph_launches, n_ops_per_graph, ): - # Check if the algorithm can run with the given configuration - if self._run_algo(algo, tune_tensor, size, nb, nt, True) != 0: + def run(stream=None): + return self._run_algorithm( + algorithm_config, + input_tensor, + output_tensor, + message_size, + nblocks, + nthreads, + stream=stream, + ) + + result = run() + failed = torch.tensor([int(result != 0)], dtype=torch.int32) + dist.all_reduce(failed, op=ReduceOp.MAX, group=self.group) + if failed.item(): + if result != 0: + logger.warning( + "MSCCL++ %s tuning candidate failed on rank %d with error " + "code %s; skipping algorithm=%s message_bytes=%d " + "nblocks=%d nthreads=%d", + algorithm_config.collective, + self.rank, + result, + algorithm_config.name, + message_size, + nblocks, + nthreads, + ) return float("inf") - # Warmup iterations to stabilize performance for _ in range(n_warmup): - self._run_algo(algo, tune_tensor, size, nb, nt, True) + run() - # Warmup on capture stream capture_stream = torch.cuda.Stream() capture_stream.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(capture_stream): - self._run_algo(algo, tune_tensor, size, nb, nt, True) + run(capture_stream) capture_stream.synchronize() - # Capture the algorithm execution in a CUDA graph - g = torch.cuda.CUDAGraph() - with torch.cuda.graph(g, stream=capture_stream): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph, stream=capture_stream): for _ in range(n_ops_per_graph): - self._run_algo(algo, tune_tensor, size, nb, nt, True) + run(capture_stream) - # Measure the execution time of the captured graph + capture_stream.synchronize() + dist.barrier(group=self.group) start_event = torch.cuda.Event(enable_timing=True) end_event = torch.cuda.Event(enable_timing=True) start_event.record(capture_stream) with torch.cuda.stream(capture_stream): for _ in range(n_graph_launches): - g.replay() + graph.replay() end_event.record(capture_stream) end_event.synchronize() - elapsed = start_event.elapsed_time(end_event) - - # Synchronize timing results across all ranks to ensure consistent algorithm selection - # replicate n times such due to algo limitations - time_tensor = torch.full( - (self.world_size,), elapsed, dtype=torch.float64, device="cuda" - ).to(dtype=torch.float32) torch.cuda.current_stream().wait_stream(capture_stream) - if self.rank == 0: - avg_time = time_tensor[self.rank].item() / self.world_size - tensor = torch.tensor([avg_time]) - else: - tensor = torch.empty(1) - dist.broadcast(tensor, src=0, group=self.group) - avg_time = tensor.item() - return avg_time - - def _tune(self, n_warmup, n_graph_launches, n_ops_per_graph, algos_config): - sizes = [1 << i for i in range(9, 24)] - dlpack = self.mscclpp.RawGpuBuffer(1 << 27).to_dlpack( - data_type=str(torch.float16) + elapsed = start_event.elapsed_time(end_event) / ( + n_graph_launches * n_ops_per_graph ) - tune_tensor = torch.utils.dlpack.from_dlpack(dlpack) + max_elapsed = torch.tensor([elapsed], dtype=torch.float64) + dist.all_reduce(max_elapsed, op=ReduceOp.MAX, group=self.group) + return max_elapsed.item() + + def _allocate_tuning_tensor(self, nbytes: int, dtype: torch.dtype) -> torch.Tensor: + element_size = torch.empty((), dtype=dtype).element_size() + if nbytes <= 0 or nbytes % element_size != 0: + raise ValueError( + f"Tuning buffer size {nbytes} is invalid for dtype {dtype}" + ) + with torch.cuda.device(self.device): + dlpack = self.mscclpp.RawGpuBuffer(nbytes).to_dlpack(data_type=str(dtype)) + return torch.utils.dlpack.from_dlpack(dlpack) + + def _tune( + self, + algorithms: list[_AlgorithmConfig], + n_warmup: int = 5, + n_graph_launches: int = 20, + n_ops_per_graph: int = 20, + ): + dtype = torch.bfloat16 + element_size = torch.empty((), dtype=dtype).element_size() + tuning_cases = [] + for message_size in _TUNING_MESSAGE_SIZES: + candidates = [ + candidate + for candidate in algorithms + if candidate.supports_message_size(message_size) + ] + if not candidates: + continue + input_size = max( + candidate.input_buffer_size(message_size, self.world_size) + for candidate in candidates + ) + output_size = max( + candidate.output_buffer_size(message_size, self.world_size) + for candidate in candidates + ) + tuning_cases.append((message_size, candidates, input_size, output_size)) + + if tuning_cases: + input_buffer = self._allocate_tuning_tensor( + max(case[2] for case in tuning_cases), dtype + ) + output_buffer = self._allocate_tuning_tensor( + max(case[3] for case in tuning_cases), dtype + ) + + for message_size, candidates, input_size, output_size in tuning_cases: + input_tensor = input_buffer[: input_size // element_size] + output_tensor = output_buffer[: output_size // element_size] - for size in sizes: best_time = float("inf") best_config = None - for i in range(len(algos_config)): - algo, candidates_nblocks, candidates_nthreads = algos_config[i] - if ( - size >= algo.message_size_range[0] - and size <= algo.message_size_range[1] - ): - for nb in candidates_nblocks: - for nt in candidates_nthreads: - avg_time = self._get_time( - algo, - tune_tensor, - size, - nb, - nt, - n_warmup, - n_graph_launches, - n_ops_per_graph, - ) - if avg_time < best_time: - best_time = avg_time - best_config = (algo, nb, nt) - if best_config: - self.best_configs[size] = best_config + for candidate in candidates: + candidate_output = input_tensor if candidate.in_place else output_tensor + for nblocks, nthreads in candidate.tuning_launches(): + if self.rank == 0: + threads_per_block = ( + candidate.algo_spec.num_threads_per_block + if isinstance(candidate, _DslAlgorithmConfig) + else nthreads + ) + logger.debug( + "MSCCL++ %s tuning candidate: message_bytes=%d " + "algorithm=%s nblocks=%d nthreads=%d", + candidate.collective, + message_size, + candidate.algorithm.name, + nblocks, + threads_per_block, + ) + avg_time = self._get_time( + candidate, + input_tensor, + candidate_output, + message_size, + nblocks, + nthreads, + n_warmup, + n_graph_launches, + n_ops_per_graph, + ) + config = (candidate, nblocks, nthreads) + if avg_time < best_time: + best_time = avg_time + best_config = config + if best_config is not None: + self._best_configs[candidate.collective][message_size] = best_config[ + 0 + ].select( + best_config[1], + best_config[2], + ) + self._best_config_times[candidate.collective][message_size] = best_time + if self.rank == 0: + threads_per_block = ( + best_config[0].algo_spec.num_threads_per_block + if isinstance(best_config[0], _DslAlgorithmConfig) + else best_config[2] + ) + logger.info( + "MSCCL++ %s tuning: message_bytes=%d " + "algorithm=%s nblocks=%d nthreads=%d time_ms=%.4f", + candidate.collective, + message_size, + best_config[0].algorithm.name, + best_config[1], + threads_per_block, + best_time, + ) + del input_tensor, output_tensor torch.cuda.synchronize() - for algo, _, _ in algos_config: - algo.reset() + for candidate in algorithms: + candidate.reset() - def _run_algo(self, algo, tensor, size, nblocks, nthreads, sym_mem_enabled=False): - return algo.execute( + def _run_algorithm( + self, + config: _AlgorithmConfig, + input_tensor, + output_tensor, + message_size, + nblocks, + nthreads, + *, + stream=None, + ): + if stream is None: + stream = torch.cuda.current_stream() + + if config.algorithm is None: + raise RuntimeError(f"Algorithm {config.name} has not been bound") + input_size = config.input_buffer_size(message_size, self.world_size) + output_size = config.output_buffer_size(message_size, self.world_size) + return config.algorithm.execute( comm=self.comm.communicator, executor=self.executor, - input_buffer=tensor.data_ptr(), - output_buffer=tensor.data_ptr(), - input_size=size, - output_size=size, - dtype=self.dtype_to_mscclpp_dtype(tensor.dtype), - op=self.mscclpp.ReduceOp.SUM, - stream=torch.cuda.current_stream().cuda_stream, + input_buffer=input_tensor.data_ptr(), + output_buffer=output_tensor.data_ptr(), + input_size=input_size, + output_size=output_size, + dtype=self.dtype_to_mscclpp_dtype(input_tensor.dtype), + op=config.resolve_reduce_op(self.mscclpp.ReduceOp), + stream=stream.cuda_stream, nblocks=nblocks, nthreads_per_block=nthreads, - symmetric_memory=sym_mem_enabled, + symmetric_memory=self.symm_mem_enabled, ) def __init__( self, group: ProcessGroup, device: Union[int, str, torch.device], + group_name: str = "anonymous", + collectives: Optional[set[str]] = None, ) -> None: """Args: - group: the process group to work on. If None, it will use the - default process group. - device: the device to bind the CustomAllreduce to. If None, - it will be bind to f"cuda:{local_rank}". - It is the caller's responsibility to make sure each communicator - is bind to a unique device, and all communicators in this group - are in the same node. + group: A non-NCCL process group used to bootstrap MSCCL++ and + synchronize tuning results. + device: The CUDA device used by this rank. + group_name: a human-readable process-group name for logging. + collectives: Public collectives to compile and tune. Reduce-scatter + may be tuned internally to form an all-reduce candidate. + + Ranks must be consecutive, and each rank must be bound to its own + device. Supported multi-node layouts contain 2, 4, or 8 IPC domains. """ - self._IS_CAPTURING = False self.disabled = True + self.available = False + self._best_configs = { + collective: {} for collective in ("allreduce", "allgather", "reducescatter") + } + self._best_config_times = { + collective: {} for collective in ("allreduce", "allgather", "reducescatter") + } + self.scratch_buffer = None + self.collectives = frozenset( + ("allreduce", "allgather") if collectives is None else collectives + ) + unsupported_collectives = self.collectives - {"allreduce", "allgather"} + if unsupported_collectives: + raise ValueError( + f"Unsupported MSCCL++ collectives: {unsupported_collectives}" + ) try: self.mscclpp = importlib.import_module("mscclpp") self.mscclpp_ext = importlib.import_module("mscclpp.ext") self.def_algo = importlib.import_module("mscclpp.default_algos") - except ImportError: - self.available = False + except ImportError as exc: + logger.warning( + "PyMscclpp is unavailable because its dependencies failed to import: %s", + exc, + ) self.mscclpp = None return - self.available = True + self._registered_algorithm_configs = _create_algorithm_configs( + self.mscclpp.language + ) + self.group = group assert dist.get_backend(group) != dist.Backend.NCCL, ( "CustomAllreduce should be attached to a non-NCCL group." ) - rank = dist.get_rank(group=self.group) - world_size = dist.get_world_size(group=self.group) - if world_size == 1: + self.rank = dist.get_rank(group=self.group) + self.world_size = dist.get_world_size(group=self.group) + if self.world_size == 1: # No need to initialize mscclpp for single GPU case. return - if world_size not in PyMscclppCommunicator._SUPPORTED_WORLD_SIZES: + if self.world_size not in _SUPPORTED_WORLD_SIZES: logger.warning( "PyMscclpp is disabled due to an unsupported world" " size: %d. Supported world sizes: %s. To silence this " "warning, specify disable_mscclpp=True explicitly.", - world_size, - str(PyMscclppCommunicator._SUPPORTED_WORLD_SIZES), + self.world_size, + str(_SUPPORTED_WORLD_SIZES), ) return self.ranks = torch.distributed.get_process_group_ranks(group) - self.nranks_per_node = torch.cuda.device_count() # for now mscclpp with stride in the communicator is not tested - if not (abs(self.ranks[-1] - self.ranks[0]) == world_size - 1): + if not (abs(self.ranks[-1] - self.ranks[0]) == self.world_size - 1): logger.warning( "PyMscclpp is disabled due to an unsupported group %s." "Please ensure all ranks in the group are consecutive." @@ -307,39 +1101,52 @@ class PyMscclppCommunicator: assert isinstance(device, torch.device) self.device = device - self.rank = rank - self.world_size = world_size self.comm = self.mscclpp.CommGroup( - torch_group=self.group, rank=rank, size=world_size + torch_group=self.group, rank=self.rank, size=self.world_size ) + self.nranks_per_ipc_domain = self.comm.nranks_per_ipc_domain self.executor = self.mscclpp.Executor(self.comm.communicator) self.symm_mem_enabled = self._is_symm_mem_enabled() - self.best_configs = {} - self._create_algorithms() + try: + self._create_algorithms() + self.executor.reset() + except Exception: + logger.exception("Failed to initialize MSCCL++ collectives") + raise + self.available = bool( + self._best_configs["allreduce"] or self._best_configs["allgather"] + ) + if not self.available: + logger.warning("PyMscclpp did not produce any usable tuned configurations.") + else: + logger.info( + "Created MSCCL++ communicator: group=%s world_size=%d device=%s " + "allreduce_configs=%d allgather_configs=%d", + group_name, + self.world_size, + self.device, + len(self._best_configs["allreduce"]), + len(self._best_configs["allgather"]), + ) def destroy(self): - self.algos_config = None - self.best_configs = None + self._best_configs = None + self._best_config_times = None self.executor = None self.scratch_buffer = None - self.flag_buffer = None self.comm = None def should_mscclpp_allreduce( self, inp: torch.Tensor, op: ReduceOp = ReduceOp.SUM ) -> bool: - if ( - self.disabled - or self.world_size not in PyMscclppCommunicator._SUPPORTED_WORLD_SIZES - ): - return False - if inp.dtype not in PyMscclppCommunicator._SUPPORTED_DTYPE: + if self.disabled or self.world_size not in _SUPPORTED_WORLD_SIZES: return False if not self._is_weak_contiguous(inp): return False if op is not ReduceOp.SUM: return False - if self._get_tuned_config(inp.numel() * inp.element_size()) is None: + config = self._get_allreduce_tuned_config(inp.numel() * inp.element_size()) + if config is None or not config.supports_dtype(inp.dtype): return False # mscclpp must not be used during any piecewise CUDA graph phase # (compile, capture, or replay) as it changes the allreduce dispatch @@ -352,6 +1159,36 @@ class PyMscclppCommunicator: return False return True + def should_mscclpp_allgather( + self, + output_tensor: torch.Tensor, + input_tensor: torch.Tensor, + ) -> bool: + if self.disabled or self.world_size not in _SUPPORTED_WORLD_SIZES: + return False + if output_tensor.dtype != input_tensor.dtype: + return False + if not input_tensor.is_contiguous() or not output_tensor.is_contiguous(): + return False + if output_tensor.numel() != input_tensor.numel() * self.world_size: + return False + if output_tensor.device != input_tensor.device: + return False + input_nbytes = input_tensor.numel() * input_tensor.element_size() + if input_nbytes % 16 != 0: + return False + output_nbytes = output_tensor.numel() * output_tensor.element_size() + config = self._get_allgather_tuned_config(output_nbytes) + if config is None or not config.supports_dtype(input_tensor.dtype): + return False + if ( + is_in_tc_piecewise_cuda_graph() + or is_in_torch_compile_warmup() + or get_pcg_capture_stream() is not None + ): + return False + return True + def dtype_to_mscclpp_dtype(self, dtype: torch.dtype): if dtype == torch.float16: return self.mscclpp.DataType.float16 @@ -370,25 +1207,72 @@ class PyMscclppCommunicator: op: ReduceOp = ReduceOp.SUM, stream: torch.cuda.Stream = None, ): - assert op == torch.distributed.ReduceOp.SUM + if op != torch.distributed.ReduceOp.SUM: + raise ValueError("MSCCL++ AllReduce only supports SUM") nbytes = tensor.numel() * tensor.element_size() - algo, nblocks, nthreads = self._get_tuned_config(nbytes) - self._run_algo(algo, tensor, nbytes, nblocks, nthreads, self.symm_mem_enabled) + config = self._get_allreduce_tuned_config(nbytes) + if config is None: + raise RuntimeError( + f"No tuned MSCCL++ AllReduce configuration for {nbytes} bytes" + ) + nblocks, threads_per_block = config.selected_launch() + result = self._run_algorithm( + config, + tensor, + tensor, + nbytes, + nblocks, + threads_per_block, + stream=stream, + ) + if result != 0: + raise RuntimeError(f"MSCCL++ AllReduce failed with error code {result}") return tensor + def all_gather( + self, + output_tensor: torch.Tensor, + input_tensor: torch.Tensor, + stream: torch.cuda.Stream = None, + ): + input_nbytes = input_tensor.numel() * input_tensor.element_size() + output_nbytes = output_tensor.numel() * output_tensor.element_size() + if output_nbytes != input_nbytes * self.world_size: + raise ValueError( + f"MSCCL++ AllGather output has {output_nbytes} bytes; expected " + f"{input_nbytes * self.world_size} bytes" + ) + config = self._get_allgather_tuned_config(output_nbytes) + if config is None: + raise RuntimeError( + "No tuned MSCCL++ AllGather configuration for " + f"{output_nbytes} output bytes" + ) + nblocks, threads_per_block = config.selected_launch() + result = self._run_algorithm( + config, + input_tensor, + output_tensor, + output_nbytes, + nblocks, + threads_per_block, + stream=stream, + ) + if result != 0: + raise RuntimeError(f"MSCCL++ AllGather failed with error code {result}") + return output_tensor + @contextmanager def change_state( self, enable: Optional[bool] = None, ): if enable is None or self.available is False: - # guess a default value when not specified - # DO: Decided if raise an exception here or not enable = self.available old_disable = self.disabled self.disabled = not enable - - yield - - self.disabled = old_disable + try: + yield + finally: + self.disabled = old_disable diff --git a/python/sglang/srt/distributed/parallel_state.py b/python/sglang/srt/distributed/parallel_state.py index e7bcc1d88e..fcf4379c87 100644 --- a/python/sglang/srt/distributed/parallel_state.py +++ b/python/sglang/srt/distributed/parallel_state.py @@ -267,7 +267,7 @@ class GroupCoordinator: cpu_group: ProcessGroup # group for CPU communication device_group: ProcessGroup # group for device communication use_pynccl: bool # a hint of whether to use PyNccl - use_pymscclpp: bool # a hint of whether to use PyMsccl + use_mscclpp: bool # a hint of whether to use MSCCL++ use_custom_allreduce: bool # a hint of whether to use CustomAllreduce use_torch_symm_mem_all_reduce: ( bool # a hint of whether to use TorchSymmMemAllReduce @@ -287,7 +287,7 @@ class GroupCoordinator: local_rank: int, torch_distributed_backend: Union[str, Backend], use_pynccl: bool, - use_pymscclpp: bool, + use_mscclpp: bool, use_custom_allreduce: bool, use_torch_symm_mem_all_reduce: bool, use_hpu_communicator: bool, @@ -424,7 +424,7 @@ class GroupCoordinator: # Import communicators self.use_pynccl = use_pynccl - self.use_pymscclpp = use_pymscclpp + self.use_mscclpp = use_mscclpp self.use_custom_allreduce = use_custom_allreduce self.use_torch_symm_mem_all_reduce = use_torch_symm_mem_all_reduce self.use_hpu_communicator = use_hpu_communicator @@ -471,10 +471,11 @@ class GroupCoordinator: ) self.pymscclpp_comm: Optional[PyMscclppCommunicator] = None - if use_pymscclpp and self.world_size > 1: + if use_mscclpp and self.world_size > 1: self.pymscclpp_comm = PyMscclppCommunicator( group=self.cpu_group, device=self.device, + group_name=group_name, ) self.ca_comm: Optional[Any] = None @@ -1285,6 +1286,13 @@ class GroupCoordinator: ca_comm.all_gather_unreg(input, out=output, dim=0) return + pymscclpp_comm = self.pymscclpp_comm + if pymscclpp_comm is not None and pymscclpp_comm.should_mscclpp_allgather( + output, input + ): + pymscclpp_comm.all_gather(output, input) + return + pynccl_comm = self.pynccl_comm if pynccl_comm is not None and ( not pynccl_comm.disabled or self.is_symmetric_memory_enabled() @@ -2087,7 +2095,7 @@ def init_world_group( local_rank=local_rank, torch_distributed_backend=backend, use_pynccl=False, - use_pymscclpp=False, + use_mscclpp=False, use_custom_allreduce=False, use_torch_symm_mem_all_reduce=False, use_hpu_communicator=False, @@ -2107,7 +2115,7 @@ def init_model_parallel_group( use_custom_allreduce: Optional[bool] = None, use_message_queue_broadcaster: bool = False, group_name: Optional[str] = None, - use_mscclpp_allreduce: Optional[bool] = None, + use_mscclpp: Optional[bool] = None, use_torch_symm_mem_allreduce: Optional[bool] = None, recovered_rank: bool = False, rank_offset: int = 0, @@ -2115,8 +2123,8 @@ def init_model_parallel_group( ) -> GroupCoordinator: if use_custom_allreduce is None: use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE - if use_mscclpp_allreduce is None: - use_mscclpp_allreduce = _ENABLE_MSCCLPP_ALL_REDUCE + if use_mscclpp is None: + use_mscclpp = _ENABLE_MSCCLPP if use_torch_symm_mem_allreduce is None: use_torch_symm_mem_allreduce = _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE return GroupCoordinator( @@ -2128,7 +2136,7 @@ def init_model_parallel_group( if use_pynccl is None else use_pynccl ), - use_pymscclpp=use_mscclpp_allreduce, + use_mscclpp=use_mscclpp, use_custom_allreduce=use_custom_allreduce, use_torch_symm_mem_all_reduce=use_torch_symm_mem_allreduce, use_hpu_communicator=True, @@ -2270,7 +2278,7 @@ def graph_capture(stream=None): logger = logging.getLogger(__name__) _ENABLE_CUSTOM_ALL_REDUCE = True -_ENABLE_MSCCLPP_ALL_REDUCE = False +_ENABLE_MSCCLPP = False _ENABLE_TORCH_SYMM_MEM_ALL_REDUCE = False _ENABLE_FLASHINFER_ALLREDUCE_ONLY = False @@ -2280,9 +2288,9 @@ def set_custom_all_reduce(enable: bool): _ENABLE_CUSTOM_ALL_REDUCE = enable -def set_mscclpp_all_reduce(enable: bool): - global _ENABLE_MSCCLPP_ALL_REDUCE - _ENABLE_MSCCLPP_ALL_REDUCE = enable +def set_mscclpp(enable: bool): + global _ENABLE_MSCCLPP + _ENABLE_MSCCLPP = enable def set_torch_symm_mem_all_reduce(enable: bool): diff --git a/test/registered/unit/distributed/test_parallel_state.py b/test/registered/unit/distributed/test_parallel_state.py index edb86d6547..652f909f8f 100644 --- a/test/registered/unit/distributed/test_parallel_state.py +++ b/test/registered/unit/distributed/test_parallel_state.py @@ -410,7 +410,7 @@ def _read_group_descs(group_name): local_rank=0, torch_distributed_backend="gloo", use_pynccl=False, - use_pymscclpp=False, + use_mscclpp=False, use_custom_allreduce=False, use_torch_symm_mem_all_reduce=False, use_hpu_communicator=False,