diff --git a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py index d8cf0c8a88..cb08879d67 100644 --- a/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py +++ b/python/sglang/multimodal_gen/runtime/distributed/group_coordinator.py @@ -8,11 +8,12 @@ # Copyright 2023 The vLLM team. # Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. import pickle +import weakref from collections import namedtuple from contextlib import contextmanager, nullcontext from dataclasses import dataclass from pkgutil import resolve_name -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union import torch import torch.distributed @@ -28,6 +29,7 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import ( suppress_stdout, ) from sglang.srt.utils import is_shm_available +from sglang.srt.utils.custom_op import register_custom_op logger = init_logger(__name__) @@ -73,6 +75,17 @@ def _get_unique_name(name: str) -> str: return newname +_groups: dict[str, Callable[[], "GroupCoordinator | None"]] = {} + + +# Opaque to torch.compile: Dynamo cannot trace the custom all-reduce's +# communicator, so calling it inline breaks the graph at every row-parallel +# layer. srt routes its all-reduce the same way; ops take the group by name. +@register_custom_op(out_shape="tensor") +def diffusion_all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor: + return _groups[group_name]()._all_reduce_out_of_place(tensor) + + def _split_tensor_dict( tensor_dict: Dict[str, Union[torch.Tensor, Any]], prefix: str = "" ) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]: @@ -182,6 +195,7 @@ class GroupCoordinator: group_name: str | None = None, ): self.unique_name = _get_unique_name(group_name) + _groups[self.unique_name] = weakref.ref(self) self.rank = torch.distributed.get_rank() self.local_rank = local_rank self.device_group = None @@ -357,6 +371,14 @@ class GroupCoordinator: # Bypass the function if we are using only 1 GPU. if self.world_size == 1: return input_ + elif ( + torch.compiler.is_compiling() + and self.srt_custom_allreduce is not None + and not async_op + and op == torch.distributed.ReduceOp.SUM + and not input_.is_cpu + ): + return diffusion_all_reduce(input_, group_name=self.unique_name) else: custom_ar = self.srt_custom_allreduce if ( @@ -385,6 +407,17 @@ class GroupCoordinator: ) return input_ + def _all_reduce_out_of_place(self, input_: torch.Tensor) -> torch.Tensor: + """Sum over the group into a new tensor; the input is left untouched.""" + custom_ar = self.srt_custom_allreduce + if not custom_ar.disabled and custom_ar.should_custom_ar(input_): + output = custom_ar.custom_all_reduce(input_) + if output is not None: + return output + output = input_.clone() + torch.distributed.all_reduce(output, group=self.device_group) + return output + def all_gather( self, input_: torch.Tensor, dim: int = 0, separate_tensors: bool = False ) -> Union[torch.Tensor, List[torch.Tensor]]: diff --git a/python/sglang/multimodal_gen/test/unit/test_group_coordinator_compiled_all_reduce.py b/python/sglang/multimodal_gen/test/unit/test_group_coordinator_compiled_all_reduce.py new file mode 100644 index 0000000000..531c3bdf95 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_group_coordinator_compiled_all_reduce.py @@ -0,0 +1,92 @@ +"""A compiled TP all-reduce must not graph-break on the custom all-reduce.""" + +import unittest +import weakref +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.multimodal_gen.runtime.distributed import group_coordinator +from sglang.multimodal_gen.runtime.distributed.group_coordinator import ( + GroupCoordinator, +) +from sglang.test.test_utils import CustomTestCase + + +class _Group(SimpleNamespace): + """Weak-referenceable, as the op registry holds groups by weak reference.""" + + def _all_reduce_out_of_place(self, tensor): + return GroupCoordinator._all_reduce_out_of_place(self, tensor) + + +def _group(name: str, *, accepts: bool) -> _Group: + custom_ar = SimpleNamespace( + disabled=False, + should_custom_ar=lambda tensor: accepts, + # Stands in for the tvm_ffi communicator call Dynamo cannot trace. + custom_all_reduce=torch._dynamo.disable(lambda tensor: tensor * 2), + ) + group = _Group( + world_size=2, + unique_name=name, + srt_custom_allreduce=custom_ar, + device_group=None, + ) + group_coordinator._groups[name] = weakref.ref(group) + return group + + +class TestCompiledAllReduce(CustomTestCase): + def test_compiled_all_reduce_is_one_graph(self): + group = _group("tp:compiled", accepts=True) + compiled = torch.compile( + lambda x: GroupCoordinator.all_reduce(group, x), + fullgraph=True, + backend="eager", + ) + + out = compiled(torch.empty(4, 8, device="meta")) + + self.assertEqual(out.shape, (4, 8)) + + def test_op_leaves_its_input_untouched(self): + schema = torch.ops.sglang.diffusion_all_reduce.default._schema + self.assertFalse( + any(a.alias_info and a.alias_info.is_write for a in schema.arguments) + ) + + @unittest.skipUnless(torch.cuda.is_available(), "the op is registered for CUDA") + def test_op_reaches_the_group_by_name(self): + group = _group("tp:dispatch", accepts=True) + x = torch.ones(4, device="cuda") + + out = torch.ops.sglang.diffusion_all_reduce(x, group.unique_name) + + torch.testing.assert_close(out, x * 2) + torch.testing.assert_close(x, torch.ones(4, device="cuda")) + + def test_out_of_place_reduce_uses_the_custom_all_reduce(self): + group = _group("tp:custom", accepts=True) + x = torch.ones(4) + + torch.testing.assert_close(group._all_reduce_out_of_place(x), x * 2) + + def test_out_of_place_fallback_reduces_a_copy(self): + group = _group("tp:fallback", accepts=False) + x = torch.ones(4) + + with patch.object( + group_coordinator.torch.distributed, + "all_reduce", + side_effect=lambda tensor, group: tensor.mul_(3), + ): + out = group._all_reduce_out_of_place(x) + + torch.testing.assert_close(out, torch.full((4,), 3.0)) + torch.testing.assert_close(x, torch.ones(4)) + + +if __name__ == "__main__": + unittest.main()