[diffusion] feat: keep the TP all-reduce inside the graph (#43602)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
664e4c6499
commit
5e872557cb
@@ -8,11 +8,12 @@
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||
import pickle
|
||||
import weakref
|
||||
from collections import namedtuple
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from pkgutil import resolve_name
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
import torch
|
||||
import torch.distributed
|
||||
@@ -28,6 +29,7 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import (
|
||||
suppress_stdout,
|
||||
)
|
||||
from sglang.srt.utils import is_shm_available
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -73,6 +75,17 @@ def _get_unique_name(name: str) -> str:
|
||||
return newname
|
||||
|
||||
|
||||
_groups: dict[str, Callable[[], "GroupCoordinator | None"]] = {}
|
||||
|
||||
|
||||
# Opaque to torch.compile: Dynamo cannot trace the custom all-reduce's
|
||||
# communicator, so calling it inline breaks the graph at every row-parallel
|
||||
# layer. srt routes its all-reduce the same way; ops take the group by name.
|
||||
@register_custom_op(out_shape="tensor")
|
||||
def diffusion_all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
|
||||
return _groups[group_name]()._all_reduce_out_of_place(tensor)
|
||||
|
||||
|
||||
def _split_tensor_dict(
|
||||
tensor_dict: Dict[str, Union[torch.Tensor, Any]], prefix: str = ""
|
||||
) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
|
||||
@@ -182,6 +195,7 @@ class GroupCoordinator:
|
||||
group_name: str | None = None,
|
||||
):
|
||||
self.unique_name = _get_unique_name(group_name)
|
||||
_groups[self.unique_name] = weakref.ref(self)
|
||||
self.rank = torch.distributed.get_rank()
|
||||
self.local_rank = local_rank
|
||||
self.device_group = None
|
||||
@@ -357,6 +371,14 @@ class GroupCoordinator:
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if self.world_size == 1:
|
||||
return input_
|
||||
elif (
|
||||
torch.compiler.is_compiling()
|
||||
and self.srt_custom_allreduce is not None
|
||||
and not async_op
|
||||
and op == torch.distributed.ReduceOp.SUM
|
||||
and not input_.is_cpu
|
||||
):
|
||||
return diffusion_all_reduce(input_, group_name=self.unique_name)
|
||||
else:
|
||||
custom_ar = self.srt_custom_allreduce
|
||||
if (
|
||||
@@ -385,6 +407,17 @@ class GroupCoordinator:
|
||||
)
|
||||
return input_
|
||||
|
||||
def _all_reduce_out_of_place(self, input_: torch.Tensor) -> torch.Tensor:
|
||||
"""Sum over the group into a new tensor; the input is left untouched."""
|
||||
custom_ar = self.srt_custom_allreduce
|
||||
if not custom_ar.disabled and custom_ar.should_custom_ar(input_):
|
||||
output = custom_ar.custom_all_reduce(input_)
|
||||
if output is not None:
|
||||
return output
|
||||
output = input_.clone()
|
||||
torch.distributed.all_reduce(output, group=self.device_group)
|
||||
return output
|
||||
|
||||
def all_gather(
|
||||
self, input_: torch.Tensor, dim: int = 0, separate_tensors: bool = False
|
||||
) -> Union[torch.Tensor, List[torch.Tensor]]:
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""A compiled TP all-reduce must not graph-break on the custom all-reduce."""
|
||||
|
||||
import unittest
|
||||
import weakref
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import group_coordinator
|
||||
from sglang.multimodal_gen.runtime.distributed.group_coordinator import (
|
||||
GroupCoordinator,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class _Group(SimpleNamespace):
|
||||
"""Weak-referenceable, as the op registry holds groups by weak reference."""
|
||||
|
||||
def _all_reduce_out_of_place(self, tensor):
|
||||
return GroupCoordinator._all_reduce_out_of_place(self, tensor)
|
||||
|
||||
|
||||
def _group(name: str, *, accepts: bool) -> _Group:
|
||||
custom_ar = SimpleNamespace(
|
||||
disabled=False,
|
||||
should_custom_ar=lambda tensor: accepts,
|
||||
# Stands in for the tvm_ffi communicator call Dynamo cannot trace.
|
||||
custom_all_reduce=torch._dynamo.disable(lambda tensor: tensor * 2),
|
||||
)
|
||||
group = _Group(
|
||||
world_size=2,
|
||||
unique_name=name,
|
||||
srt_custom_allreduce=custom_ar,
|
||||
device_group=None,
|
||||
)
|
||||
group_coordinator._groups[name] = weakref.ref(group)
|
||||
return group
|
||||
|
||||
|
||||
class TestCompiledAllReduce(CustomTestCase):
|
||||
def test_compiled_all_reduce_is_one_graph(self):
|
||||
group = _group("tp:compiled", accepts=True)
|
||||
compiled = torch.compile(
|
||||
lambda x: GroupCoordinator.all_reduce(group, x),
|
||||
fullgraph=True,
|
||||
backend="eager",
|
||||
)
|
||||
|
||||
out = compiled(torch.empty(4, 8, device="meta"))
|
||||
|
||||
self.assertEqual(out.shape, (4, 8))
|
||||
|
||||
def test_op_leaves_its_input_untouched(self):
|
||||
schema = torch.ops.sglang.diffusion_all_reduce.default._schema
|
||||
self.assertFalse(
|
||||
any(a.alias_info and a.alias_info.is_write for a in schema.arguments)
|
||||
)
|
||||
|
||||
@unittest.skipUnless(torch.cuda.is_available(), "the op is registered for CUDA")
|
||||
def test_op_reaches_the_group_by_name(self):
|
||||
group = _group("tp:dispatch", accepts=True)
|
||||
x = torch.ones(4, device="cuda")
|
||||
|
||||
out = torch.ops.sglang.diffusion_all_reduce(x, group.unique_name)
|
||||
|
||||
torch.testing.assert_close(out, x * 2)
|
||||
torch.testing.assert_close(x, torch.ones(4, device="cuda"))
|
||||
|
||||
def test_out_of_place_reduce_uses_the_custom_all_reduce(self):
|
||||
group = _group("tp:custom", accepts=True)
|
||||
x = torch.ones(4)
|
||||
|
||||
torch.testing.assert_close(group._all_reduce_out_of_place(x), x * 2)
|
||||
|
||||
def test_out_of_place_fallback_reduces_a_copy(self):
|
||||
group = _group("tp:fallback", accepts=False)
|
||||
x = torch.ones(4)
|
||||
|
||||
with patch.object(
|
||||
group_coordinator.torch.distributed,
|
||||
"all_reduce",
|
||||
side_effect=lambda tensor, group: tensor.mul_(3),
|
||||
):
|
||||
out = group._all_reduce_out_of_place(x)
|
||||
|
||||
torch.testing.assert_close(out, torch.full((4,), 3.0))
|
||||
torch.testing.assert_close(x, torch.ones(4))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user