[diffusion] feat: keep the TP all-reduce inside the graph (#43602)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-11 08:45:54 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 664e4c6499
commit 5e872557cb
2 changed files with 126 additions and 1 deletions
@@ -8,11 +8,12 @@
# Copyright 2023 The vLLM team.
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
import pickle
import weakref
from collections import namedtuple
from contextlib import contextmanager, nullcontext
from dataclasses import dataclass
from pkgutil import resolve_name
from typing import Any, Dict, List, Optional, Tuple, Union
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
import torch
import torch.distributed
@@ -28,6 +29,7 @@ from sglang.multimodal_gen.runtime.utils.logging_utils import (
suppress_stdout,
)
from sglang.srt.utils import is_shm_available
from sglang.srt.utils.custom_op import register_custom_op
logger = init_logger(__name__)
@@ -73,6 +75,17 @@ def _get_unique_name(name: str) -> str:
return newname
_groups: dict[str, Callable[[], "GroupCoordinator | None"]] = {}
# Opaque to torch.compile: Dynamo cannot trace the custom all-reduce's
# communicator, so calling it inline breaks the graph at every row-parallel
# layer. srt routes its all-reduce the same way; ops take the group by name.
@register_custom_op(out_shape="tensor")
def diffusion_all_reduce(tensor: torch.Tensor, group_name: str) -> torch.Tensor:
return _groups[group_name]()._all_reduce_out_of_place(tensor)
def _split_tensor_dict(
tensor_dict: Dict[str, Union[torch.Tensor, Any]], prefix: str = ""
) -> Tuple[List[Tuple[str, Any]], List[torch.Tensor]]:
@@ -182,6 +195,7 @@ class GroupCoordinator:
group_name: str | None = None,
):
self.unique_name = _get_unique_name(group_name)
_groups[self.unique_name] = weakref.ref(self)
self.rank = torch.distributed.get_rank()
self.local_rank = local_rank
self.device_group = None
@@ -357,6 +371,14 @@ class GroupCoordinator:
# Bypass the function if we are using only 1 GPU.
if self.world_size == 1:
return input_
elif (
torch.compiler.is_compiling()
and self.srt_custom_allreduce is not None
and not async_op
and op == torch.distributed.ReduceOp.SUM
and not input_.is_cpu
):
return diffusion_all_reduce(input_, group_name=self.unique_name)
else:
custom_ar = self.srt_custom_allreduce
if (
@@ -385,6 +407,17 @@ class GroupCoordinator:
)
return input_
def _all_reduce_out_of_place(self, input_: torch.Tensor) -> torch.Tensor:
"""Sum over the group into a new tensor; the input is left untouched."""
custom_ar = self.srt_custom_allreduce
if not custom_ar.disabled and custom_ar.should_custom_ar(input_):
output = custom_ar.custom_all_reduce(input_)
if output is not None:
return output
output = input_.clone()
torch.distributed.all_reduce(output, group=self.device_group)
return output
def all_gather(
self, input_: torch.Tensor, dim: int = 0, separate_tensors: bool = False
) -> Union[torch.Tensor, List[torch.Tensor]]:
@@ -0,0 +1,92 @@
"""A compiled TP all-reduce must not graph-break on the custom all-reduce."""
import unittest
import weakref
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.multimodal_gen.runtime.distributed import group_coordinator
from sglang.multimodal_gen.runtime.distributed.group_coordinator import (
GroupCoordinator,
)
from sglang.test.test_utils import CustomTestCase
class _Group(SimpleNamespace):
"""Weak-referenceable, as the op registry holds groups by weak reference."""
def _all_reduce_out_of_place(self, tensor):
return GroupCoordinator._all_reduce_out_of_place(self, tensor)
def _group(name: str, *, accepts: bool) -> _Group:
custom_ar = SimpleNamespace(
disabled=False,
should_custom_ar=lambda tensor: accepts,
# Stands in for the tvm_ffi communicator call Dynamo cannot trace.
custom_all_reduce=torch._dynamo.disable(lambda tensor: tensor * 2),
)
group = _Group(
world_size=2,
unique_name=name,
srt_custom_allreduce=custom_ar,
device_group=None,
)
group_coordinator._groups[name] = weakref.ref(group)
return group
class TestCompiledAllReduce(CustomTestCase):
def test_compiled_all_reduce_is_one_graph(self):
group = _group("tp:compiled", accepts=True)
compiled = torch.compile(
lambda x: GroupCoordinator.all_reduce(group, x),
fullgraph=True,
backend="eager",
)
out = compiled(torch.empty(4, 8, device="meta"))
self.assertEqual(out.shape, (4, 8))
def test_op_leaves_its_input_untouched(self):
schema = torch.ops.sglang.diffusion_all_reduce.default._schema
self.assertFalse(
any(a.alias_info and a.alias_info.is_write for a in schema.arguments)
)
@unittest.skipUnless(torch.cuda.is_available(), "the op is registered for CUDA")
def test_op_reaches_the_group_by_name(self):
group = _group("tp:dispatch", accepts=True)
x = torch.ones(4, device="cuda")
out = torch.ops.sglang.diffusion_all_reduce(x, group.unique_name)
torch.testing.assert_close(out, x * 2)
torch.testing.assert_close(x, torch.ones(4, device="cuda"))
def test_out_of_place_reduce_uses_the_custom_all_reduce(self):
group = _group("tp:custom", accepts=True)
x = torch.ones(4)
torch.testing.assert_close(group._all_reduce_out_of_place(x), x * 2)
def test_out_of_place_fallback_reduces_a_copy(self):
group = _group("tp:fallback", accepts=False)
x = torch.ones(4)
with patch.object(
group_coordinator.torch.distributed,
"all_reduce",
side_effect=lambda tensor, group: tensor.mul_(3),
):
out = group._all_reduce_out_of_place(x)
torch.testing.assert_close(out, torch.full((4,), 3.0))
torch.testing.assert_close(x, torch.ones(4))
if __name__ == "__main__":
unittest.main()