[Codex] Diffusion handle non-contiguous CFG communication (#24332)
Co-authored-by: BBuf Codex <bbuf-codex@users.noreply.github.com>
This commit is contained in:
co-authored by
BBuf Codex
parent
c8bc23522f
commit
b67df7cd1b
@@ -61,4 +61,6 @@ def cfg_model_parallel_all_reduce(
|
||||
op: torch._C._distributed_c10d.ReduceOp = torch._C._distributed_c10d.ReduceOp.SUM,
|
||||
) -> torch.Tensor:
|
||||
"""All-reduce the input tensor across CFG parallel group."""
|
||||
if not input_.is_contiguous():
|
||||
input_ = input_.contiguous()
|
||||
return get_cfg_group().all_reduce(input_, op=op)
|
||||
|
||||
@@ -435,6 +435,8 @@ class GroupCoordinator:
|
||||
if self.world_size == 1:
|
||||
return input_
|
||||
# Broadcast.
|
||||
if not input_.is_contiguous():
|
||||
input_ = input_.contiguous()
|
||||
torch.distributed.broadcast(
|
||||
input_,
|
||||
src=self.ranks[src],
|
||||
|
||||
Reference in New Issue
Block a user