[diffusion] fix: fix hybrid SP+TP for I2V models (#41917)
Co-authored-by: Mick <mickjagger19@icloud.com> Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
co-authored by
Mick
Mick Qian
parent
1e86143bf0
commit
d9a0f1cb34
@@ -7,6 +7,7 @@ import torch.distributed as dist
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_decode_parallel_group_coordinator,
|
||||
get_decode_parallel_rank,
|
||||
@@ -78,6 +79,14 @@ def spatial_parallel_decode_disabled() -> bool:
|
||||
return _SPATIAL_PARALLEL_DECODE_DISABLED.get()
|
||||
|
||||
|
||||
def _parallel_rank_and_world_size(
|
||||
parallel_group: GroupCoordinator | None,
|
||||
) -> tuple[int, int]:
|
||||
if parallel_group is not None:
|
||||
return parallel_group.rank_in_group, parallel_group.world_size
|
||||
return get_decode_parallel_rank(), get_decode_parallel_world_size()
|
||||
|
||||
|
||||
def _tensor_pad(x: torch.Tensor, len_to_pad: int, dim: int = -2):
|
||||
return torch.cat(
|
||||
[
|
||||
@@ -191,31 +200,40 @@ def _maybe_contiguous_for_sp_gather(x: torch.Tensor) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
|
||||
def gather_and_trim_height(x: torch.Tensor, expected_height: int | None):
|
||||
def gather_and_trim_height(
|
||||
x: torch.Tensor,
|
||||
expected_height: int | None,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
):
|
||||
if spatial_parallel_decode_disabled():
|
||||
return x
|
||||
if expected_height is None:
|
||||
return x
|
||||
x, _ = gather_variable_height(x)
|
||||
x, _ = gather_variable_height(x, parallel_group=parallel_group)
|
||||
if x.shape[-2] != expected_height:
|
||||
x = x[..., :expected_height, :].contiguous()
|
||||
return x
|
||||
|
||||
|
||||
def gather_height_for_global_op(x: torch.Tensor) -> torch.Tensor:
|
||||
def gather_height_for_global_op(
|
||||
x: torch.Tensor, parallel_group: GroupCoordinator | None = None
|
||||
) -> torch.Tensor:
|
||||
if spatial_parallel_decode_disabled():
|
||||
return x
|
||||
return gather_variable_height(x)[0]
|
||||
return gather_variable_height(x, parallel_group=parallel_group)[0]
|
||||
|
||||
|
||||
def chunk_height_for_parallel_decode(x: torch.Tensor) -> torch.Tensor:
|
||||
def chunk_height_for_parallel_decode(
|
||||
x: torch.Tensor, parallel_group: GroupCoordinator | None = None
|
||||
) -> torch.Tensor:
|
||||
if spatial_parallel_decode_disabled():
|
||||
return x
|
||||
rank, world_size = _parallel_rank_and_world_size(parallel_group)
|
||||
return _tensor_chunk(
|
||||
x,
|
||||
dim=-2,
|
||||
world_size=get_decode_parallel_world_size(),
|
||||
rank=get_decode_parallel_rank(),
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
)
|
||||
|
||||
|
||||
@@ -229,38 +247,50 @@ def chunk_height_by_sizes(x: torch.Tensor, heights: list[int]) -> torch.Tensor:
|
||||
)
|
||||
|
||||
|
||||
def gather_height_sizes(x: torch.Tensor) -> list[int]:
|
||||
def gather_height_sizes(
|
||||
x: torch.Tensor, parallel_group: GroupCoordinator | None = None
|
||||
) -> list[int]:
|
||||
"""gather heights of sharded feature_maps from peers"""
|
||||
if spatial_parallel_decode_disabled():
|
||||
return [x.shape[-2]]
|
||||
world_size = get_decode_parallel_world_size()
|
||||
world_size = (
|
||||
parallel_group.world_size
|
||||
if parallel_group is not None
|
||||
else get_decode_parallel_world_size()
|
||||
)
|
||||
if world_size <= 1:
|
||||
return [x.shape[-2]]
|
||||
parallel_group = parallel_group or get_decode_parallel_group_coordinator()
|
||||
local_height = torch.tensor([x.shape[-2]], device=x.device, dtype=torch.int64)
|
||||
gathered = [torch.empty_like(local_height) for _ in range(world_size)]
|
||||
dist.all_gather(
|
||||
gathered,
|
||||
local_height,
|
||||
group=get_decode_parallel_group_coordinator().device_group,
|
||||
group=parallel_group.device_group,
|
||||
)
|
||||
return [int(height.item()) for height in gathered]
|
||||
|
||||
|
||||
def gather_variable_height(x: torch.Tensor) -> tuple[torch.Tensor, list[int]]:
|
||||
def gather_variable_height(
|
||||
x: torch.Tensor, parallel_group: GroupCoordinator | None = None
|
||||
) -> tuple[torch.Tensor, list[int]]:
|
||||
if spatial_parallel_decode_disabled():
|
||||
return x, [x.shape[-2]]
|
||||
world_size = get_decode_parallel_world_size()
|
||||
world_size = (
|
||||
parallel_group.world_size
|
||||
if parallel_group is not None
|
||||
else get_decode_parallel_world_size()
|
||||
)
|
||||
if world_size <= 1:
|
||||
return x, [x.shape[-2]]
|
||||
parallel_group = parallel_group or get_decode_parallel_group_coordinator()
|
||||
|
||||
heights = gather_height_sizes(x)
|
||||
heights = gather_height_sizes(x, parallel_group=parallel_group)
|
||||
max_height = max(heights)
|
||||
if x.shape[-2] < max_height:
|
||||
x = _tensor_pad(x, max_height - x.shape[-2], dim=-2)
|
||||
|
||||
gathered = get_decode_parallel_group_coordinator().all_gather(
|
||||
_maybe_contiguous_for_sp_gather(x), dim=-2
|
||||
)
|
||||
gathered = parallel_group.all_gather(_maybe_contiguous_for_sp_gather(x), dim=-2)
|
||||
chunks = torch.split(gathered, max_height, dim=-2)
|
||||
return (
|
||||
torch.cat(
|
||||
@@ -305,6 +335,7 @@ def halo_exchange(
|
||||
recv_top_buf: torch.Tensor | None = None,
|
||||
recv_bottom_buf: torch.Tensor | None = None,
|
||||
height_pad_mode: str = "zeros",
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""exchange(send and recv) top/bottom conv-input halos with adjacent spatial ranks"""
|
||||
if spatial_parallel_decode_disabled():
|
||||
@@ -312,11 +343,11 @@ def halo_exchange(
|
||||
if height_halo_size == 0:
|
||||
return x, recv_top_buf, recv_bottom_buf
|
||||
|
||||
decode_group = get_decode_parallel_group_coordinator()
|
||||
rank = get_decode_parallel_rank()
|
||||
world_size = get_decode_parallel_world_size()
|
||||
group = decode_group.device_group
|
||||
group_ranks = decode_group.ranks
|
||||
parallel_group = parallel_group or get_decode_parallel_group_coordinator()
|
||||
rank = parallel_group.rank_in_group
|
||||
world_size = parallel_group.world_size
|
||||
group = parallel_group.device_group
|
||||
group_ranks = parallel_group.ranks
|
||||
|
||||
top_row_ref = x[..., :height_halo_size, :]
|
||||
bottom_row_ref = x[..., -height_halo_size:, :]
|
||||
@@ -472,6 +503,7 @@ def _spatial_parallel_conv_forward(
|
||||
recv_top_buf=module._halo_recv_top_buf,
|
||||
recv_bottom_buf=module._halo_recv_bottom_buf,
|
||||
height_pad_mode=height_pad_mode,
|
||||
parallel_group=module.parallel_group,
|
||||
)
|
||||
if match_conv3d_format:
|
||||
x_padded = _match_conv3d_input_format(x_padded, module.weight)
|
||||
@@ -489,7 +521,7 @@ def _spatial_parallel_conv_forward(
|
||||
):
|
||||
return conv_forward(x_padded)
|
||||
|
||||
heights = gather_height_sizes(x)
|
||||
heights = gather_height_sizes(x, parallel_group=module.parallel_group)
|
||||
global_start = sum(heights[: module.rank])
|
||||
global_height = sum(heights)
|
||||
if stride > 1:
|
||||
@@ -532,6 +564,7 @@ class SpatialParallelConv2d(nn.Conv2d):
|
||||
bias: bool = True,
|
||||
padding_mode: str = "zeros",
|
||||
height_padding: tuple[int, int] | None = None,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
in_channels=in_channels,
|
||||
@@ -558,8 +591,8 @@ class SpatialParallelConv2d(nn.Conv2d):
|
||||
_set_conv_padding(self, (0, self.padding[1]))
|
||||
self._halo_recv_top_buf: torch.Tensor | None = None
|
||||
self._halo_recv_bottom_buf: torch.Tensor | None = None
|
||||
self.rank = get_decode_parallel_rank()
|
||||
self.world_size = get_decode_parallel_world_size()
|
||||
self.parallel_group = parallel_group
|
||||
self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group)
|
||||
|
||||
def forward(self, x):
|
||||
if spatial_parallel_decode_disabled():
|
||||
@@ -604,6 +637,7 @@ class SpatialParallelCausalConv3d(nn.Conv3d):
|
||||
kernel_size: int | tuple[int, int, int],
|
||||
stride: int | tuple[int, int, int] = 1,
|
||||
padding: int | tuple[int, int, int] = 0,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
in_channels=in_channels,
|
||||
@@ -642,8 +676,8 @@ class SpatialParallelCausalConv3d(nn.Conv3d):
|
||||
# Set only by the ROCm Conv3D->Conv2D fast path, to swap the inner
|
||||
# conv without displacing the halo exchange and output trim.
|
||||
self._halo_conv_forward = None
|
||||
self.rank = get_decode_parallel_rank()
|
||||
self.world_size = get_decode_parallel_world_size()
|
||||
self.parallel_group = parallel_group
|
||||
self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group)
|
||||
|
||||
def forward(self, x, cache_x=None):
|
||||
padding = list(self._padding)
|
||||
@@ -696,6 +730,7 @@ class SpatialParallelConv3d(nn.Conv3d):
|
||||
bias: bool = True,
|
||||
padding_mode: str = "zeros",
|
||||
height_padding: tuple[int, int] | None = None,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
in_channels=in_channels,
|
||||
@@ -732,8 +767,8 @@ class SpatialParallelConv3d(nn.Conv3d):
|
||||
# Set only by the ROCm Conv3D->Conv2D fast path, to swap the inner
|
||||
# conv without displacing the halo exchange and output trim.
|
||||
self._halo_conv_forward = None
|
||||
self.rank = get_decode_parallel_rank()
|
||||
self.world_size = get_decode_parallel_world_size()
|
||||
self.parallel_group = parallel_group
|
||||
self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group)
|
||||
|
||||
def forward(self, x):
|
||||
if spatial_parallel_decode_disabled():
|
||||
@@ -785,11 +820,15 @@ class SpatialParallelConv3d(nn.Conv3d):
|
||||
|
||||
|
||||
class SpatialParallelZeroPad2d(nn.Module):
|
||||
def __init__(self, padding: tuple[int, int, int, int]) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
padding: tuple[int, int, int, int],
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.padding = padding
|
||||
self.rank = get_decode_parallel_rank()
|
||||
self.world_size = get_decode_parallel_world_size()
|
||||
self.parallel_group = parallel_group
|
||||
self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if spatial_parallel_decode_disabled():
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
|
||||
import contextvars
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -30,9 +31,11 @@ from sglang.multimodal_gen.configs.models.vaes.base import (
|
||||
should_use_spatial_shard_parallel_decode,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
|
||||
from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_decode_parallel_rank,
|
||||
get_decode_parallel_world_size,
|
||||
get_sp_group,
|
||||
get_sp_parallel_rank,
|
||||
get_sp_world_size,
|
||||
)
|
||||
@@ -764,10 +767,22 @@ class WanAttentionBlock(nn.Module):
|
||||
dim (int): The number of channels in the input tensor.
|
||||
"""
|
||||
|
||||
def __init__(self, dim, *, spatial_parallel: bool = False) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
dim,
|
||||
*,
|
||||
spatial_parallel: bool = False,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.world_size = get_decode_parallel_world_size() if spatial_parallel else 1
|
||||
self.parallel_group = parallel_group
|
||||
if parallel_group is not None:
|
||||
self.world_size = parallel_group.world_size
|
||||
else:
|
||||
self.world_size = (
|
||||
get_decode_parallel_world_size() if spatial_parallel else 1
|
||||
)
|
||||
|
||||
# layers
|
||||
self.norm = WanRMS_norm(dim)
|
||||
@@ -776,10 +791,12 @@ class WanAttentionBlock(nn.Module):
|
||||
|
||||
def forward(self, x):
|
||||
if self.world_size > 1:
|
||||
x = gather_height_for_global_op(x).contiguous()
|
||||
x = gather_height_for_global_op(
|
||||
x, parallel_group=self.parallel_group
|
||||
).contiguous()
|
||||
x = attention_block_forward(self, x)
|
||||
if self.world_size > 1:
|
||||
x = chunk_height_for_parallel_decode(x)
|
||||
x = chunk_height_for_parallel_decode(x, parallel_group=self.parallel_group)
|
||||
return x
|
||||
|
||||
|
||||
@@ -863,13 +880,21 @@ class WanResidualDownBlock(nn.Module):
|
||||
|
||||
|
||||
class WanDistResample(WanResample):
|
||||
def __init__(self, dim: int, mode: str, upsample_out_dim: int = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
mode: str,
|
||||
upsample_out_dim: int = None,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
dim,
|
||||
mode,
|
||||
upsample_out_dim=upsample_out_dim,
|
||||
conv2d_cls=SpatialParallelConv2d,
|
||||
zero_pad2d_cls=SpatialParallelZeroPad2d,
|
||||
conv2d_cls=partial(SpatialParallelConv2d, parallel_group=parallel_group),
|
||||
zero_pad2d_cls=partial(
|
||||
SpatialParallelZeroPad2d, parallel_group=parallel_group
|
||||
),
|
||||
spatial_parallel=True,
|
||||
)
|
||||
|
||||
@@ -881,19 +906,22 @@ class WanDistResidualBlock(WanResidualBlock):
|
||||
out_dim: int,
|
||||
dropout: float = 0.0,
|
||||
non_linearity: str = "silu",
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
in_dim,
|
||||
out_dim,
|
||||
dropout,
|
||||
non_linearity,
|
||||
causal_conv3d_cls=SpatialParallelCausalConv3d,
|
||||
causal_conv3d_cls=partial(
|
||||
SpatialParallelCausalConv3d, parallel_group=parallel_group
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class WanDistAttentionBlock(WanAttentionBlock):
|
||||
def __init__(self, dim) -> None:
|
||||
super().__init__(dim, spatial_parallel=True)
|
||||
def __init__(self, dim, parallel_group: GroupCoordinator | None = None) -> None:
|
||||
super().__init__(dim, spatial_parallel=True, parallel_group=parallel_group)
|
||||
|
||||
|
||||
class WanDistMidBlock(WanMidBlock):
|
||||
@@ -903,14 +931,19 @@ class WanDistMidBlock(WanMidBlock):
|
||||
dropout: float = 0.0,
|
||||
non_linearity: str = "silu",
|
||||
num_layers: int = 1,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
dim,
|
||||
dropout,
|
||||
non_linearity,
|
||||
num_layers=num_layers,
|
||||
residual_block_cls=WanDistResidualBlock,
|
||||
attention_block_cls=WanDistAttentionBlock,
|
||||
residual_block_cls=partial(
|
||||
WanDistResidualBlock, parallel_group=parallel_group
|
||||
),
|
||||
attention_block_cls=partial(
|
||||
WanDistAttentionBlock, parallel_group=parallel_group
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -923,6 +956,7 @@ class WanDistResidualDownBlock(WanResidualDownBlock):
|
||||
num_res_blocks,
|
||||
temperal_downsample=False,
|
||||
down_flag=False,
|
||||
parallel_group: GroupCoordinator | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
in_dim,
|
||||
@@ -931,8 +965,10 @@ class WanDistResidualDownBlock(WanResidualDownBlock):
|
||||
num_res_blocks,
|
||||
temperal_downsample=temperal_downsample,
|
||||
down_flag=down_flag,
|
||||
residual_block_cls=WanDistResidualBlock,
|
||||
resample_cls=WanDistResample,
|
||||
residual_block_cls=partial(
|
||||
WanDistResidualBlock, parallel_group=parallel_group
|
||||
),
|
||||
resample_cls=partial(WanDistResample, parallel_group=parallel_group),
|
||||
)
|
||||
|
||||
|
||||
@@ -986,12 +1022,21 @@ class WanEncoder3d(nn.Module):
|
||||
world_size = get_sp_world_size()
|
||||
|
||||
if use_parallel_encode and world_size > 1:
|
||||
CausalConv3d = SpatialParallelCausalConv3d
|
||||
ResidualDownBlock = WanDistResidualDownBlock
|
||||
ResidualBlock = WanDistResidualBlock
|
||||
AttentionBlock = WanDistAttentionBlock
|
||||
Resample = WanDistResample
|
||||
MidBlock = WanDistMidBlock
|
||||
self.parallel_group = get_sp_group()
|
||||
CausalConv3d = partial(
|
||||
SpatialParallelCausalConv3d, parallel_group=self.parallel_group
|
||||
)
|
||||
ResidualDownBlock = partial(
|
||||
WanDistResidualDownBlock, parallel_group=self.parallel_group
|
||||
)
|
||||
ResidualBlock = partial(
|
||||
WanDistResidualBlock, parallel_group=self.parallel_group
|
||||
)
|
||||
AttentionBlock = partial(
|
||||
WanDistAttentionBlock, parallel_group=self.parallel_group
|
||||
)
|
||||
Resample = partial(WanDistResample, parallel_group=self.parallel_group)
|
||||
MidBlock = partial(WanDistMidBlock, parallel_group=self.parallel_group)
|
||||
else:
|
||||
CausalConv3d = WanCausalConv3d
|
||||
ResidualDownBlock = WanResidualDownBlock
|
||||
@@ -1091,7 +1136,9 @@ class WanEncoder3d(nn.Module):
|
||||
x = self.conv_out(x)
|
||||
|
||||
if self.use_parallel_encode and self.world_size > 1:
|
||||
x = gather_and_trim_height(x, expected_height)
|
||||
x = gather_and_trim_height(
|
||||
x, expected_height, parallel_group=self.parallel_group
|
||||
)
|
||||
return x
|
||||
|
||||
|
||||
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
"""Wan VAE communication must follow the group that shards its activations."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
destroy_model_parallel,
|
||||
get_decode_parallel_group_coordinator,
|
||||
get_sp_group,
|
||||
get_tp_group,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.parallel_conv import (
|
||||
SpatialParallelConv2d,
|
||||
chunk_height_for_parallel_decode,
|
||||
gather_variable_height,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.wanvae import (
|
||||
WanDecoder3d,
|
||||
WanEncoder3d,
|
||||
)
|
||||
|
||||
|
||||
def _worker(sp_size, tp_size):
|
||||
rank = int(os.environ["LOCAL_RANK"])
|
||||
torch.cuda.set_device(rank)
|
||||
init_distributed_environment(world_size=4, rank=rank, local_rank=rank)
|
||||
initialize_model_parallel(
|
||||
tensor_parallel_degree=tp_size,
|
||||
sequence_parallel_degree=sp_size,
|
||||
ulysses_degree=sp_size,
|
||||
ring_degree=1,
|
||||
)
|
||||
torch.backends.cuda.matmul.allow_tf32 = False
|
||||
torch.backends.cudnn.allow_tf32 = False
|
||||
sp_group = get_sp_group()
|
||||
decode_group = get_decode_parallel_group_coordinator()
|
||||
assert sp_group.world_size == sp_size
|
||||
assert decode_group.world_size == 4
|
||||
|
||||
# distinct TP replicas catch collectives that accidentally use the decode group
|
||||
torch.manual_seed(42)
|
||||
full = torch.randn(1, 2, 9, 8, device="cuda") + get_tp_group().rank_in_group
|
||||
local = chunk_height_for_parallel_decode(full, parallel_group=sp_group)
|
||||
gathered, heights = gather_variable_height(local, parallel_group=sp_group)
|
||||
torch.testing.assert_close(gathered, full, rtol=0, atol=0)
|
||||
assert heights == [part.shape[-2] for part in torch.tensor_split(full, sp_size, -2)]
|
||||
conv = SpatialParallelConv2d(
|
||||
2, 3, 3, stride=2, padding=1, parallel_group=sp_group
|
||||
).cuda()
|
||||
expected = F.conv2d(full, conv.weight, conv.bias, stride=2, padding=1)
|
||||
actual, _ = gather_variable_height(conv(local), parallel_group=sp_group)
|
||||
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
|
||||
|
||||
# exercise real Wan block construction, attention, downsampling and final gather
|
||||
for residual in (False, True):
|
||||
torch.manual_seed(123)
|
||||
kwargs = dict(
|
||||
dim=4,
|
||||
z_dim=2,
|
||||
dim_mult=(1, 2, 2),
|
||||
num_res_blocks=1,
|
||||
temperal_downsample=(False, False),
|
||||
is_residual=residual,
|
||||
)
|
||||
reference = WanEncoder3d(**kwargs).cuda().eval()
|
||||
parallel = WanEncoder3d(**kwargs, use_parallel_encode=True).cuda().eval()
|
||||
parallel.load_state_dict(reference.state_dict())
|
||||
if sp_size > 1:
|
||||
assert parallel.parallel_group is sp_group
|
||||
assert parallel.conv_in.parallel_group is sp_group
|
||||
assert parallel.conv_out.parallel_group is sp_group
|
||||
x = torch.randn(1, 3, 3, 32, 16, device="cuda") + get_tp_group().rank_in_group
|
||||
with torch.inference_mode():
|
||||
torch.testing.assert_close(parallel(x), reference(x), rtol=1e-4, atol=1e-4)
|
||||
|
||||
# default decoder callers must still use the full decode group, not SP
|
||||
torch.manual_seed(321)
|
||||
kwargs = dict(
|
||||
dim=4, z_dim=2, dim_mult=(1, 1), num_res_blocks=1, temperal_upsample=(False,)
|
||||
)
|
||||
reference = WanDecoder3d(**kwargs).cuda().eval()
|
||||
parallel = WanDecoder3d(**kwargs, use_parallel_decode=True).cuda().eval()
|
||||
parallel.load_state_dict(reference.state_dict())
|
||||
assert parallel.conv_in.parallel_group is None
|
||||
assert parallel.conv_in.world_size == decode_group.world_size
|
||||
z = torch.randn(1, 2, 1, 8, 4, device="cuda")
|
||||
with torch.inference_mode():
|
||||
torch.testing.assert_close(parallel(z), reference(z), rtol=1e-4, atol=1e-4)
|
||||
destroy_model_parallel()
|
||||
torch.distributed.destroy_process_group()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sp_size,tp_size", [(1, 4), (2, 2), (4, 1)])
|
||||
def test_wan_vae_parallel_groups(sp_size, tp_size):
|
||||
if not torch.cuda.is_available() or torch.cuda.device_count() < 4:
|
||||
pytest.fail("requires four CUDA GPUs")
|
||||
process = subprocess.Popen(
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"torch.distributed.run",
|
||||
"--standalone",
|
||||
"--nproc-per-node=4",
|
||||
__file__,
|
||||
"--worker",
|
||||
"--sp-size",
|
||||
str(sp_size),
|
||||
"--tp-size",
|
||||
str(tp_size),
|
||||
],
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
returncode = process.wait(timeout=180)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
raise
|
||||
if returncode:
|
||||
raise subprocess.CalledProcessError(returncode, process.args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "--worker" in sys.argv:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--worker", action="store_true")
|
||||
parser.add_argument("--sp-size", type=int, required=True)
|
||||
parser.add_argument("--tp-size", type=int, required=True)
|
||||
args = parser.parse_args()
|
||||
_worker(args.sp_size, args.tp_size)
|
||||
else:
|
||||
sys.exit(pytest.main([__file__, *sys.argv[1:]]))
|
||||
Reference in New Issue
Block a user