[diffusion] fix: fix hybrid SP+TP for I2V models (#41917)

Co-authored-by: Mick <mickjagger19@icloud.com>
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Артем Савкин
2026-10-07 22:33:03 +08:00
committed by GitHub
co-authored by Mick Mick Qian
parent 1e86143bf0
commit d9a0f1cb34
4 changed files with 289 additions and 53 deletions
@@ -7,6 +7,7 @@ import torch.distributed as dist
import torch.nn as nn
import torch.nn.functional as F
from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_decode_parallel_group_coordinator,
get_decode_parallel_rank,
@@ -78,6 +79,14 @@ def spatial_parallel_decode_disabled() -> bool:
return _SPATIAL_PARALLEL_DECODE_DISABLED.get()
def _parallel_rank_and_world_size(
parallel_group: GroupCoordinator | None,
) -> tuple[int, int]:
if parallel_group is not None:
return parallel_group.rank_in_group, parallel_group.world_size
return get_decode_parallel_rank(), get_decode_parallel_world_size()
def _tensor_pad(x: torch.Tensor, len_to_pad: int, dim: int = -2):
return torch.cat(
[
@@ -191,31 +200,40 @@ def _maybe_contiguous_for_sp_gather(x: torch.Tensor) -> torch.Tensor:
return x
def gather_and_trim_height(x: torch.Tensor, expected_height: int | None):
def gather_and_trim_height(
x: torch.Tensor,
expected_height: int | None,
parallel_group: GroupCoordinator | None = None,
):
if spatial_parallel_decode_disabled():
return x
if expected_height is None:
return x
x, _ = gather_variable_height(x)
x, _ = gather_variable_height(x, parallel_group=parallel_group)
if x.shape[-2] != expected_height:
x = x[..., :expected_height, :].contiguous()
return x
def gather_height_for_global_op(x: torch.Tensor) -> torch.Tensor:
def gather_height_for_global_op(
x: torch.Tensor, parallel_group: GroupCoordinator | None = None
) -> torch.Tensor:
if spatial_parallel_decode_disabled():
return x
return gather_variable_height(x)[0]
return gather_variable_height(x, parallel_group=parallel_group)[0]
def chunk_height_for_parallel_decode(x: torch.Tensor) -> torch.Tensor:
def chunk_height_for_parallel_decode(
x: torch.Tensor, parallel_group: GroupCoordinator | None = None
) -> torch.Tensor:
if spatial_parallel_decode_disabled():
return x
rank, world_size = _parallel_rank_and_world_size(parallel_group)
return _tensor_chunk(
x,
dim=-2,
world_size=get_decode_parallel_world_size(),
rank=get_decode_parallel_rank(),
world_size=world_size,
rank=rank,
)
@@ -229,38 +247,50 @@ def chunk_height_by_sizes(x: torch.Tensor, heights: list[int]) -> torch.Tensor:
)
def gather_height_sizes(x: torch.Tensor) -> list[int]:
def gather_height_sizes(
x: torch.Tensor, parallel_group: GroupCoordinator | None = None
) -> list[int]:
"""gather heights of sharded feature_maps from peers"""
if spatial_parallel_decode_disabled():
return [x.shape[-2]]
world_size = get_decode_parallel_world_size()
world_size = (
parallel_group.world_size
if parallel_group is not None
else get_decode_parallel_world_size()
)
if world_size <= 1:
return [x.shape[-2]]
parallel_group = parallel_group or get_decode_parallel_group_coordinator()
local_height = torch.tensor([x.shape[-2]], device=x.device, dtype=torch.int64)
gathered = [torch.empty_like(local_height) for _ in range(world_size)]
dist.all_gather(
gathered,
local_height,
group=get_decode_parallel_group_coordinator().device_group,
group=parallel_group.device_group,
)
return [int(height.item()) for height in gathered]
def gather_variable_height(x: torch.Tensor) -> tuple[torch.Tensor, list[int]]:
def gather_variable_height(
x: torch.Tensor, parallel_group: GroupCoordinator | None = None
) -> tuple[torch.Tensor, list[int]]:
if spatial_parallel_decode_disabled():
return x, [x.shape[-2]]
world_size = get_decode_parallel_world_size()
world_size = (
parallel_group.world_size
if parallel_group is not None
else get_decode_parallel_world_size()
)
if world_size <= 1:
return x, [x.shape[-2]]
parallel_group = parallel_group or get_decode_parallel_group_coordinator()
heights = gather_height_sizes(x)
heights = gather_height_sizes(x, parallel_group=parallel_group)
max_height = max(heights)
if x.shape[-2] < max_height:
x = _tensor_pad(x, max_height - x.shape[-2], dim=-2)
gathered = get_decode_parallel_group_coordinator().all_gather(
_maybe_contiguous_for_sp_gather(x), dim=-2
)
gathered = parallel_group.all_gather(_maybe_contiguous_for_sp_gather(x), dim=-2)
chunks = torch.split(gathered, max_height, dim=-2)
return (
torch.cat(
@@ -305,6 +335,7 @@ def halo_exchange(
recv_top_buf: torch.Tensor | None = None,
recv_bottom_buf: torch.Tensor | None = None,
height_pad_mode: str = "zeros",
parallel_group: GroupCoordinator | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""exchange(send and recv) top/bottom conv-input halos with adjacent spatial ranks"""
if spatial_parallel_decode_disabled():
@@ -312,11 +343,11 @@ def halo_exchange(
if height_halo_size == 0:
return x, recv_top_buf, recv_bottom_buf
decode_group = get_decode_parallel_group_coordinator()
rank = get_decode_parallel_rank()
world_size = get_decode_parallel_world_size()
group = decode_group.device_group
group_ranks = decode_group.ranks
parallel_group = parallel_group or get_decode_parallel_group_coordinator()
rank = parallel_group.rank_in_group
world_size = parallel_group.world_size
group = parallel_group.device_group
group_ranks = parallel_group.ranks
top_row_ref = x[..., :height_halo_size, :]
bottom_row_ref = x[..., -height_halo_size:, :]
@@ -472,6 +503,7 @@ def _spatial_parallel_conv_forward(
recv_top_buf=module._halo_recv_top_buf,
recv_bottom_buf=module._halo_recv_bottom_buf,
height_pad_mode=height_pad_mode,
parallel_group=module.parallel_group,
)
if match_conv3d_format:
x_padded = _match_conv3d_input_format(x_padded, module.weight)
@@ -489,7 +521,7 @@ def _spatial_parallel_conv_forward(
):
return conv_forward(x_padded)
heights = gather_height_sizes(x)
heights = gather_height_sizes(x, parallel_group=module.parallel_group)
global_start = sum(heights[: module.rank])
global_height = sum(heights)
if stride > 1:
@@ -532,6 +564,7 @@ class SpatialParallelConv2d(nn.Conv2d):
bias: bool = True,
padding_mode: str = "zeros",
height_padding: tuple[int, int] | None = None,
parallel_group: GroupCoordinator | None = None,
):
super().__init__(
in_channels=in_channels,
@@ -558,8 +591,8 @@ class SpatialParallelConv2d(nn.Conv2d):
_set_conv_padding(self, (0, self.padding[1]))
self._halo_recv_top_buf: torch.Tensor | None = None
self._halo_recv_bottom_buf: torch.Tensor | None = None
self.rank = get_decode_parallel_rank()
self.world_size = get_decode_parallel_world_size()
self.parallel_group = parallel_group
self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group)
def forward(self, x):
if spatial_parallel_decode_disabled():
@@ -604,6 +637,7 @@ class SpatialParallelCausalConv3d(nn.Conv3d):
kernel_size: int | tuple[int, int, int],
stride: int | tuple[int, int, int] = 1,
padding: int | tuple[int, int, int] = 0,
parallel_group: GroupCoordinator | None = None,
):
super().__init__(
in_channels=in_channels,
@@ -642,8 +676,8 @@ class SpatialParallelCausalConv3d(nn.Conv3d):
# Set only by the ROCm Conv3D->Conv2D fast path, to swap the inner
# conv without displacing the halo exchange and output trim.
self._halo_conv_forward = None
self.rank = get_decode_parallel_rank()
self.world_size = get_decode_parallel_world_size()
self.parallel_group = parallel_group
self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group)
def forward(self, x, cache_x=None):
padding = list(self._padding)
@@ -696,6 +730,7 @@ class SpatialParallelConv3d(nn.Conv3d):
bias: bool = True,
padding_mode: str = "zeros",
height_padding: tuple[int, int] | None = None,
parallel_group: GroupCoordinator | None = None,
):
super().__init__(
in_channels=in_channels,
@@ -732,8 +767,8 @@ class SpatialParallelConv3d(nn.Conv3d):
# Set only by the ROCm Conv3D->Conv2D fast path, to swap the inner
# conv without displacing the halo exchange and output trim.
self._halo_conv_forward = None
self.rank = get_decode_parallel_rank()
self.world_size = get_decode_parallel_world_size()
self.parallel_group = parallel_group
self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group)
def forward(self, x):
if spatial_parallel_decode_disabled():
@@ -785,11 +820,15 @@ class SpatialParallelConv3d(nn.Conv3d):
class SpatialParallelZeroPad2d(nn.Module):
def __init__(self, padding: tuple[int, int, int, int]) -> None:
def __init__(
self,
padding: tuple[int, int, int, int],
parallel_group: GroupCoordinator | None = None,
) -> None:
super().__init__()
self.padding = padding
self.rank = get_decode_parallel_rank()
self.world_size = get_decode_parallel_world_size()
self.parallel_group = parallel_group
self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if spatial_parallel_decode_disabled():
@@ -18,6 +18,7 @@
import contextvars
from contextlib import contextmanager, nullcontext
from functools import partial
import torch
import torch.distributed as dist
@@ -30,9 +31,11 @@ from sglang.multimodal_gen.configs.models.vaes.base import (
should_use_spatial_shard_parallel_decode,
)
from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode
from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_decode_parallel_rank,
get_decode_parallel_world_size,
get_sp_group,
get_sp_parallel_rank,
get_sp_world_size,
)
@@ -764,10 +767,22 @@ class WanAttentionBlock(nn.Module):
dim (int): The number of channels in the input tensor.
"""
def __init__(self, dim, *, spatial_parallel: bool = False) -> None:
def __init__(
self,
dim,
*,
spatial_parallel: bool = False,
parallel_group: GroupCoordinator | None = None,
) -> None:
super().__init__()
self.dim = dim
self.world_size = get_decode_parallel_world_size() if spatial_parallel else 1
self.parallel_group = parallel_group
if parallel_group is not None:
self.world_size = parallel_group.world_size
else:
self.world_size = (
get_decode_parallel_world_size() if spatial_parallel else 1
)
# layers
self.norm = WanRMS_norm(dim)
@@ -776,10 +791,12 @@ class WanAttentionBlock(nn.Module):
def forward(self, x):
if self.world_size > 1:
x = gather_height_for_global_op(x).contiguous()
x = gather_height_for_global_op(
x, parallel_group=self.parallel_group
).contiguous()
x = attention_block_forward(self, x)
if self.world_size > 1:
x = chunk_height_for_parallel_decode(x)
x = chunk_height_for_parallel_decode(x, parallel_group=self.parallel_group)
return x
@@ -863,13 +880,21 @@ class WanResidualDownBlock(nn.Module):
class WanDistResample(WanResample):
def __init__(self, dim: int, mode: str, upsample_out_dim: int = None) -> None:
def __init__(
self,
dim: int,
mode: str,
upsample_out_dim: int = None,
parallel_group: GroupCoordinator | None = None,
) -> None:
super().__init__(
dim,
mode,
upsample_out_dim=upsample_out_dim,
conv2d_cls=SpatialParallelConv2d,
zero_pad2d_cls=SpatialParallelZeroPad2d,
conv2d_cls=partial(SpatialParallelConv2d, parallel_group=parallel_group),
zero_pad2d_cls=partial(
SpatialParallelZeroPad2d, parallel_group=parallel_group
),
spatial_parallel=True,
)
@@ -881,19 +906,22 @@ class WanDistResidualBlock(WanResidualBlock):
out_dim: int,
dropout: float = 0.0,
non_linearity: str = "silu",
parallel_group: GroupCoordinator | None = None,
) -> None:
super().__init__(
in_dim,
out_dim,
dropout,
non_linearity,
causal_conv3d_cls=SpatialParallelCausalConv3d,
causal_conv3d_cls=partial(
SpatialParallelCausalConv3d, parallel_group=parallel_group
),
)
class WanDistAttentionBlock(WanAttentionBlock):
def __init__(self, dim) -> None:
super().__init__(dim, spatial_parallel=True)
def __init__(self, dim, parallel_group: GroupCoordinator | None = None) -> None:
super().__init__(dim, spatial_parallel=True, parallel_group=parallel_group)
class WanDistMidBlock(WanMidBlock):
@@ -903,14 +931,19 @@ class WanDistMidBlock(WanMidBlock):
dropout: float = 0.0,
non_linearity: str = "silu",
num_layers: int = 1,
parallel_group: GroupCoordinator | None = None,
):
super().__init__(
dim,
dropout,
non_linearity,
num_layers=num_layers,
residual_block_cls=WanDistResidualBlock,
attention_block_cls=WanDistAttentionBlock,
residual_block_cls=partial(
WanDistResidualBlock, parallel_group=parallel_group
),
attention_block_cls=partial(
WanDistAttentionBlock, parallel_group=parallel_group
),
)
@@ -923,6 +956,7 @@ class WanDistResidualDownBlock(WanResidualDownBlock):
num_res_blocks,
temperal_downsample=False,
down_flag=False,
parallel_group: GroupCoordinator | None = None,
):
super().__init__(
in_dim,
@@ -931,8 +965,10 @@ class WanDistResidualDownBlock(WanResidualDownBlock):
num_res_blocks,
temperal_downsample=temperal_downsample,
down_flag=down_flag,
residual_block_cls=WanDistResidualBlock,
resample_cls=WanDistResample,
residual_block_cls=partial(
WanDistResidualBlock, parallel_group=parallel_group
),
resample_cls=partial(WanDistResample, parallel_group=parallel_group),
)
@@ -986,12 +1022,21 @@ class WanEncoder3d(nn.Module):
world_size = get_sp_world_size()
if use_parallel_encode and world_size > 1:
CausalConv3d = SpatialParallelCausalConv3d
ResidualDownBlock = WanDistResidualDownBlock
ResidualBlock = WanDistResidualBlock
AttentionBlock = WanDistAttentionBlock
Resample = WanDistResample
MidBlock = WanDistMidBlock
self.parallel_group = get_sp_group()
CausalConv3d = partial(
SpatialParallelCausalConv3d, parallel_group=self.parallel_group
)
ResidualDownBlock = partial(
WanDistResidualDownBlock, parallel_group=self.parallel_group
)
ResidualBlock = partial(
WanDistResidualBlock, parallel_group=self.parallel_group
)
AttentionBlock = partial(
WanDistAttentionBlock, parallel_group=self.parallel_group
)
Resample = partial(WanDistResample, parallel_group=self.parallel_group)
MidBlock = partial(WanDistMidBlock, parallel_group=self.parallel_group)
else:
CausalConv3d = WanCausalConv3d
ResidualDownBlock = WanResidualDownBlock
@@ -1091,7 +1136,9 @@ class WanEncoder3d(nn.Module):
x = self.conv_out(x)
if self.use_parallel_encode and self.world_size > 1:
x = gather_and_trim_height(x, expected_height)
x = gather_and_trim_height(
x, expected_height, parallel_group=self.parallel_group
)
return x
@@ -0,0 +1,145 @@
"""Wan VAE communication must follow the group that shards its activations."""
import argparse
import os
import signal
import subprocess
import sys
import pytest
import torch
import torch.nn.functional as F
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
destroy_model_parallel,
get_decode_parallel_group_coordinator,
get_sp_group,
get_tp_group,
init_distributed_environment,
initialize_model_parallel,
)
from sglang.multimodal_gen.runtime.layers.parallel_conv import (
SpatialParallelConv2d,
chunk_height_for_parallel_decode,
gather_variable_height,
)
from sglang.multimodal_gen.runtime.models.vaes.wanvae import (
WanDecoder3d,
WanEncoder3d,
)
def _worker(sp_size, tp_size):
rank = int(os.environ["LOCAL_RANK"])
torch.cuda.set_device(rank)
init_distributed_environment(world_size=4, rank=rank, local_rank=rank)
initialize_model_parallel(
tensor_parallel_degree=tp_size,
sequence_parallel_degree=sp_size,
ulysses_degree=sp_size,
ring_degree=1,
)
torch.backends.cuda.matmul.allow_tf32 = False
torch.backends.cudnn.allow_tf32 = False
sp_group = get_sp_group()
decode_group = get_decode_parallel_group_coordinator()
assert sp_group.world_size == sp_size
assert decode_group.world_size == 4
# distinct TP replicas catch collectives that accidentally use the decode group
torch.manual_seed(42)
full = torch.randn(1, 2, 9, 8, device="cuda") + get_tp_group().rank_in_group
local = chunk_height_for_parallel_decode(full, parallel_group=sp_group)
gathered, heights = gather_variable_height(local, parallel_group=sp_group)
torch.testing.assert_close(gathered, full, rtol=0, atol=0)
assert heights == [part.shape[-2] for part in torch.tensor_split(full, sp_size, -2)]
conv = SpatialParallelConv2d(
2, 3, 3, stride=2, padding=1, parallel_group=sp_group
).cuda()
expected = F.conv2d(full, conv.weight, conv.bias, stride=2, padding=1)
actual, _ = gather_variable_height(conv(local), parallel_group=sp_group)
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
# exercise real Wan block construction, attention, downsampling and final gather
for residual in (False, True):
torch.manual_seed(123)
kwargs = dict(
dim=4,
z_dim=2,
dim_mult=(1, 2, 2),
num_res_blocks=1,
temperal_downsample=(False, False),
is_residual=residual,
)
reference = WanEncoder3d(**kwargs).cuda().eval()
parallel = WanEncoder3d(**kwargs, use_parallel_encode=True).cuda().eval()
parallel.load_state_dict(reference.state_dict())
if sp_size > 1:
assert parallel.parallel_group is sp_group
assert parallel.conv_in.parallel_group is sp_group
assert parallel.conv_out.parallel_group is sp_group
x = torch.randn(1, 3, 3, 32, 16, device="cuda") + get_tp_group().rank_in_group
with torch.inference_mode():
torch.testing.assert_close(parallel(x), reference(x), rtol=1e-4, atol=1e-4)
# default decoder callers must still use the full decode group, not SP
torch.manual_seed(321)
kwargs = dict(
dim=4, z_dim=2, dim_mult=(1, 1), num_res_blocks=1, temperal_upsample=(False,)
)
reference = WanDecoder3d(**kwargs).cuda().eval()
parallel = WanDecoder3d(**kwargs, use_parallel_decode=True).cuda().eval()
parallel.load_state_dict(reference.state_dict())
assert parallel.conv_in.parallel_group is None
assert parallel.conv_in.world_size == decode_group.world_size
z = torch.randn(1, 2, 1, 8, 4, device="cuda")
with torch.inference_mode():
torch.testing.assert_close(parallel(z), reference(z), rtol=1e-4, atol=1e-4)
destroy_model_parallel()
torch.distributed.destroy_process_group()
@pytest.mark.parametrize("sp_size,tp_size", [(1, 4), (2, 2), (4, 1)])
def test_wan_vae_parallel_groups(sp_size, tp_size):
if not torch.cuda.is_available() or torch.cuda.device_count() < 4:
pytest.fail("requires four CUDA GPUs")
process = subprocess.Popen(
[
sys.executable,
"-m",
"torch.distributed.run",
"--standalone",
"--nproc-per-node=4",
__file__,
"--worker",
"--sp-size",
str(sp_size),
"--tp-size",
str(tp_size),
],
start_new_session=True,
)
try:
returncode = process.wait(timeout=180)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGTERM)
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait()
raise
if returncode:
raise subprocess.CalledProcessError(returncode, process.args)
if __name__ == "__main__":
if "--worker" in sys.argv:
parser = argparse.ArgumentParser()
parser.add_argument("--worker", action="store_true")
parser.add_argument("--sp-size", type=int, required=True)
parser.add_argument("--tp-size", type=int, required=True)
args = parser.parse_args()
_worker(args.sp_size, args.tp_size)
else:
sys.exit(pytest.main([__file__, *sys.argv[1:]]))