diff --git a/.github/workflows/pr-test-multimodal-gen.yml b/.github/workflows/pr-test-multimodal-gen.yml index 21ed1453a9..ada973468a 100644 --- a/.github/workflows/pr-test-multimodal-gen.yml +++ b/.github/workflows/pr-test-multimodal-gen.yml @@ -391,7 +391,7 @@ jobs: with: artifact-suffix: component-accuracy - multimodal-gen-test-1-b200: + multimodal-gen-test-4-gpu-b200: if: | ((github.event_name == 'schedule' || inputs.test_parallel_dispatch == 'true') || (inputs.caller_needs_failure != 'true' && !cancelled())) && inputs.multimodal_gen == 'true' @@ -422,6 +422,11 @@ jobs: run: | CUSTOM_BUILD_SGL_KERNEL=${{inputs.sgl_kernel}} bash scripts/ci/cuda/ci_install_dependency.sh diffusion + - name: Run VAE parallel-group regression + timeout-minutes: 10 + run: | + python3 -m pytest -v python/sglang/multimodal_gen/test/single_test_file/test_wan_vae_parallel_groups_4_gpu.py + - name: Run diffusion server tests timeout-minutes: 240 env: diff --git a/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py b/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py index 04daace521..f1b45d781a 100644 --- a/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py +++ b/python/sglang/multimodal_gen/runtime/layers/parallel_conv.py @@ -7,6 +7,7 @@ import torch.distributed as dist import torch.nn as nn import torch.nn.functional as F +from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_decode_parallel_group_coordinator, get_decode_parallel_rank, @@ -78,6 +79,14 @@ def spatial_parallel_decode_disabled() -> bool: return _SPATIAL_PARALLEL_DECODE_DISABLED.get() +def _parallel_rank_and_world_size( + parallel_group: GroupCoordinator | None, +) -> tuple[int, int]: + if parallel_group is not None: + return parallel_group.rank_in_group, parallel_group.world_size + return get_decode_parallel_rank(), get_decode_parallel_world_size() + + def _tensor_pad(x: torch.Tensor, len_to_pad: int, dim: int = -2): return torch.cat( [ @@ -191,31 +200,40 @@ def _maybe_contiguous_for_sp_gather(x: torch.Tensor) -> torch.Tensor: return x -def gather_and_trim_height(x: torch.Tensor, expected_height: int | None): +def gather_and_trim_height( + x: torch.Tensor, + expected_height: int | None, + parallel_group: GroupCoordinator | None = None, +): if spatial_parallel_decode_disabled(): return x if expected_height is None: return x - x, _ = gather_variable_height(x) + x, _ = gather_variable_height(x, parallel_group=parallel_group) if x.shape[-2] != expected_height: x = x[..., :expected_height, :].contiguous() return x -def gather_height_for_global_op(x: torch.Tensor) -> torch.Tensor: +def gather_height_for_global_op( + x: torch.Tensor, parallel_group: GroupCoordinator | None = None +) -> torch.Tensor: if spatial_parallel_decode_disabled(): return x - return gather_variable_height(x)[0] + return gather_variable_height(x, parallel_group=parallel_group)[0] -def chunk_height_for_parallel_decode(x: torch.Tensor) -> torch.Tensor: +def chunk_height_for_parallel_decode( + x: torch.Tensor, parallel_group: GroupCoordinator | None = None +) -> torch.Tensor: if spatial_parallel_decode_disabled(): return x + rank, world_size = _parallel_rank_and_world_size(parallel_group) return _tensor_chunk( x, dim=-2, - world_size=get_decode_parallel_world_size(), - rank=get_decode_parallel_rank(), + world_size=world_size, + rank=rank, ) @@ -229,38 +247,50 @@ def chunk_height_by_sizes(x: torch.Tensor, heights: list[int]) -> torch.Tensor: ) -def gather_height_sizes(x: torch.Tensor) -> list[int]: +def gather_height_sizes( + x: torch.Tensor, parallel_group: GroupCoordinator | None = None +) -> list[int]: """gather heights of sharded feature_maps from peers""" if spatial_parallel_decode_disabled(): return [x.shape[-2]] - world_size = get_decode_parallel_world_size() + world_size = ( + parallel_group.world_size + if parallel_group is not None + else get_decode_parallel_world_size() + ) if world_size <= 1: return [x.shape[-2]] + parallel_group = parallel_group or get_decode_parallel_group_coordinator() local_height = torch.tensor([x.shape[-2]], device=x.device, dtype=torch.int64) gathered = [torch.empty_like(local_height) for _ in range(world_size)] dist.all_gather( gathered, local_height, - group=get_decode_parallel_group_coordinator().device_group, + group=parallel_group.device_group, ) return [int(height.item()) for height in gathered] -def gather_variable_height(x: torch.Tensor) -> tuple[torch.Tensor, list[int]]: +def gather_variable_height( + x: torch.Tensor, parallel_group: GroupCoordinator | None = None +) -> tuple[torch.Tensor, list[int]]: if spatial_parallel_decode_disabled(): return x, [x.shape[-2]] - world_size = get_decode_parallel_world_size() + world_size = ( + parallel_group.world_size + if parallel_group is not None + else get_decode_parallel_world_size() + ) if world_size <= 1: return x, [x.shape[-2]] + parallel_group = parallel_group or get_decode_parallel_group_coordinator() - heights = gather_height_sizes(x) + heights = gather_height_sizes(x, parallel_group=parallel_group) max_height = max(heights) if x.shape[-2] < max_height: x = _tensor_pad(x, max_height - x.shape[-2], dim=-2) - gathered = get_decode_parallel_group_coordinator().all_gather( - _maybe_contiguous_for_sp_gather(x), dim=-2 - ) + gathered = parallel_group.all_gather(_maybe_contiguous_for_sp_gather(x), dim=-2) chunks = torch.split(gathered, max_height, dim=-2) return ( torch.cat( @@ -305,6 +335,7 @@ def halo_exchange( recv_top_buf: torch.Tensor | None = None, recv_bottom_buf: torch.Tensor | None = None, height_pad_mode: str = "zeros", + parallel_group: GroupCoordinator | None = None, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """exchange(send and recv) top/bottom conv-input halos with adjacent spatial ranks""" if spatial_parallel_decode_disabled(): @@ -312,11 +343,11 @@ def halo_exchange( if height_halo_size == 0: return x, recv_top_buf, recv_bottom_buf - decode_group = get_decode_parallel_group_coordinator() - rank = get_decode_parallel_rank() - world_size = get_decode_parallel_world_size() - group = decode_group.device_group - group_ranks = decode_group.ranks + parallel_group = parallel_group or get_decode_parallel_group_coordinator() + rank = parallel_group.rank_in_group + world_size = parallel_group.world_size + group = parallel_group.device_group + group_ranks = parallel_group.ranks top_row_ref = x[..., :height_halo_size, :] bottom_row_ref = x[..., -height_halo_size:, :] @@ -472,6 +503,7 @@ def _spatial_parallel_conv_forward( recv_top_buf=module._halo_recv_top_buf, recv_bottom_buf=module._halo_recv_bottom_buf, height_pad_mode=height_pad_mode, + parallel_group=module.parallel_group, ) if match_conv3d_format: x_padded = _match_conv3d_input_format(x_padded, module.weight) @@ -489,7 +521,7 @@ def _spatial_parallel_conv_forward( ): return conv_forward(x_padded) - heights = gather_height_sizes(x) + heights = gather_height_sizes(x, parallel_group=module.parallel_group) global_start = sum(heights[: module.rank]) global_height = sum(heights) if stride > 1: @@ -532,6 +564,7 @@ class SpatialParallelConv2d(nn.Conv2d): bias: bool = True, padding_mode: str = "zeros", height_padding: tuple[int, int] | None = None, + parallel_group: GroupCoordinator | None = None, ): super().__init__( in_channels=in_channels, @@ -558,8 +591,8 @@ class SpatialParallelConv2d(nn.Conv2d): _set_conv_padding(self, (0, self.padding[1])) self._halo_recv_top_buf: torch.Tensor | None = None self._halo_recv_bottom_buf: torch.Tensor | None = None - self.rank = get_decode_parallel_rank() - self.world_size = get_decode_parallel_world_size() + self.parallel_group = parallel_group + self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group) def forward(self, x): if spatial_parallel_decode_disabled(): @@ -604,6 +637,7 @@ class SpatialParallelCausalConv3d(nn.Conv3d): kernel_size: int | tuple[int, int, int], stride: int | tuple[int, int, int] = 1, padding: int | tuple[int, int, int] = 0, + parallel_group: GroupCoordinator | None = None, ): super().__init__( in_channels=in_channels, @@ -642,8 +676,8 @@ class SpatialParallelCausalConv3d(nn.Conv3d): # Set only by the ROCm Conv3D->Conv2D fast path, to swap the inner # conv without displacing the halo exchange and output trim. self._halo_conv_forward = None - self.rank = get_decode_parallel_rank() - self.world_size = get_decode_parallel_world_size() + self.parallel_group = parallel_group + self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group) def forward(self, x, cache_x=None): padding = list(self._padding) @@ -696,6 +730,7 @@ class SpatialParallelConv3d(nn.Conv3d): bias: bool = True, padding_mode: str = "zeros", height_padding: tuple[int, int] | None = None, + parallel_group: GroupCoordinator | None = None, ): super().__init__( in_channels=in_channels, @@ -732,8 +767,8 @@ class SpatialParallelConv3d(nn.Conv3d): # Set only by the ROCm Conv3D->Conv2D fast path, to swap the inner # conv without displacing the halo exchange and output trim. self._halo_conv_forward = None - self.rank = get_decode_parallel_rank() - self.world_size = get_decode_parallel_world_size() + self.parallel_group = parallel_group + self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group) def forward(self, x): if spatial_parallel_decode_disabled(): @@ -785,11 +820,15 @@ class SpatialParallelConv3d(nn.Conv3d): class SpatialParallelZeroPad2d(nn.Module): - def __init__(self, padding: tuple[int, int, int, int]) -> None: + def __init__( + self, + padding: tuple[int, int, int, int], + parallel_group: GroupCoordinator | None = None, + ) -> None: super().__init__() self.padding = padding - self.rank = get_decode_parallel_rank() - self.world_size = get_decode_parallel_world_size() + self.parallel_group = parallel_group + self.rank, self.world_size = _parallel_rank_and_world_size(parallel_group) def forward(self, x: torch.Tensor) -> torch.Tensor: if spatial_parallel_decode_disabled(): diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py index a6b1489300..45bb722e26 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py @@ -18,6 +18,7 @@ import contextvars from contextlib import contextmanager, nullcontext +from functools import partial import torch import torch.distributed as dist @@ -30,9 +31,11 @@ from sglang.multimodal_gen.configs.models.vaes.base import ( should_use_spatial_shard_parallel_decode, ) from sglang.multimodal_gen.runtime.cache.conditioning import cached_vae_encode +from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator from sglang.multimodal_gen.runtime.distributed.parallel_state import ( get_decode_parallel_rank, get_decode_parallel_world_size, + get_sp_group, get_sp_parallel_rank, get_sp_world_size, ) @@ -764,10 +767,22 @@ class WanAttentionBlock(nn.Module): dim (int): The number of channels in the input tensor. """ - def __init__(self, dim, *, spatial_parallel: bool = False) -> None: + def __init__( + self, + dim, + *, + spatial_parallel: bool = False, + parallel_group: GroupCoordinator | None = None, + ) -> None: super().__init__() self.dim = dim - self.world_size = get_decode_parallel_world_size() if spatial_parallel else 1 + self.parallel_group = parallel_group + if parallel_group is not None: + self.world_size = parallel_group.world_size + else: + self.world_size = ( + get_decode_parallel_world_size() if spatial_parallel else 1 + ) # layers self.norm = WanRMS_norm(dim) @@ -776,10 +791,12 @@ class WanAttentionBlock(nn.Module): def forward(self, x): if self.world_size > 1: - x = gather_height_for_global_op(x).contiguous() + x = gather_height_for_global_op( + x, parallel_group=self.parallel_group + ).contiguous() x = attention_block_forward(self, x) if self.world_size > 1: - x = chunk_height_for_parallel_decode(x) + x = chunk_height_for_parallel_decode(x, parallel_group=self.parallel_group) return x @@ -863,13 +880,21 @@ class WanResidualDownBlock(nn.Module): class WanDistResample(WanResample): - def __init__(self, dim: int, mode: str, upsample_out_dim: int = None) -> None: + def __init__( + self, + dim: int, + mode: str, + upsample_out_dim: int = None, + parallel_group: GroupCoordinator | None = None, + ) -> None: super().__init__( dim, mode, upsample_out_dim=upsample_out_dim, - conv2d_cls=SpatialParallelConv2d, - zero_pad2d_cls=SpatialParallelZeroPad2d, + conv2d_cls=partial(SpatialParallelConv2d, parallel_group=parallel_group), + zero_pad2d_cls=partial( + SpatialParallelZeroPad2d, parallel_group=parallel_group + ), spatial_parallel=True, ) @@ -881,19 +906,22 @@ class WanDistResidualBlock(WanResidualBlock): out_dim: int, dropout: float = 0.0, non_linearity: str = "silu", + parallel_group: GroupCoordinator | None = None, ) -> None: super().__init__( in_dim, out_dim, dropout, non_linearity, - causal_conv3d_cls=SpatialParallelCausalConv3d, + causal_conv3d_cls=partial( + SpatialParallelCausalConv3d, parallel_group=parallel_group + ), ) class WanDistAttentionBlock(WanAttentionBlock): - def __init__(self, dim) -> None: - super().__init__(dim, spatial_parallel=True) + def __init__(self, dim, parallel_group: GroupCoordinator | None = None) -> None: + super().__init__(dim, spatial_parallel=True, parallel_group=parallel_group) class WanDistMidBlock(WanMidBlock): @@ -903,14 +931,19 @@ class WanDistMidBlock(WanMidBlock): dropout: float = 0.0, non_linearity: str = "silu", num_layers: int = 1, + parallel_group: GroupCoordinator | None = None, ): super().__init__( dim, dropout, non_linearity, num_layers=num_layers, - residual_block_cls=WanDistResidualBlock, - attention_block_cls=WanDistAttentionBlock, + residual_block_cls=partial( + WanDistResidualBlock, parallel_group=parallel_group + ), + attention_block_cls=partial( + WanDistAttentionBlock, parallel_group=parallel_group + ), ) @@ -923,6 +956,7 @@ class WanDistResidualDownBlock(WanResidualDownBlock): num_res_blocks, temperal_downsample=False, down_flag=False, + parallel_group: GroupCoordinator | None = None, ): super().__init__( in_dim, @@ -931,8 +965,10 @@ class WanDistResidualDownBlock(WanResidualDownBlock): num_res_blocks, temperal_downsample=temperal_downsample, down_flag=down_flag, - residual_block_cls=WanDistResidualBlock, - resample_cls=WanDistResample, + residual_block_cls=partial( + WanDistResidualBlock, parallel_group=parallel_group + ), + resample_cls=partial(WanDistResample, parallel_group=parallel_group), ) @@ -986,12 +1022,21 @@ class WanEncoder3d(nn.Module): world_size = get_sp_world_size() if use_parallel_encode and world_size > 1: - CausalConv3d = SpatialParallelCausalConv3d - ResidualDownBlock = WanDistResidualDownBlock - ResidualBlock = WanDistResidualBlock - AttentionBlock = WanDistAttentionBlock - Resample = WanDistResample - MidBlock = WanDistMidBlock + self.parallel_group = get_sp_group() + CausalConv3d = partial( + SpatialParallelCausalConv3d, parallel_group=self.parallel_group + ) + ResidualDownBlock = partial( + WanDistResidualDownBlock, parallel_group=self.parallel_group + ) + ResidualBlock = partial( + WanDistResidualBlock, parallel_group=self.parallel_group + ) + AttentionBlock = partial( + WanDistAttentionBlock, parallel_group=self.parallel_group + ) + Resample = partial(WanDistResample, parallel_group=self.parallel_group) + MidBlock = partial(WanDistMidBlock, parallel_group=self.parallel_group) else: CausalConv3d = WanCausalConv3d ResidualDownBlock = WanResidualDownBlock @@ -1091,7 +1136,9 @@ class WanEncoder3d(nn.Module): x = self.conv_out(x) if self.use_parallel_encode and self.world_size > 1: - x = gather_and_trim_height(x, expected_height) + x = gather_and_trim_height( + x, expected_height, parallel_group=self.parallel_group + ) return x diff --git a/python/sglang/multimodal_gen/test/single_test_file/test_wan_vae_parallel_groups_4_gpu.py b/python/sglang/multimodal_gen/test/single_test_file/test_wan_vae_parallel_groups_4_gpu.py new file mode 100644 index 0000000000..660190db4c --- /dev/null +++ b/python/sglang/multimodal_gen/test/single_test_file/test_wan_vae_parallel_groups_4_gpu.py @@ -0,0 +1,145 @@ +"""Wan VAE communication must follow the group that shards its activations.""" + +import argparse +import os +import signal +import subprocess +import sys + +import pytest +import torch +import torch.nn.functional as F + +from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + destroy_model_parallel, + get_decode_parallel_group_coordinator, + get_sp_group, + get_tp_group, + init_distributed_environment, + initialize_model_parallel, +) +from sglang.multimodal_gen.runtime.layers.parallel_conv import ( + SpatialParallelConv2d, + chunk_height_for_parallel_decode, + gather_variable_height, +) +from sglang.multimodal_gen.runtime.models.vaes.wanvae import ( + WanDecoder3d, + WanEncoder3d, +) + + +def _worker(sp_size, tp_size): + rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(rank) + init_distributed_environment(world_size=4, rank=rank, local_rank=rank) + initialize_model_parallel( + tensor_parallel_degree=tp_size, + sequence_parallel_degree=sp_size, + ulysses_degree=sp_size, + ring_degree=1, + ) + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + sp_group = get_sp_group() + decode_group = get_decode_parallel_group_coordinator() + assert sp_group.world_size == sp_size + assert decode_group.world_size == 4 + + # distinct TP replicas catch collectives that accidentally use the decode group + torch.manual_seed(42) + full = torch.randn(1, 2, 9, 8, device="cuda") + get_tp_group().rank_in_group + local = chunk_height_for_parallel_decode(full, parallel_group=sp_group) + gathered, heights = gather_variable_height(local, parallel_group=sp_group) + torch.testing.assert_close(gathered, full, rtol=0, atol=0) + assert heights == [part.shape[-2] for part in torch.tensor_split(full, sp_size, -2)] + conv = SpatialParallelConv2d( + 2, 3, 3, stride=2, padding=1, parallel_group=sp_group + ).cuda() + expected = F.conv2d(full, conv.weight, conv.bias, stride=2, padding=1) + actual, _ = gather_variable_height(conv(local), parallel_group=sp_group) + torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5) + + # exercise real Wan block construction, attention, downsampling and final gather + for residual in (False, True): + torch.manual_seed(123) + kwargs = dict( + dim=4, + z_dim=2, + dim_mult=(1, 2, 2), + num_res_blocks=1, + temperal_downsample=(False, False), + is_residual=residual, + ) + reference = WanEncoder3d(**kwargs).cuda().eval() + parallel = WanEncoder3d(**kwargs, use_parallel_encode=True).cuda().eval() + parallel.load_state_dict(reference.state_dict()) + if sp_size > 1: + assert parallel.parallel_group is sp_group + assert parallel.conv_in.parallel_group is sp_group + assert parallel.conv_out.parallel_group is sp_group + x = torch.randn(1, 3, 3, 32, 16, device="cuda") + get_tp_group().rank_in_group + with torch.inference_mode(): + torch.testing.assert_close(parallel(x), reference(x), rtol=1e-4, atol=1e-4) + + # default decoder callers must still use the full decode group, not SP + torch.manual_seed(321) + kwargs = dict( + dim=4, z_dim=2, dim_mult=(1, 1), num_res_blocks=1, temperal_upsample=(False,) + ) + reference = WanDecoder3d(**kwargs).cuda().eval() + parallel = WanDecoder3d(**kwargs, use_parallel_decode=True).cuda().eval() + parallel.load_state_dict(reference.state_dict()) + assert parallel.conv_in.parallel_group is None + assert parallel.conv_in.world_size == decode_group.world_size + z = torch.randn(1, 2, 1, 8, 4, device="cuda") + with torch.inference_mode(): + torch.testing.assert_close(parallel(z), reference(z), rtol=1e-4, atol=1e-4) + destroy_model_parallel() + torch.distributed.destroy_process_group() + + +@pytest.mark.parametrize("sp_size,tp_size", [(1, 4), (2, 2), (4, 1)]) +def test_wan_vae_parallel_groups(sp_size, tp_size): + if not torch.cuda.is_available() or torch.cuda.device_count() < 4: + pytest.fail("requires four CUDA GPUs") + process = subprocess.Popen( + [ + sys.executable, + "-m", + "torch.distributed.run", + "--standalone", + "--nproc-per-node=4", + __file__, + "--worker", + "--sp-size", + str(sp_size), + "--tp-size", + str(tp_size), + ], + start_new_session=True, + ) + try: + returncode = process.wait(timeout=180) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + raise + if returncode: + raise subprocess.CalledProcessError(returncode, process.args) + + +if __name__ == "__main__": + if "--worker" in sys.argv: + parser = argparse.ArgumentParser() + parser.add_argument("--worker", action="store_true") + parser.add_argument("--sp-size", type=int, required=True) + parser.add_argument("--tp-size", type=int, required=True) + args = parser.parse_args() + _worker(args.sp_size, args.tp_size) + else: + sys.exit(pytest.main([__file__, *sys.argv[1:]]))