[diffusion] refactor: deduplicate MOVA initialization (#42983)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-08 09:28:06 +08:00
committed by GitHub
co-authored by Mick Qian
parent 243f3dc35f
commit 28c08b02c3
2 changed files with 23 additions and 71 deletions
@@ -71,9 +71,6 @@ class Conv1dLocalIsland(nn.Conv1d):
placements can be customized).
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def forward(self, input):
if isinstance(input, DTensor):
x_local = input.to_local() # type: ignore[attr-defined]
@@ -166,21 +163,6 @@ class WanAudioModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self.num_attention_heads = num_heads
self.num_channels_latents = out_dim
self.layer_names = ["blocks"]
self.cnt = 0
self.teacache_thresh = 0
self.coefficients = []
self.accumulated_rel_l1_distance = 0
self.previous_modulated_input = None
self.previous_resiual = None
self.previous_e0_even = None
self.previous_e0_odd = None
self.previous_residual_even = None
self.previous_residual_odd = None
self.is_even = False
self.should_calc_even = True
self.should_calc_odd = True
self.accumulated_rel_l1_distance_even = 0
self.accumulated_rel_l1_distance_odd = 0
self.__post_init__()
def _init_freqs(self):
@@ -86,14 +86,8 @@ def precompute_freqs_cis(
return freqs_cis
class SelfAttention(nn.Module):
"""
Self-Attention module for MOVA DiT with Sequence Parallelism support.
SP is handled at the pipeline level (latents are pre-sharded before DiT forward).
USPAttention internally handles the all-to-all communication for distributed attention.
Input x should already be the local shard [B, S_local, D] when SP is enabled.
"""
class _MOVAAttention(nn.Module):
"""shared projections, normalization and TP head partitioning for MOVA"""
def __init__(
self,
@@ -130,6 +124,25 @@ class SelfAttention(nn.Module):
self.norm_q = RMSNorm(dim, eps=eps)
self.norm_k = RMSNorm(dim, eps=eps)
class SelfAttention(_MOVAAttention):
"""
Self-Attention module for MOVA DiT with Sequence Parallelism support.
SP is handled at the pipeline level (latents are pre-sharded before DiT forward).
USPAttention internally handles the all-to-all communication for distributed attention.
Input x should already be the local shard [B, S_local, D] when SP is enabled.
"""
def __init__(
self,
dim: int,
num_heads: int,
eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
):
super().__init__(dim, num_heads, eps, quant_config)
self.rotary_emb = RotaryEmbedding(
head_size=self.head_dim,
rotary_dim=self.head_dim,
@@ -196,7 +209,7 @@ class SelfAttention(nn.Module):
return out
class CrossAttention(nn.Module):
class CrossAttention(_MOVAAttention):
"""
Cross-Attention module for MOVA DiT.
@@ -214,32 +227,7 @@ class CrossAttention(nn.Module):
eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
):
super().__init__()
self.dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.tp_size = get_tp_world_size()
if self.num_heads % self.tp_size != 0:
raise ValueError(
f"num_heads ({self.num_heads}) must be divisible by tp_size ({self.tp_size})."
)
self.num_heads_per_rank = self.num_heads // self.tp_size
self.q = ColumnParallelLinear(
dim, dim, bias=True, gather_output=False, quant_config=quant_config
)
self.k = ColumnParallelLinear(
dim, dim, bias=True, gather_output=False, quant_config=quant_config
)
self.v = ColumnParallelLinear(
dim, dim, bias=True, gather_output=False, quant_config=quant_config
)
self.o = RowParallelLinear(
dim, dim, bias=True, input_is_parallel=True, quant_config=quant_config
)
self.norm_q = RMSNorm(dim, eps=eps)
self.norm_k = RMSNorm(dim, eps=eps)
super().__init__(dim, num_heads, eps, quant_config)
# Use LocalAttention for cross-attention (no SP communication needed)
self.attn = LocalAttention(
@@ -405,9 +393,6 @@ class Conv3dLocalIsland(nn.Conv3d):
but placements can be customized).
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def forward(self, input):
if isinstance(input, DTensor):
# NOTE: DTensor typing stubs are incomplete; at runtime DTensor has
@@ -499,21 +484,6 @@ class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin):
self.num_attention_heads = num_heads
self.num_channels_latents = out_dim
self.layer_names = ["blocks"]
self.cnt = 0
self.teacache_thresh = 0
self.coefficients = []
self.accumulated_rel_l1_distance = 0
self.previous_modulated_input = None
self.previous_resiual = None
self.previous_e0_even = None
self.previous_e0_odd = None
self.previous_residual_even = None
self.previous_residual_odd = None
self.is_even = False
self.should_calc_even = True
self.should_calc_odd = True
self.accumulated_rel_l1_distance_even = 0
self.accumulated_rel_l1_distance_odd = 0
self.__post_init__()
def _init_freqs(self):