diff --git a/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py b/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py index f857faf77a..0dc76d999f 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/mova_audio_dit.py @@ -71,9 +71,6 @@ class Conv1dLocalIsland(nn.Conv1d): placements can be customized). """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def forward(self, input): if isinstance(input, DTensor): x_local = input.to_local() # type: ignore[attr-defined] @@ -166,21 +163,6 @@ class WanAudioModel(CachableDiT, LayerwiseOffloadableModuleMixin): self.num_attention_heads = num_heads self.num_channels_latents = out_dim self.layer_names = ["blocks"] - self.cnt = 0 - self.teacache_thresh = 0 - self.coefficients = [] - self.accumulated_rel_l1_distance = 0 - self.previous_modulated_input = None - self.previous_resiual = None - self.previous_e0_even = None - self.previous_e0_odd = None - self.previous_residual_even = None - self.previous_residual_odd = None - self.is_even = False - self.should_calc_even = True - self.should_calc_odd = True - self.accumulated_rel_l1_distance_even = 0 - self.accumulated_rel_l1_distance_odd = 0 self.__post_init__() def _init_freqs(self): diff --git a/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py b/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py index 444ca1a274..eff8494627 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/mova_video_dit.py @@ -86,14 +86,8 @@ def precompute_freqs_cis( return freqs_cis -class SelfAttention(nn.Module): - """ - Self-Attention module for MOVA DiT with Sequence Parallelism support. - - SP is handled at the pipeline level (latents are pre-sharded before DiT forward). - USPAttention internally handles the all-to-all communication for distributed attention. - Input x should already be the local shard [B, S_local, D] when SP is enabled. - """ +class _MOVAAttention(nn.Module): + """shared projections, normalization and TP head partitioning for MOVA""" def __init__( self, @@ -130,6 +124,25 @@ class SelfAttention(nn.Module): self.norm_q = RMSNorm(dim, eps=eps) self.norm_k = RMSNorm(dim, eps=eps) + +class SelfAttention(_MOVAAttention): + """ + Self-Attention module for MOVA DiT with Sequence Parallelism support. + + SP is handled at the pipeline level (latents are pre-sharded before DiT forward). + USPAttention internally handles the all-to-all communication for distributed attention. + Input x should already be the local shard [B, S_local, D] when SP is enabled. + """ + + def __init__( + self, + dim: int, + num_heads: int, + eps: float = 1e-6, + quant_config: QuantizationConfig | None = None, + ): + super().__init__(dim, num_heads, eps, quant_config) + self.rotary_emb = RotaryEmbedding( head_size=self.head_dim, rotary_dim=self.head_dim, @@ -196,7 +209,7 @@ class SelfAttention(nn.Module): return out -class CrossAttention(nn.Module): +class CrossAttention(_MOVAAttention): """ Cross-Attention module for MOVA DiT. @@ -214,32 +227,7 @@ class CrossAttention(nn.Module): eps: float = 1e-6, quant_config: QuantizationConfig | None = None, ): - super().__init__() - self.dim = dim - self.num_heads = num_heads - self.head_dim = dim // num_heads - - self.tp_size = get_tp_world_size() - if self.num_heads % self.tp_size != 0: - raise ValueError( - f"num_heads ({self.num_heads}) must be divisible by tp_size ({self.tp_size})." - ) - self.num_heads_per_rank = self.num_heads // self.tp_size - - self.q = ColumnParallelLinear( - dim, dim, bias=True, gather_output=False, quant_config=quant_config - ) - self.k = ColumnParallelLinear( - dim, dim, bias=True, gather_output=False, quant_config=quant_config - ) - self.v = ColumnParallelLinear( - dim, dim, bias=True, gather_output=False, quant_config=quant_config - ) - self.o = RowParallelLinear( - dim, dim, bias=True, input_is_parallel=True, quant_config=quant_config - ) - self.norm_q = RMSNorm(dim, eps=eps) - self.norm_k = RMSNorm(dim, eps=eps) + super().__init__(dim, num_heads, eps, quant_config) # Use LocalAttention for cross-attention (no SP communication needed) self.attn = LocalAttention( @@ -405,9 +393,6 @@ class Conv3dLocalIsland(nn.Conv3d): but placements can be customized). """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - def forward(self, input): if isinstance(input, DTensor): # NOTE: DTensor typing stubs are incomplete; at runtime DTensor has @@ -499,21 +484,6 @@ class WanModel(CachableDiT, LayerwiseOffloadableModuleMixin): self.num_attention_heads = num_heads self.num_channels_latents = out_dim self.layer_names = ["blocks"] - self.cnt = 0 - self.teacache_thresh = 0 - self.coefficients = [] - self.accumulated_rel_l1_distance = 0 - self.previous_modulated_input = None - self.previous_resiual = None - self.previous_e0_even = None - self.previous_e0_odd = None - self.previous_residual_even = None - self.previous_residual_odd = None - self.is_even = False - self.should_calc_even = True - self.should_calc_odd = True - self.accumulated_rel_l1_distance_even = 0 - self.accumulated_rel_l1_distance_odd = 0 self.__post_init__() def _init_freqs(self):