[diffusion] refactor: share SANA-WM refiner block computation (#42980)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
+4
-1
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -242,6 +243,7 @@ def _forward_diffusers_video_only(
|
||||
for block in transformer.transformer_blocks:
|
||||
hidden_states = _forward_diffusers_video_block(
|
||||
block=block,
|
||||
self_attention=_streaming_diffusers_self_attention,
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
temb=temb,
|
||||
@@ -262,6 +264,7 @@ def _forward_diffusers_video_only(
|
||||
def _forward_diffusers_video_block(
|
||||
*,
|
||||
block: nn.Module,
|
||||
self_attention: Callable[..., torch.Tensor],
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
temb: torch.Tensor,
|
||||
@@ -281,7 +284,7 @@ def _forward_diffusers_video_block(
|
||||
)
|
||||
norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa
|
||||
|
||||
attn_hidden_states = _streaming_diffusers_self_attention(
|
||||
attn_hidden_states = self_attention(
|
||||
attn=block.attn1,
|
||||
hidden_states=norm_hidden_states,
|
||||
query_rotary_emb=video_rotary_emb,
|
||||
|
||||
+3
-40
@@ -31,6 +31,7 @@ from .refiner import (
|
||||
STAGE_2_DISTILLED_SIGMA_VALUES,
|
||||
SanaWMLTX2RefinerStage,
|
||||
_as_additive_attention_mask,
|
||||
_forward_diffusers_video_block,
|
||||
_unwrap_diffusers_ltx2_refiner,
|
||||
log_sana_wm_tensor_stats,
|
||||
sana_wm_skip_refiner_enabled,
|
||||
@@ -240,45 +241,6 @@ def streaming_self_attention(
|
||||
return attn.to_out[1](hidden_states)
|
||||
|
||||
|
||||
def forward_video_block(
|
||||
*,
|
||||
block,
|
||||
hidden_states,
|
||||
encoder_hidden_states,
|
||||
temb,
|
||||
video_rotary_emb,
|
||||
encoder_attention_mask,
|
||||
n_context_tokens,
|
||||
):
|
||||
batch = hidden_states.size(0)
|
||||
norm_hidden_states = block.norm1(hidden_states)
|
||||
num_ada_params = block.scale_shift_table.shape[0]
|
||||
ada_values = block.scale_shift_table[None, None].to(temb.device) + temb.reshape(
|
||||
batch, temb.size(1), num_ada_params, -1
|
||||
)
|
||||
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ada_values.unbind(
|
||||
dim=2
|
||||
)
|
||||
norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa
|
||||
attn_hidden_states = streaming_self_attention(
|
||||
attn=block.attn1,
|
||||
hidden_states=norm_hidden_states,
|
||||
query_rotary_emb=video_rotary_emb,
|
||||
n_context_tokens=n_context_tokens,
|
||||
)
|
||||
hidden_states = hidden_states + attn_hidden_states * gate_msa
|
||||
norm_hidden_states = block.norm2(hidden_states)
|
||||
attn_hidden_states = block.attn2(
|
||||
norm_hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
query_rotary_emb=None,
|
||||
attention_mask=encoder_attention_mask,
|
||||
)
|
||||
hidden_states = hidden_states + attn_hidden_states
|
||||
norm_hidden_states = block.norm3(hidden_states) * (1 + scale_mlp) + shift_mlp
|
||||
return hidden_states + block.ff(norm_hidden_states) * gate_mlp
|
||||
|
||||
|
||||
class _RefinerCore:
|
||||
"""Adapts the unwrapped diffusers refiner transformer to the
|
||||
DiffusersLTX2Refiner interface RefinerChunkRunner expects."""
|
||||
@@ -321,8 +283,9 @@ class _RefinerCore:
|
||||
batch, -1, hidden_states.size(-1)
|
||||
)
|
||||
for block in transformer.transformer_blocks:
|
||||
hidden_states = forward_video_block(
|
||||
hidden_states = _forward_diffusers_video_block(
|
||||
block=block,
|
||||
self_attention=streaming_self_attention,
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
temb=temb,
|
||||
|
||||
Reference in New Issue
Block a user