[diffusion] refactor: share SANA-WM refiner block computation (#42980)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-09 10:38:36 +08:00
committed by GitHub
co-authored by Mick Qian
parent f6569bce4d
commit df1e4f3f2b
2 changed files with 7 additions and 41 deletions
@@ -3,6 +3,7 @@
from __future__ import annotations
import os
from collections.abc import Callable
from typing import Any
import torch
@@ -242,6 +243,7 @@ def _forward_diffusers_video_only(
for block in transformer.transformer_blocks:
hidden_states = _forward_diffusers_video_block(
block=block,
self_attention=_streaming_diffusers_self_attention,
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
temb=temb,
@@ -262,6 +264,7 @@ def _forward_diffusers_video_only(
def _forward_diffusers_video_block(
*,
block: nn.Module,
self_attention: Callable[..., torch.Tensor],
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
temb: torch.Tensor,
@@ -281,7 +284,7 @@ def _forward_diffusers_video_block(
)
norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa
attn_hidden_states = _streaming_diffusers_self_attention(
attn_hidden_states = self_attention(
attn=block.attn1,
hidden_states=norm_hidden_states,
query_rotary_emb=video_rotary_emb,
@@ -31,6 +31,7 @@ from .refiner import (
STAGE_2_DISTILLED_SIGMA_VALUES,
SanaWMLTX2RefinerStage,
_as_additive_attention_mask,
_forward_diffusers_video_block,
_unwrap_diffusers_ltx2_refiner,
log_sana_wm_tensor_stats,
sana_wm_skip_refiner_enabled,
@@ -240,45 +241,6 @@ def streaming_self_attention(
return attn.to_out[1](hidden_states)
def forward_video_block(
*,
block,
hidden_states,
encoder_hidden_states,
temb,
video_rotary_emb,
encoder_attention_mask,
n_context_tokens,
):
batch = hidden_states.size(0)
norm_hidden_states = block.norm1(hidden_states)
num_ada_params = block.scale_shift_table.shape[0]
ada_values = block.scale_shift_table[None, None].to(temb.device) + temb.reshape(
batch, temb.size(1), num_ada_params, -1
)
shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ada_values.unbind(
dim=2
)
norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa
attn_hidden_states = streaming_self_attention(
attn=block.attn1,
hidden_states=norm_hidden_states,
query_rotary_emb=video_rotary_emb,
n_context_tokens=n_context_tokens,
)
hidden_states = hidden_states + attn_hidden_states * gate_msa
norm_hidden_states = block.norm2(hidden_states)
attn_hidden_states = block.attn2(
norm_hidden_states,
encoder_hidden_states=encoder_hidden_states,
query_rotary_emb=None,
attention_mask=encoder_attention_mask,
)
hidden_states = hidden_states + attn_hidden_states
norm_hidden_states = block.norm3(hidden_states) * (1 + scale_mlp) + shift_mlp
return hidden_states + block.ff(norm_hidden_states) * gate_mlp
class _RefinerCore:
"""Adapts the unwrapped diffusers refiner transformer to the
DiffusersLTX2Refiner interface RefinerChunkRunner expects."""
@@ -321,8 +283,9 @@ class _RefinerCore:
batch, -1, hidden_states.size(-1)
)
for block in transformer.transformer_blocks:
hidden_states = forward_video_block(
hidden_states = _forward_diffusers_video_block(
block=block,
self_attention=streaming_self_attention,
hidden_states=hidden_states,
encoder_hidden_states=encoder_hidden_states,
temb=temb,