From df1e4f3f2bf14bdbd9ac97672056a4cb77da2471 Mon Sep 17 00:00:00 2001 From: Mick Date: Fri, 9 Oct 2026 10:38:36 +0800 Subject: [PATCH] [diffusion] refactor: share SANA-WM refiner block computation (#42980) Co-authored-by: Mick Qian --- .../model_specific_stages/sana_wm/refiner.py | 5 ++- .../sana_wm/streaming_refiner.py | 43 ++----------------- 2 files changed, 7 insertions(+), 41 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/refiner.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/refiner.py index 282ae050f9..205a9e1c20 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/refiner.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/refiner.py @@ -3,6 +3,7 @@ from __future__ import annotations import os +from collections.abc import Callable from typing import Any import torch @@ -242,6 +243,7 @@ def _forward_diffusers_video_only( for block in transformer.transformer_blocks: hidden_states = _forward_diffusers_video_block( block=block, + self_attention=_streaming_diffusers_self_attention, hidden_states=hidden_states, encoder_hidden_states=encoder_hidden_states, temb=temb, @@ -262,6 +264,7 @@ def _forward_diffusers_video_only( def _forward_diffusers_video_block( *, block: nn.Module, + self_attention: Callable[..., torch.Tensor], hidden_states: torch.Tensor, encoder_hidden_states: torch.Tensor, temb: torch.Tensor, @@ -281,7 +284,7 @@ def _forward_diffusers_video_block( ) norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa - attn_hidden_states = _streaming_diffusers_self_attention( + attn_hidden_states = self_attention( attn=block.attn1, hidden_states=norm_hidden_states, query_rotary_emb=video_rotary_emb, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming_refiner.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming_refiner.py index 445247e852..7ba671187f 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming_refiner.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_wm/streaming_refiner.py @@ -31,6 +31,7 @@ from .refiner import ( STAGE_2_DISTILLED_SIGMA_VALUES, SanaWMLTX2RefinerStage, _as_additive_attention_mask, + _forward_diffusers_video_block, _unwrap_diffusers_ltx2_refiner, log_sana_wm_tensor_stats, sana_wm_skip_refiner_enabled, @@ -240,45 +241,6 @@ def streaming_self_attention( return attn.to_out[1](hidden_states) -def forward_video_block( - *, - block, - hidden_states, - encoder_hidden_states, - temb, - video_rotary_emb, - encoder_attention_mask, - n_context_tokens, -): - batch = hidden_states.size(0) - norm_hidden_states = block.norm1(hidden_states) - num_ada_params = block.scale_shift_table.shape[0] - ada_values = block.scale_shift_table[None, None].to(temb.device) + temb.reshape( - batch, temb.size(1), num_ada_params, -1 - ) - shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ada_values.unbind( - dim=2 - ) - norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa - attn_hidden_states = streaming_self_attention( - attn=block.attn1, - hidden_states=norm_hidden_states, - query_rotary_emb=video_rotary_emb, - n_context_tokens=n_context_tokens, - ) - hidden_states = hidden_states + attn_hidden_states * gate_msa - norm_hidden_states = block.norm2(hidden_states) - attn_hidden_states = block.attn2( - norm_hidden_states, - encoder_hidden_states=encoder_hidden_states, - query_rotary_emb=None, - attention_mask=encoder_attention_mask, - ) - hidden_states = hidden_states + attn_hidden_states - norm_hidden_states = block.norm3(hidden_states) * (1 + scale_mlp) + shift_mlp - return hidden_states + block.ff(norm_hidden_states) * gate_mlp - - class _RefinerCore: """Adapts the unwrapped diffusers refiner transformer to the DiffusersLTX2Refiner interface RefinerChunkRunner expects.""" @@ -321,8 +283,9 @@ class _RefinerCore: batch, -1, hidden_states.size(-1) ) for block in transformer.transformer_blocks: - hidden_states = forward_video_block( + hidden_states = _forward_diffusers_video_block( block=block, + self_attention=streaming_self_attention, hidden_states=hidden_states, encoder_hidden_states=encoder_hidden_states, temb=temb,