[diffusion][model] Add native SANA-Video 2.0 support (#41492)
Signed-off-by: cr-gao <gaochenrui@sjtu.edu.cn> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
@@ -0,0 +1,52 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class SanaVideo2ArchConfig(DiTArchConfig):
|
||||
input_size: int = 15
|
||||
patch_size: tuple[int, int, int] = (1, 1, 1)
|
||||
in_channels: int = 128
|
||||
hidden_size: int = 2560
|
||||
depth: int = 32
|
||||
num_heads: int = 20
|
||||
linear_head_dim: int = 128
|
||||
softmax_head_dim: int = 256
|
||||
softmax_ratio: float = 0.25
|
||||
mlp_ratio: float = 4.0
|
||||
caption_channels: int = 2304
|
||||
model_max_length: int = 300
|
||||
qk_norm: bool = True
|
||||
cross_norm: bool = True
|
||||
y_norm: bool = True
|
||||
y_norm_scale_factor: float = 0.01
|
||||
norm_eps: float = 1e-5
|
||||
attn_res_block_size: int = 8
|
||||
timestep_norm_scale_factor: float = 1.0
|
||||
fp32_attention: bool = True
|
||||
|
||||
def __post_init__(self):
|
||||
super().__post_init__()
|
||||
self.patch_size = tuple(self.patch_size)
|
||||
if self.patch_size != (1, 1, 1):
|
||||
raise ValueError("SANA-Video 2.0 requires patch_size=(1, 1, 1)")
|
||||
if self.depth < 1 or self.attn_res_block_size < 1:
|
||||
raise ValueError("depth and attn_res_block_size must be positive")
|
||||
if not 0 < self.softmax_ratio <= 1:
|
||||
raise ValueError("softmax_ratio must be in (0, 1]")
|
||||
if any(
|
||||
self.hidden_size % dim
|
||||
for dim in (self.linear_head_dim, self.softmax_head_dim, self.num_heads)
|
||||
):
|
||||
raise ValueError("hidden_size must divide evenly into attention heads")
|
||||
self.num_attention_heads = self.num_heads
|
||||
self.num_channels_latents = self.in_channels
|
||||
|
||||
|
||||
@dataclass
|
||||
class SanaVideo2Config(DiTConfig):
|
||||
arch_config: DiTArchConfig = field(default_factory=SanaVideo2ArchConfig)
|
||||
prefix: str = "SanaVideo2"
|
||||
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Native SANA-Video 2.0 pipeline configuration."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.sana_video2 import SanaVideo2Config
|
||||
from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.sana_video import (
|
||||
SanaVideoPipelineConfig,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SanaVideo2VAEConfig(LTXVideoVAEConfig):
|
||||
def update_model_arch(self, source_model_dict):
|
||||
config = dict(source_model_dict)
|
||||
if "upsample_type" not in config and "decoder_upsample_type" in config:
|
||||
config["upsample_type"] = list(reversed(config["decoder_upsample_type"]))
|
||||
super().update_model_arch(config)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SanaVideo2PipelineConfig(SanaVideoPipelineConfig):
|
||||
task_type: ModelTaskType = ModelTaskType.TI2V
|
||||
flow_shift: float | None = 12.0
|
||||
skip_input_image_preprocess: bool = True
|
||||
dit_config: SanaVideo2Config = field(default_factory=SanaVideo2Config)
|
||||
vae_config: SanaVideo2VAEConfig = field(default_factory=SanaVideo2VAEConfig)
|
||||
vae_precision: str = "bf16"
|
||||
vae_decode_precision: str = "bf16"
|
||||
prompt_instruction: str = ""
|
||||
|
||||
def __post_init__(self):
|
||||
self.vae_config.load_encoder = True
|
||||
self.vae_config.load_decoder = True
|
||||
|
||||
def supports_sequential_multi_output_inference(self):
|
||||
return True
|
||||
|
||||
def get_decode_scale_and_shift(self, device, dtype, vae):
|
||||
mean = vae.latents_mean.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1)
|
||||
std = vae.latents_std.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1)
|
||||
return self.vae_config.arch_config.scaling_factor / std, mean
|
||||
|
||||
|
||||
def register():
|
||||
from sglang.multimodal_gen.configs.sample.sana_video2 import (
|
||||
SanaVideo2SamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.registry import register_configs
|
||||
|
||||
register_configs(
|
||||
sampling_param_cls=SanaVideo2SamplingParams,
|
||||
pipeline_config_cls=SanaVideo2PipelineConfig,
|
||||
hf_model_paths=["Efficient-Large-Model/SANA-Video_2.0_5B_720p"],
|
||||
model_detectors=[
|
||||
lambda name: any(
|
||||
key in name.lower()
|
||||
for key in (
|
||||
"sana-video_2.0",
|
||||
"sana-video2",
|
||||
"sana_video2",
|
||||
"sanavideo2pipeline",
|
||||
)
|
||||
)
|
||||
],
|
||||
)
|
||||
@@ -0,0 +1,28 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Sampling defaults for the SANA-Video 2.0 5B checkpoint."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sana_video import SanaVideoSamplingParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class SanaVideo2SamplingParams(SanaVideoSamplingParams):
|
||||
height: int = 736
|
||||
width: int = 1280
|
||||
num_frames: int = 193
|
||||
fps: int = 24
|
||||
guidance_scale: float = 8.0
|
||||
motion_score: int = 10
|
||||
high_motion: bool = False
|
||||
|
||||
def build_request_extra(self):
|
||||
return {
|
||||
**super().build_request_extra(),
|
||||
"motion_score": self.motion_score,
|
||||
"high_motion": self.high_motion,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def video_request_extra_fields(cls):
|
||||
return super().video_request_extra_fields() | {"motion_score", "high_motion"}
|
||||
@@ -173,6 +173,9 @@ _MODEL_NAME_DETECTORS: List[Tuple[str, Callable[[str], bool]]] = []
|
||||
# aliases next to the resolver that consumes them so CLI detection and
|
||||
# pipeline selection cannot drift apart
|
||||
KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = {
|
||||
"sana-video_2.0": "SanaVideo2Pipeline",
|
||||
"sana-video2": "SanaVideo2Pipeline",
|
||||
"sana_video2": "SanaVideo2Pipeline",
|
||||
"ming-image-0.1-design": "MingImagePipeline",
|
||||
"minimaxai/minimax-h3": "MiniMaxH3Pipeline",
|
||||
"minimax/minimax-h3": "MiniMaxH3Pipeline",
|
||||
@@ -291,7 +294,7 @@ _configs_discovered: bool = False
|
||||
|
||||
# SANA-WM (register BEFORE generic SANA T2I to prevent "sana" detector false-match)
|
||||
# SANA-Video (register before generic SANA to avoid detector overlap).
|
||||
_CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video")
|
||||
_CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video2", "sana_video")
|
||||
|
||||
|
||||
def _discover_and_register_configs() -> None:
|
||||
@@ -462,8 +465,11 @@ def _get_config_info(
|
||||
return _CONFIG_REGISTRY.get(model_id)
|
||||
|
||||
# 3. Use detectors
|
||||
config = maybe_download_model_index(model_path)
|
||||
pipeline_name = config.get("_class_name", "").lower()
|
||||
pipeline_name = get_non_diffusers_pipeline_name(model_path)
|
||||
if pipeline_name is None:
|
||||
config = maybe_download_model_index(model_path)
|
||||
pipeline_name = config.get("_class_name", "")
|
||||
pipeline_name = pipeline_name.lower()
|
||||
|
||||
matched_model_names = []
|
||||
for model_id, detector in _MODEL_NAME_DETECTORS:
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
# Copyright 2026 NVIDIA CORPORATION & AFFILIATES
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""SANA-Video 2.0 inference transformer with grouped Attention Residuals."""
|
||||
|
||||
import math
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.sana_video2 import SanaVideo2Config
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
|
||||
from sglang.multimodal_gen.runtime.models.dits.sana_wm_components import (
|
||||
CaptionEmbedder,
|
||||
PatchEmbedMS3D,
|
||||
T2IFinalLayer,
|
||||
TimestepEmbedder,
|
||||
WanRotaryPosEmbed,
|
||||
_RMSNorm,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
|
||||
|
||||
def _apply_rope(x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
|
||||
rotated = torch.view_as_complex(x.to(torch.float64).unflatten(-1, (-1, 2)))
|
||||
return torch.view_as_real(rotated * freqs).flatten(-2).type_as(x)
|
||||
|
||||
|
||||
class SanaVideo2RotaryPosEmbed(WanRotaryPosEmbed):
|
||||
def _apply(self, fn, recurse=True):
|
||||
# Keep complex128 frequencies; forward moves them to the input device.
|
||||
return self
|
||||
|
||||
|
||||
class ChannelRMSNorm(_RMSNorm):
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
normalized = (
|
||||
x.float() * (x.float().square().mean(-2, keepdim=True) + self.eps).rsqrt()
|
||||
)
|
||||
return (self.weight[None, None, :, None] * normalized).type_as(x)
|
||||
|
||||
|
||||
class GatedLinearAttention(nn.Module):
|
||||
def __init__(self, dim, head_dim, qk_norm=True, fp32_attention=True):
|
||||
super().__init__()
|
||||
self.heads = dim // head_dim
|
||||
self.dim = head_dim
|
||||
self.fp32_attention = fp32_attention
|
||||
self.qkv = nn.Linear(dim, 3 * dim, bias=False)
|
||||
self.proj = nn.Linear(dim, dim)
|
||||
self.q_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity()
|
||||
self.k_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity()
|
||||
self.beta_proj = nn.Linear(dim, self.heads)
|
||||
self.output_gate = nn.Linear(dim, dim)
|
||||
self.o_norm = ChannelRMSNorm(head_dim, eps=1e-5)
|
||||
|
||||
def forward(self, x, rotary_emb=None):
|
||||
batch, tokens, channels = x.shape
|
||||
q, k, v = self.qkv(x).reshape(batch, tokens, 3, channels).unbind(2)
|
||||
dtype = q.dtype
|
||||
q = (
|
||||
self.q_norm(q)
|
||||
.transpose(-1, -2)
|
||||
.reshape(batch, self.heads, self.dim, tokens)
|
||||
)
|
||||
k = (
|
||||
self.k_norm(k)
|
||||
.transpose(-1, -2)
|
||||
.reshape(batch, self.heads, self.dim, tokens)
|
||||
)
|
||||
v = v.transpose(-1, -2).reshape(batch, self.heads, self.dim, tokens)
|
||||
if rotary_emb is not None:
|
||||
q = _apply_rope(q.transpose(-1, -2), rotary_emb).transpose(-1, -2)
|
||||
k = _apply_rope(k.transpose(-1, -2), rotary_emb).transpose(-1, -2)
|
||||
beta = self.beta_proj(x).sigmoid().transpose(1, 2).unsqueeze(2)
|
||||
k = k * beta
|
||||
if self.fp32_attention:
|
||||
q, k, v = q.float(), k.float(), v.float()
|
||||
out = (v @ k.transpose(-1, -2)) @ q
|
||||
out = self.o_norm(out.to(dtype))
|
||||
out = out.reshape(batch, channels, tokens).transpose(1, 2)
|
||||
return self.proj(out * self.output_gate(x).sigmoid())
|
||||
|
||||
|
||||
class GatedSoftmaxAttention(nn.Module):
|
||||
def __init__(self, dim, head_dim, qk_norm=True, fp32_attention=True):
|
||||
super().__init__()
|
||||
self.heads = dim // head_dim
|
||||
self.dim = head_dim
|
||||
self.fp32_attention = fp32_attention
|
||||
self.qkv = nn.Linear(dim, 3 * dim, bias=False)
|
||||
self.proj = nn.Linear(dim, dim)
|
||||
self.q_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity()
|
||||
self.k_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity()
|
||||
self.output_gate = nn.Linear(dim, dim)
|
||||
|
||||
def forward(self, x, rotary_emb=None):
|
||||
batch, tokens, channels = x.shape
|
||||
q, k, v = self.qkv(x).reshape(batch, tokens, 3, channels).unbind(2)
|
||||
dtype = q.dtype
|
||||
q = self.q_norm(q).reshape(batch, tokens, self.heads, self.dim).transpose(1, 2)
|
||||
k = self.k_norm(k).reshape(batch, tokens, self.heads, self.dim).transpose(1, 2)
|
||||
v = v.reshape(batch, tokens, self.heads, self.dim).transpose(1, 2)
|
||||
if rotary_emb is not None:
|
||||
q, k = _apply_rope(q, rotary_emb), _apply_rope(k, rotary_emb)
|
||||
if self.fp32_attention:
|
||||
q, k, v = q.float(), k.float(), v.float()
|
||||
out = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0)
|
||||
out = out.transpose(1, 2).reshape(batch, tokens, channels).to(dtype)
|
||||
return self.proj(out * self.output_gate(x).sigmoid())
|
||||
|
||||
|
||||
class CrossAttention(nn.Module):
|
||||
def __init__(self, dim, num_heads, qk_norm=True):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
self.q_linear = nn.Linear(dim, dim)
|
||||
self.kv_linear = nn.Linear(dim, 2 * dim)
|
||||
self.proj = nn.Linear(dim, dim)
|
||||
self.q_norm = _RMSNorm(dim, eps=1e-6) if qk_norm else nn.Identity()
|
||||
self.k_norm = _RMSNorm(dim, eps=1e-6) if qk_norm else nn.Identity()
|
||||
|
||||
def forward(self, x, y, mask=None):
|
||||
batch, tokens, channels = x.shape
|
||||
q = self.q_norm(self.q_linear(x))
|
||||
k, v = self.kv_linear(y).reshape(batch, -1, 2, channels).unbind(2)
|
||||
q = q.reshape(batch, tokens, self.num_heads, self.head_dim).transpose(1, 2)
|
||||
k = (
|
||||
self.k_norm(k)
|
||||
.reshape(batch, -1, self.num_heads, self.head_dim)
|
||||
.transpose(1, 2)
|
||||
)
|
||||
v = v.reshape(batch, -1, self.num_heads, self.head_dim).transpose(1, 2)
|
||||
if mask is not None:
|
||||
mask = (1 - mask.to(q.dtype))[:, None, None, :] * -10000.0
|
||||
out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0)
|
||||
return self.proj(out.transpose(1, 2).reshape(batch, tokens, channels))
|
||||
|
||||
|
||||
class SwiGLU(nn.Module):
|
||||
def __init__(self, dim, hidden_dim):
|
||||
super().__init__()
|
||||
self.gate_proj = nn.Linear(dim, hidden_dim)
|
||||
self.up_proj = nn.Linear(dim, hidden_dim)
|
||||
self.down_proj = nn.Linear(hidden_dim, dim)
|
||||
|
||||
def forward(self, x):
|
||||
return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
|
||||
|
||||
|
||||
class SanaVideo2Block(nn.Module):
|
||||
def __init__(self, config, attention):
|
||||
super().__init__()
|
||||
dim = config.hidden_size
|
||||
self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
|
||||
attention_cls = (
|
||||
GatedSoftmaxAttention if attention == "softmax" else GatedLinearAttention
|
||||
)
|
||||
head_dim = (
|
||||
config.softmax_head_dim
|
||||
if attention == "softmax"
|
||||
else config.linear_head_dim
|
||||
)
|
||||
self.attn = attention_cls(dim, head_dim, config.qk_norm, config.fp32_attention)
|
||||
self.cross_attn = CrossAttention(dim, config.num_heads, config.cross_norm)
|
||||
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
|
||||
self.mlp = SwiGLU(dim, int(dim * config.mlp_ratio))
|
||||
self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5)
|
||||
|
||||
def _modulation(self, t, batch):
|
||||
if t.ndim == 2:
|
||||
return (self.scale_shift_table[None] + t.reshape(batch, 6, -1)).chunk(
|
||||
6, dim=1
|
||||
)
|
||||
groups = t.shape[2]
|
||||
return (
|
||||
self.scale_shift_table[None, None] + t.reshape(batch, groups, 6, -1)
|
||||
).chunk(6, dim=2)
|
||||
|
||||
def forward_attn_sublayer(self, x, y, t, mask=None, rotary_emb=None):
|
||||
batch, _, channels = x.shape
|
||||
shift, scale, gate, _, _, _ = self._modulation(t, batch)
|
||||
grouped_shape = (
|
||||
(batch, -1, channels) if t.ndim == 2 else (batch, t.shape[2], -1, channels)
|
||||
)
|
||||
attn_input = self.norm1(x).reshape(grouped_shape) * (1 + scale) + shift
|
||||
delta = self.attn(attn_input.reshape_as(x), rotary_emb)
|
||||
delta = (gate * delta.reshape(grouped_shape)).reshape_as(x)
|
||||
return delta + self.cross_attn(x + delta, y, mask)
|
||||
|
||||
def forward_mlp_sublayer(self, x, t):
|
||||
batch, _, channels = x.shape
|
||||
_, _, _, shift, scale, gate = self._modulation(t, batch)
|
||||
grouped_shape = (
|
||||
(batch, -1, channels) if t.ndim == 2 else (batch, t.shape[2], -1, channels)
|
||||
)
|
||||
mlp_input = self.norm2(x).reshape(grouped_shape) * (1 + scale) + shift
|
||||
delta = self.mlp(mlp_input.reshape_as(x)).reshape(grouped_shape)
|
||||
return (gate * delta).reshape_as(x)
|
||||
|
||||
|
||||
class DepthRMSNorm(nn.Module):
|
||||
def forward(self, x):
|
||||
return x * (x.square().mean(-1, keepdim=True) + 1e-6).rsqrt()
|
||||
|
||||
|
||||
class BlockAttentionResidual(nn.Module):
|
||||
def __init__(self, hidden_size):
|
||||
super().__init__()
|
||||
self.attn_proj = nn.Linear(hidden_size, 1, bias=False)
|
||||
self.mlp_proj = nn.Linear(hidden_size, 1, bias=False)
|
||||
self.final_proj = nn.Linear(hidden_size, 1, bias=False)
|
||||
self.key_norm = DepthRMSNorm()
|
||||
for projection in (self.attn_proj, self.mlp_proj, self.final_proj):
|
||||
nn.init.zeros_(projection.weight)
|
||||
|
||||
def attend_buffer(self, projection, values, keys, active_count, partial):
|
||||
source_count = active_count
|
||||
if partial is not None:
|
||||
values[active_count] = partial
|
||||
keys[active_count] = self.key_norm(partial.unsqueeze(0)).squeeze(0)
|
||||
source_count += 1
|
||||
if source_count == 1:
|
||||
return values[0]
|
||||
values, keys = values[:source_count], keys[:source_count]
|
||||
logits = torch.einsum("d,nbtd->nbt", projection.weight.squeeze(0), keys)
|
||||
return torch.einsum("nbt,nbtd->btd", logits.softmax(0), values)
|
||||
|
||||
|
||||
class SanaVideo2Transformer3DModel(BaseDiT):
|
||||
_fsdp_shard_conditions: ClassVar[list] = []
|
||||
_compile_conditions: ClassVar[list] = []
|
||||
_supported_attention_backends: ClassVar[set] = {AttentionBackendEnum.TORCH_SDPA}
|
||||
param_names_mapping: ClassVar[dict] = {}
|
||||
reverse_param_names_mapping: ClassVar[dict] = {}
|
||||
lora_param_names_mapping: ClassVar[dict] = {}
|
||||
|
||||
def __init__(self, config: SanaVideo2Config, hf_config=None, **kwargs):
|
||||
super().__init__(config, hf_config=hf_config or {}, **kwargs)
|
||||
c = self.config
|
||||
self.hidden_size = c.hidden_size
|
||||
self.num_attention_heads = c.num_heads
|
||||
self.num_channels_latents = c.in_channels
|
||||
self.register_buffer(
|
||||
"pos_embed", torch.zeros(1, c.input_size**2, c.hidden_size)
|
||||
)
|
||||
self.x_embedder = PatchEmbedMS3D(c.patch_size, c.in_channels, c.hidden_size)
|
||||
self.t_embedder = TimestepEmbedder(c.hidden_size)
|
||||
self.t_block = nn.Sequential(
|
||||
nn.SiLU(), nn.Linear(c.hidden_size, 6 * c.hidden_size)
|
||||
)
|
||||
self.y_embedder = CaptionEmbedder(
|
||||
c.caption_channels, c.hidden_size, c.model_max_length
|
||||
)
|
||||
if c.y_norm:
|
||||
self.attention_y_norm = _RMSNorm(
|
||||
c.hidden_size, c.y_norm_scale_factor, c.norm_eps
|
||||
)
|
||||
self.rope_linear = SanaVideo2RotaryPosEmbed(c.linear_head_dim, c.patch_size)
|
||||
self.rope_softmax = SanaVideo2RotaryPosEmbed(c.softmax_head_dim, c.patch_size)
|
||||
anchor_count = max(1, int(c.depth * c.softmax_ratio))
|
||||
self.softmax_layer_indices = [
|
||||
int((i + 1) * c.depth / anchor_count) - 1 for i in range(anchor_count)
|
||||
]
|
||||
self.block_attention_types = [
|
||||
"softmax" if i in self.softmax_layer_indices else "linear"
|
||||
for i in range(c.depth)
|
||||
]
|
||||
self.blocks = nn.ModuleList(
|
||||
[SanaVideo2Block(c, attention) for attention in self.block_attention_types]
|
||||
)
|
||||
self.final_layer = T2IFinalLayer(c.hidden_size, c.patch_size, c.in_channels)
|
||||
self.attn_res = BlockAttentionResidual(c.hidden_size)
|
||||
|
||||
@property
|
||||
def dtype(self):
|
||||
return self.x_embedder.proj.weight.dtype
|
||||
|
||||
def post_load_weights(self):
|
||||
for rope in (self.rope_linear, self.rope_softmax):
|
||||
if rope._freqs.is_meta:
|
||||
rope._init_freqs_buffer()
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states,
|
||||
timestep,
|
||||
encoder_hidden_states,
|
||||
encoder_attention_mask=None,
|
||||
**kwargs,
|
||||
):
|
||||
c = self.config
|
||||
batch, _, frames, height, width = hidden_states.shape
|
||||
if timestep.ndim == 5:
|
||||
timestep = timestep.reshape(batch, 1, frames)
|
||||
if c.timestep_norm_scale_factor == 1.0:
|
||||
timestep = timestep.long().float()
|
||||
else:
|
||||
timestep = timestep.float() / c.timestep_norm_scale_factor
|
||||
x = self.x_embedder(hidden_states.to(self.dtype))
|
||||
ropes = {
|
||||
"linear": self.rope_linear((frames, height, width), x.device),
|
||||
"softmax": self.rope_softmax((frames, height, width), x.device),
|
||||
}
|
||||
t = self.t_embedder(timestep.flatten())
|
||||
t0 = self.t_block(t).unflatten(0, timestep.shape)
|
||||
t = t.unflatten(0, timestep.shape)
|
||||
y = self.y_embedder(encoder_hidden_states.to(self.dtype))
|
||||
if c.y_norm:
|
||||
y = self.attention_y_norm(y)
|
||||
mask = encoder_attention_mask
|
||||
if mask is not None:
|
||||
mask = mask.reshape(batch, -1)
|
||||
# Keep the grouped depth buffer layout stable across denoising steps.
|
||||
count = math.ceil(c.depth / c.attn_res_block_size)
|
||||
values = x.new_empty(count + 2, batch, x.shape[1], c.hidden_size)
|
||||
keys = torch.empty_like(values)
|
||||
values[0] = x
|
||||
keys[0] = self.attn_res.key_norm(x.unsqueeze(0)).squeeze(0)
|
||||
active_count, partial = 1, None
|
||||
for index, block in enumerate(self.blocks):
|
||||
hidden = self.attn_res.attend_buffer(
|
||||
self.attn_res.attn_proj, values, keys, active_count, partial
|
||||
)
|
||||
delta = block.forward_attn_sublayer(
|
||||
hidden, y, t0, mask, ropes[self.block_attention_types[index]]
|
||||
)
|
||||
partial = delta if partial is None else partial + delta
|
||||
hidden = self.attn_res.attend_buffer(
|
||||
self.attn_res.mlp_proj, values, keys, active_count, partial
|
||||
)
|
||||
partial = partial + block.forward_mlp_sublayer(hidden, t0)
|
||||
if (index + 1) % c.attn_res_block_size == 0 or index + 1 == c.depth:
|
||||
values[active_count] = partial
|
||||
keys[active_count] = self.attn_res.key_norm(
|
||||
partial.unsqueeze(0)
|
||||
).squeeze(0)
|
||||
active_count += 1
|
||||
partial = None
|
||||
x = self.attn_res.attend_buffer(
|
||||
self.attn_res.final_proj, values, keys, active_count, None
|
||||
)
|
||||
return (
|
||||
self.final_layer(x, t)
|
||||
.transpose(1, 2)
|
||||
.reshape(batch, c.in_channels, frames, height, width)
|
||||
)
|
||||
|
||||
|
||||
EntryClass = SanaVideo2Transformer3DModel
|
||||
@@ -0,0 +1,132 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Native SANA-Video 2.0 loading and T2V/TI2V pipeline."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
import yaml
|
||||
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
|
||||
ComponentLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
|
||||
SanaVideo2Transformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages import InputValidationStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import (
|
||||
SanaVideo2DenoisingStage,
|
||||
SanaVideo2LatentPreparationStage,
|
||||
SanaVideo2TextEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import maybe_download_model
|
||||
from sglang.multimodal_gen.runtime.utils.precision import resolve_component_precision
|
||||
|
||||
|
||||
class SanaVideo2TransformerLoader(ComponentLoader):
|
||||
expected_library = "diffusers"
|
||||
|
||||
def component_load_precision(self, server_args, component_name):
|
||||
return server_args.component_precisions.get(component_name)
|
||||
|
||||
def load_customized(self, component_model_path, server_args, component_name):
|
||||
root = Path(component_model_path)
|
||||
with (root / "config.yaml").open() as file:
|
||||
source = yaml.safe_load(file)
|
||||
checkpoint = torch.load(
|
||||
root / "checkpoints/SANA_Video_2.0_5B_720p.pth",
|
||||
map_location="cpu",
|
||||
weights_only=True,
|
||||
mmap=True,
|
||||
)["state_dict"]
|
||||
config = server_args.pipeline_config.dit_config
|
||||
model_fields = config.arch_config.__dataclass_fields__
|
||||
values = {
|
||||
name: value
|
||||
for section in (source["model"], source["text_encoder"])
|
||||
for name, value in section.items()
|
||||
if name in model_fields
|
||||
}
|
||||
if "pos_embed" in checkpoint:
|
||||
values["input_size"] = int(checkpoint["pos_embed"].shape[1] ** 0.5)
|
||||
config.update_model_arch(values)
|
||||
dtype = resolve_component_precision(server_args, component_name)
|
||||
with torch.device("meta"):
|
||||
model = SanaVideo2Transformer3DModel(config)
|
||||
model.load_state_dict(checkpoint, strict=True, assign=True)
|
||||
model = model.to(dtype=dtype)
|
||||
model.post_load_weights()
|
||||
server_args.model_paths[component_name] = component_model_path
|
||||
return model.eval().requires_grad_(False)
|
||||
|
||||
|
||||
class SanaVideo2Pipeline(ComposedPipelineBase):
|
||||
pipeline_name = "SanaVideo2Pipeline"
|
||||
is_video_pipeline = True
|
||||
_required_config_modules: ClassVar[list[str]] = [
|
||||
"text_encoder",
|
||||
"tokenizer",
|
||||
"vae",
|
||||
"transformer",
|
||||
]
|
||||
component_loaders: ClassVar[dict[str, type[ComponentLoader]]] = {
|
||||
"transformer": SanaVideo2TransformerLoader
|
||||
}
|
||||
|
||||
def _load_config(self):
|
||||
self.model_path = maybe_download_model(
|
||||
self.model_path,
|
||||
revision=self.server_args.revision,
|
||||
allow_patterns=["config.yaml", "checkpoints/SANA_Video_2.0_5B_720p.pth"],
|
||||
)
|
||||
with (Path(self.model_path) / "config.yaml").open() as file:
|
||||
source = yaml.safe_load(file)
|
||||
self.server_args.pipeline_config.prompt_instruction = "\n".join(
|
||||
source["text_encoder"]["chi_prompt"]
|
||||
)
|
||||
return {
|
||||
"_class_name": self.pipeline_name,
|
||||
"_diffusers_version": "0.37.0",
|
||||
"text_encoder": ["transformers", "Gemma2Model"],
|
||||
"tokenizer": ["transformers", "AutoTokenizer"],
|
||||
"vae": ["diffusers", "AutoencoderKLLTX2Video"],
|
||||
"transformer": ["diffusers", "SanaVideo2Transformer3DModel"],
|
||||
}
|
||||
|
||||
def _resolve_component_path(self, server_args, module_name, load_module_name):
|
||||
path = server_args.component_paths.get(module_name)
|
||||
if path is None:
|
||||
if module_name == "transformer":
|
||||
return self.model_path
|
||||
if module_name == "vae":
|
||||
root = maybe_download_model(
|
||||
"Efficient-Large-Model/LTX-2.3-Diffusers", allow_patterns=["vae/**"]
|
||||
)
|
||||
return str(Path(root) / "vae")
|
||||
path = server_args.component_paths.get(
|
||||
"text_encoder", "Efficient-Large-Model/gemma-2-2b-it"
|
||||
)
|
||||
path = maybe_download_model(path)
|
||||
if module_name == "vae" and (Path(path) / "vae/config.json").is_file():
|
||||
path = str(Path(path) / "vae")
|
||||
return path
|
||||
|
||||
def create_pipeline_stages(self, server_args):
|
||||
self.add_stage(InputValidationStage())
|
||||
self.add_stage(
|
||||
SanaVideo2TextEncodingStage(
|
||||
[self.get_module("text_encoder")],
|
||||
[self.get_module("tokenizer")],
|
||||
server_args.pipeline_config.prompt_instruction,
|
||||
),
|
||||
"prompt_encoding_stage_primary",
|
||||
)
|
||||
self.add_stage(SanaVideo2LatentPreparationStage(self.get_module("vae")))
|
||||
self.add_stage(SanaVideo2DenoisingStage(self.get_module("transformer")))
|
||||
self.add_standard_decoding_stage()
|
||||
|
||||
|
||||
EntryClass = SanaVideo2Pipeline
|
||||
+280
@@ -0,0 +1,280 @@
|
||||
# Copyright 2024-2026 NVIDIA CORPORATION & AFFILIATES
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Prompt conditioning and request-local flow solvers for SANA-Video 2.0."""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from diffusers import FlowMatchEulerDiscreteScheduler
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from PIL import Image
|
||||
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines.sana_video import (
|
||||
select_sana_video_prompt_window,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
|
||||
TextEncodingStage,
|
||||
)
|
||||
|
||||
|
||||
def sample_flow_dpm(predict_noise, latents, steps, shift, callback=None):
|
||||
"""Second-order multistep DPM-Solver++ with first-order endpoint steps."""
|
||||
if steps < 2:
|
||||
raise ValueError("Flow DPM-Solver requires at least two inference steps")
|
||||
betas = torch.linspace(1.0, 0.001, steps + 1, device="cpu").to(latents.device)
|
||||
sigmas = 1.0 - betas
|
||||
times = (shift * sigmas / (1 + (shift - 1) * sigmas)).flip(0)
|
||||
previous_time = previous_prediction = None
|
||||
x = latents
|
||||
for index in range(steps):
|
||||
s = times[index].reshape((1,) * x.ndim)
|
||||
t = times[index + 1].reshape((1,) * x.ndim)
|
||||
noise = predict_noise(x, times[index])
|
||||
prediction = (x - s * noise) / (1 - s)
|
||||
lambda_s = torch.log(1 - s) - torch.log(s)
|
||||
lambda_t = torch.log(1 - t) - torch.log(t)
|
||||
h = lambda_t - lambda_s
|
||||
alpha_t = torch.exp(torch.log(1 - t))
|
||||
phi = torch.expm1(-h)
|
||||
if index == 0 or index == steps - 1:
|
||||
x = t / s * x - alpha_t * phi * prediction
|
||||
else:
|
||||
lambda_previous = torch.log(1 - previous_time) - torch.log(previous_time)
|
||||
ratio = (lambda_s - lambda_previous) / h
|
||||
derivative = (1.0 / ratio) * (prediction - previous_prediction)
|
||||
x = (
|
||||
t / s * x
|
||||
- alpha_t * phi * prediction
|
||||
- 0.5 * (alpha_t * phi) * derivative
|
||||
)
|
||||
previous_time, previous_prediction = s, prediction
|
||||
if callback is not None:
|
||||
callback(index, times[index + 1], x)
|
||||
return x
|
||||
|
||||
|
||||
def sample_ltx_euler(predict_flow, latents, steps, shift, callback=None):
|
||||
"""Flow Euler with a clean, fixed first latent frame."""
|
||||
with torch.device("cpu"):
|
||||
scheduler = FlowMatchEulerDiscreteScheduler(shift=shift)
|
||||
scheduler.set_timesteps(steps, device=latents.device)
|
||||
condition_mask = torch.zeros_like(latents, dtype=torch.float32)
|
||||
condition_mask[:, :, 0] = 1
|
||||
for index, time in enumerate(scheduler.timesteps):
|
||||
timestep = torch.minimum(
|
||||
time.expand(latents.shape).float(), (1 - condition_mask) * 1000.0
|
||||
)
|
||||
prediction = predict_flow(latents, timestep[:, :1, :, :1, :1])
|
||||
batch, channels = latents.shape[:2]
|
||||
updated = (
|
||||
scheduler.step(
|
||||
-prediction.reshape(batch, channels, -1).transpose(1, 2),
|
||||
time,
|
||||
latents.reshape(batch, channels, -1).transpose(1, 2),
|
||||
per_token_timesteps=timestep.reshape(batch, channels, -1)[:, 0],
|
||||
return_dict=False,
|
||||
)[0]
|
||||
.transpose(1, 2)
|
||||
.reshape(latents.shape)
|
||||
)
|
||||
denoise_mask = time / 1000 - 1e-6 < (1.0 - condition_mask)
|
||||
latents = torch.where(denoise_mask, updated, latents).to(latents.dtype)
|
||||
if callback is not None:
|
||||
callback(index, time, latents)
|
||||
return latents
|
||||
|
||||
|
||||
class SanaVideo2TextEncodingStage(TextEncodingStage):
|
||||
def __init__(self, text_encoders, tokenizers, instruction):
|
||||
super().__init__(text_encoders, tokenizers)
|
||||
self.instruction = instruction
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch, server_args):
|
||||
self.tokenizers[0].padding_side = "right"
|
||||
length = batch.max_sequence_length or 300
|
||||
prompts = [batch.prompt] if isinstance(batch.prompt, str) else batch.prompt
|
||||
motion = batch.extra.get("motion_score", 10)
|
||||
suffix = (
|
||||
f" motion score: {int(motion)}."
|
||||
if motion > 0
|
||||
else ""
|
||||
if motion < 0
|
||||
else " high motion"
|
||||
if batch.extra.get("high_motion", False)
|
||||
else " low motion"
|
||||
)
|
||||
prompts = [self.instruction + prompt.strip() + suffix for prompt in prompts]
|
||||
encoded_length = (
|
||||
len(self.tokenizers[0].encode(self.instruction)) + length - 2
|
||||
if self.instruction
|
||||
else length
|
||||
)
|
||||
outputs = list(
|
||||
self.encode_text(
|
||||
prompts,
|
||||
server_args,
|
||||
return_attention_mask=True,
|
||||
max_length=encoded_length,
|
||||
)
|
||||
)
|
||||
for index in (0, 1, 3):
|
||||
outputs[index] = [
|
||||
select_sana_video_prompt_window(value, length)
|
||||
for value in outputs[index]
|
||||
]
|
||||
outputs[4] = [
|
||||
[int(value) for value in mask.sum(dim=1).tolist()] for mask in outputs[1]
|
||||
]
|
||||
self._append_positive_text_outputs(batch, *outputs)
|
||||
if batch.do_classifier_free_guidance:
|
||||
negative = self.encode_text(
|
||||
batch.negative_prompt,
|
||||
server_args,
|
||||
return_attention_mask=True,
|
||||
max_length=length,
|
||||
)
|
||||
self._append_negative_text_outputs(batch, outputs[0], *negative)
|
||||
return batch
|
||||
|
||||
|
||||
def prepare_image(image, height, width):
|
||||
image = image.convert("RGB")
|
||||
w, h = image.size
|
||||
scale = max(height / h, width / w)
|
||||
resized = image.resize(
|
||||
(round(w * scale), round(h * scale)), Image.Resampling.BICUBIC
|
||||
)
|
||||
top = round((resized.height - height) / 2.0)
|
||||
left = round((resized.width - width) / 2.0)
|
||||
pixels = np.array(resized)[top : top + height, left : left + width].copy()
|
||||
return (
|
||||
torch.from_numpy(pixels).permute(2, 0, 1).float().div_(255).sub_(0.5).div_(0.5)
|
||||
)
|
||||
|
||||
|
||||
class SanaVideo2LatentPreparationStage(PipelineStage):
|
||||
def __init__(self, vae):
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
|
||||
def component_uses(self, server_args, stage_name=None):
|
||||
return [ComponentUse(self._component_stage_name(stage_name), "vae")]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch, server_args):
|
||||
config = server_args.pipeline_config
|
||||
if batch.height % 32 or batch.width % 32:
|
||||
raise ValueError("SANA-Video 2.0 height and width must be divisible by 32")
|
||||
batch.num_frames = config.adjust_num_frames(batch.num_frames)
|
||||
device = batch.prompt_embeds[0].device
|
||||
batch_size = batch.prompt_embeds[0].shape[0]
|
||||
shape = config.prepare_latent_shape(
|
||||
batch, batch_size, (batch.num_frames - 1) // 8 + 1
|
||||
)
|
||||
if batch.latents is None:
|
||||
batch.latents = randn_tensor(
|
||||
shape, generator=batch.generator, device=device, dtype=torch.float32
|
||||
)
|
||||
else:
|
||||
if tuple(batch.latents.shape) != shape:
|
||||
raise ValueError(
|
||||
f"Expected latents with shape {shape}, got {tuple(batch.latents.shape)}"
|
||||
)
|
||||
batch.latents = batch.latents.to(device=device, dtype=torch.float32).clone()
|
||||
if batch.condition_image is not None:
|
||||
images = (
|
||||
batch.condition_image
|
||||
if isinstance(batch.condition_image, list)
|
||||
else [batch.condition_image]
|
||||
)
|
||||
if len(images) == 1:
|
||||
images = images * batch_size
|
||||
if len(images) != batch_size:
|
||||
raise ValueError("TI2V requires one conditioning image per prompt")
|
||||
self.begin_declared_component_use(component_name="vae", module=self.vae)
|
||||
pixels = torch.stack(
|
||||
[prepare_image(image, batch.height, batch.width) for image in images]
|
||||
)[:, :, None]
|
||||
pixels = pixels.to(device=device, dtype=next(self.vae.parameters()).dtype)
|
||||
image_latents = self.vae.encode(pixels).latent_dist.mode()
|
||||
scale, mean = config.get_decode_scale_and_shift(
|
||||
device, image_latents.dtype, self.vae
|
||||
)
|
||||
batch.latents[:, :, :1] = (image_latents - mean) * scale
|
||||
return batch
|
||||
|
||||
|
||||
class SanaVideo2DenoisingStage(PipelineStage):
|
||||
def __init__(self, transformer):
|
||||
super().__init__()
|
||||
self.transformer = transformer
|
||||
|
||||
@property
|
||||
def role_affinity(self) -> RoleType:
|
||||
return RoleType.DENOISER
|
||||
|
||||
def component_uses(self, server_args, stage_name=None):
|
||||
return [ComponentUse(self._component_stage_name(stage_name), "transformer")]
|
||||
|
||||
def default_workload_iterations(self, batch, num_inference_steps):
|
||||
return num_inference_steps
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch, server_args):
|
||||
self.begin_declared_component_use(
|
||||
component_name="transformer", module=self.transformer
|
||||
)
|
||||
cfg = batch.do_classifier_free_guidance
|
||||
embeds = batch.prompt_embeds[0]
|
||||
mask = batch.prompt_attention_mask[0]
|
||||
if cfg:
|
||||
embeds = torch.cat([batch.negative_prompt_embeds[0], embeds])
|
||||
mask = torch.cat([batch.negative_attention_mask[0], mask])
|
||||
is_ti2v = batch.condition_image is not None
|
||||
|
||||
def predict(x, time):
|
||||
inputs = torch.cat([x, x]) if cfg else x
|
||||
if time.ndim == 0:
|
||||
timestep = (time * 1000).expand(inputs.shape[0])
|
||||
else:
|
||||
timestep = torch.cat([time, time]) if cfg else time
|
||||
prediction = self.transformer(
|
||||
hidden_states=inputs,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=embeds,
|
||||
encoder_attention_mask=mask,
|
||||
)
|
||||
if not is_ti2v:
|
||||
# Keep flow-to-noise conversion before CFG to preserve FP32 rounding.
|
||||
sigma = time.reshape((1,) * x.ndim).to(inputs)
|
||||
prediction = (1 - sigma) * prediction + inputs
|
||||
if cfg:
|
||||
uncond, cond = prediction.chunk(2)
|
||||
prediction = uncond + batch.guidance_scale * (cond - uncond)
|
||||
return prediction
|
||||
|
||||
steps = (
|
||||
max(2, batch.num_inference_steps)
|
||||
if batch.is_warmup
|
||||
else batch.num_inference_steps
|
||||
)
|
||||
sampler = sample_ltx_euler if is_ti2v else sample_flow_dpm
|
||||
flow_shift = (
|
||||
batch.flow_shift
|
||||
if batch.flow_shift is not None
|
||||
else server_args.pipeline_config.flow_shift
|
||||
)
|
||||
with self.progress_bar(total=steps, batch=batch) as progress:
|
||||
batch.latents = sampler(
|
||||
predict,
|
||||
batch.latents,
|
||||
steps,
|
||||
flow_shift,
|
||||
callback=lambda *_: progress.update(),
|
||||
)
|
||||
return batch
|
||||
@@ -0,0 +1,158 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import unittest
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class TestSanaVideo2(unittest.TestCase):
|
||||
def test_linear_attention_matches_explicit_token_sum(self):
|
||||
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
|
||||
GatedLinearAttention,
|
||||
)
|
||||
|
||||
torch.manual_seed(12)
|
||||
module = GatedLinearAttention(24, 12, fp32_attention=True).eval()
|
||||
x = torch.randn(2, 5, 24)
|
||||
q, k, v = module.qkv(x).chunk(3, dim=-1)
|
||||
q = module.q_norm(q).reshape(2, 5, 2, 12)
|
||||
k = module.k_norm(k).reshape(2, 5, 2, 12)
|
||||
v = v.reshape(2, 5, 2, 12)
|
||||
beta = module.beta_proj(x).sigmoid()
|
||||
scores = torch.einsum("bihd,bjhd->bhij", q, k)
|
||||
out = torch.einsum("bhij,bjh,bjhd->bihd", scores, beta, v)
|
||||
out = out.float() * (out.float().square().mean(-1, keepdim=True) + 1e-5).rsqrt()
|
||||
out = out * module.o_norm.weight
|
||||
out = out.reshape_as(x) * module.output_gate(x).sigmoid()
|
||||
expected = module.proj(out)
|
||||
torch.testing.assert_close(module(x), expected, atol=2e-6, rtol=2e-5)
|
||||
|
||||
def test_softmax_attention_matches_dense_equation(self):
|
||||
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
|
||||
GatedSoftmaxAttention,
|
||||
)
|
||||
|
||||
torch.manual_seed(13)
|
||||
module = GatedSoftmaxAttention(24, 12, fp32_attention=True).eval()
|
||||
x = torch.randn(2, 5, 24)
|
||||
q, k, v = module.qkv(x).chunk(3, -1)
|
||||
q = module.q_norm(q).reshape(2, 5, 2, 12)
|
||||
k = module.k_norm(k).reshape(2, 5, 2, 12)
|
||||
v = v.reshape(2, 5, 2, 12)
|
||||
scores = torch.einsum("bihd,bjhd->bhij", q, k) / 12**0.5
|
||||
out = torch.einsum("bhij,bjhd->bihd", scores.softmax(-1), v)
|
||||
expected = module.proj(out.reshape_as(x) * module.output_gate(x).sigmoid())
|
||||
torch.testing.assert_close(module(x), expected)
|
||||
|
||||
def test_attention_residual_weights_depth_per_token(self):
|
||||
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
|
||||
BlockAttentionResidual,
|
||||
)
|
||||
|
||||
torch.manual_seed(14)
|
||||
module = BlockAttentionResidual(4)
|
||||
with torch.no_grad():
|
||||
module.attn_proj.weight.copy_(torch.tensor([[0.3, -0.4, 0.5, 0.1]]))
|
||||
values = torch.randn(4, 2, 3, 4)
|
||||
keys = values * (values.square().mean(-1, keepdim=True) + 1e-6).rsqrt()
|
||||
scores = (keys * module.attn_proj.weight).sum(-1).softmax(0)
|
||||
expected = (scores.unsqueeze(-1) * values).sum(0)
|
||||
actual = module.attend_buffer(
|
||||
module.attn_proj, values.clone(), keys.clone(), 3, values[3]
|
||||
)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
@staticmethod
|
||||
def small_model():
|
||||
from sglang.multimodal_gen.configs.models.dits.sana_video2 import (
|
||||
SanaVideo2ArchConfig,
|
||||
SanaVideo2Config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
|
||||
SanaVideo2Transformer3DModel,
|
||||
)
|
||||
|
||||
return SanaVideo2Transformer3DModel(
|
||||
SanaVideo2Config(
|
||||
arch_config=SanaVideo2ArchConfig(
|
||||
hidden_size=24,
|
||||
depth=4,
|
||||
num_heads=2,
|
||||
linear_head_dim=12,
|
||||
softmax_head_dim=24,
|
||||
in_channels=4,
|
||||
caption_channels=8,
|
||||
model_max_length=5,
|
||||
mlp_ratio=2,
|
||||
attn_res_block_size=2,
|
||||
)
|
||||
)
|
||||
).eval()
|
||||
|
||||
def test_uniform_frame_timesteps_equal_batch_timesteps(self):
|
||||
torch.manual_seed(15)
|
||||
model = self.small_model()
|
||||
x, y = torch.randn(2, 4, 3, 2, 2), torch.randn(2, 5, 8)
|
||||
t = torch.tensor([42.9, 123.9])
|
||||
mask = torch.tensor([[1, 1, 1, 0, 0], [1, 1, 1, 1, 0]])
|
||||
with torch.no_grad():
|
||||
batch = model(x, t, y, mask)
|
||||
frames = model(
|
||||
x, t[:, None, None, None, None].expand(-1, 1, 3, 1, 1), y[:, None], mask
|
||||
)
|
||||
changed_t = t[:, None, None, None, None].expand(-1, 1, 3, 1, 1).clone()
|
||||
changed_t[:, :, 0] = 0
|
||||
changed = model(x, changed_t, y, mask)
|
||||
self.assertEqual(batch.shape, x.shape)
|
||||
torch.testing.assert_close(batch, frames)
|
||||
self.assertFalse(torch.allclose(batch, changed))
|
||||
self.assertEqual(
|
||||
model.block_attention_types, ["linear", "linear", "linear", "softmax"]
|
||||
)
|
||||
|
||||
def test_padding_tokens_do_not_change_flow_and_calls_do_not_share_shape(self):
|
||||
torch.manual_seed(16)
|
||||
model = self.small_model()
|
||||
x, y, t = torch.randn(1, 4, 2, 2, 2), torch.randn(1, 5, 8), torch.tensor([20.0])
|
||||
mask = torch.tensor([[1, 1, 1, 0, 0]])
|
||||
with torch.no_grad():
|
||||
original = model(x, t, y, mask)
|
||||
y[:, 3:] = 1000
|
||||
model(torch.randn(1, 4, 1, 1, 3), t, y, mask)
|
||||
actual = model(x, t, y, mask)
|
||||
torch.testing.assert_close(actual, original)
|
||||
|
||||
def test_dtype_cast_preserves_complex_rotary_frequencies(self):
|
||||
model = self.small_model()
|
||||
before = model.rope_linear((2, 2, 2), torch.device("cpu"))
|
||||
model.to(dtype=torch.bfloat16)
|
||||
after = model.rope_linear((2, 2, 2), torch.device("cpu"))
|
||||
self.assertTrue(after.is_complex())
|
||||
torch.testing.assert_close(after, before, rtol=0, atol=0)
|
||||
|
||||
def test_official_checkpoint_names_and_output_sign(self):
|
||||
torch.manual_seed(17)
|
||||
model = self.small_model()
|
||||
state = model.state_dict()
|
||||
for key in (
|
||||
"pos_embed",
|
||||
"y_embedder.y_embedding",
|
||||
"blocks.0.attn.beta_proj.weight",
|
||||
"blocks.3.attn.output_gate.weight",
|
||||
"blocks.0.cross_attn.kv_linear.weight",
|
||||
"blocks.0.mlp.gate_proj.weight",
|
||||
"attn_res.final_proj.weight",
|
||||
):
|
||||
self.assertIn(key, state)
|
||||
model.load_state_dict(state, strict=True)
|
||||
with torch.no_grad():
|
||||
model.final_layer.linear.weight.zero_()
|
||||
model.final_layer.linear.bias.fill_(2)
|
||||
output = model(
|
||||
torch.randn(1, 4, 2, 2, 2), torch.tensor([1.0]), torch.randn(1, 5, 8)
|
||||
)
|
||||
torch.testing.assert_close(output, torch.full_like(output, 2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,323 @@
|
||||
from contextlib import nullcontext
|
||||
from itertools import pairwise
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.sana_video2 import (
|
||||
SanaVideo2PipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sana_video2 import SanaVideo2SamplingParams
|
||||
from sglang.multimodal_gen.registry import get_model_info
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages import (
|
||||
sana_video2,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import (
|
||||
sample_flow_dpm,
|
||||
sample_ltx_euler,
|
||||
)
|
||||
|
||||
|
||||
def test_native_registry_precedes_sana_video(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"sglang.multimodal_gen.registry.maybe_download_model_index",
|
||||
lambda _: None,
|
||||
)
|
||||
get_model_info.cache_clear()
|
||||
info = get_model_info("Efficient-Large-Model/SANA-Video_2.0_5B_720p")
|
||||
assert info.pipeline_config_cls is SanaVideo2PipelineConfig
|
||||
assert info.sampling_param_cls is SanaVideo2SamplingParams
|
||||
get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_latent_layout_and_frame_alignment():
|
||||
config = SanaVideo2PipelineConfig()
|
||||
assert config.adjust_num_frames(193) == 193
|
||||
assert config.adjust_num_frames(192) == 185
|
||||
assert config.prepare_latent_shape(
|
||||
SimpleNamespace(height=736, width=1280), 2, 25
|
||||
) == (2, 128, 25, 23, 40)
|
||||
assert config.get_latent_dtype(torch.bfloat16) == torch.float32
|
||||
defaults = SanaVideo2SamplingParams()
|
||||
assert (defaults.height, defaults.width, defaults.num_frames, defaults.fps) == (
|
||||
736,
|
||||
1280,
|
||||
193,
|
||||
24,
|
||||
)
|
||||
assert defaults.num_inference_steps == 50
|
||||
|
||||
|
||||
def test_dpm_constant_data_prediction_reaches_clean_endpoint():
|
||||
initial = torch.tensor([[[[[0.4, -0.7]]]]], dtype=torch.float32)
|
||||
clean = torch.tensor([[[[[0.2, 0.9]]]]], dtype=torch.float32)
|
||||
times = []
|
||||
|
||||
def predict_noise(x, time):
|
||||
times.append(time.item())
|
||||
return (x - (1 - time) * clean) / time
|
||||
|
||||
actual = sample_flow_dpm(predict_noise, initial, steps=5, shift=12.0)
|
||||
torch.testing.assert_close(actual, clean, atol=1e-5, rtol=1e-5)
|
||||
assert len(times) == 5
|
||||
assert all(a > b for a, b in pairwise(times))
|
||||
assert actual.dtype == torch.float32
|
||||
|
||||
|
||||
def test_ti2v_euler_preserves_condition_and_uses_frame_timestep():
|
||||
initial = torch.zeros(1, 2, 3, 1, 1)
|
||||
initial[:, :, 0] = 7
|
||||
seen = []
|
||||
|
||||
def predict_flow(x, time):
|
||||
seen.append(time.clone())
|
||||
return torch.ones_like(x)
|
||||
|
||||
actual = sample_ltx_euler(predict_flow, initial.clone(), steps=4, shift=12.0)
|
||||
torch.testing.assert_close(actual[:, :, 0], initial[:, :, 0], atol=0, rtol=0)
|
||||
torch.testing.assert_close(actual[:, :, 1:], -torch.ones_like(actual[:, :, 1:]))
|
||||
assert len(seen) == 4
|
||||
assert all(time.shape == (1, 1, 3, 1, 1) for time in seen)
|
||||
assert all(torch.count_nonzero(time[:, :, 0]) == 0 for time in seen)
|
||||
assert all(torch.all(time[:, :, 1:] > 0) for time in seen)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("conditioned", [False, True])
|
||||
@pytest.mark.parametrize("flow_shift, expected", [(None, 12.0), (3.0, 3.0)])
|
||||
def test_denoising_uses_request_flow_shift(
|
||||
monkeypatch, conditioned, flow_shift, expected
|
||||
):
|
||||
shifts = []
|
||||
|
||||
def sample(predict, latents, steps, shift, callback):
|
||||
shifts.append(shift)
|
||||
return latents
|
||||
|
||||
monkeypatch.setattr(
|
||||
sana_video2, "sample_ltx_euler" if conditioned else "sample_flow_dpm", sample
|
||||
)
|
||||
stage = object.__new__(sana_video2.SanaVideo2DenoisingStage)
|
||||
stage.transformer = None
|
||||
stage.begin_declared_component_use = lambda **_: None
|
||||
stage.progress_bar = lambda **_: nullcontext()
|
||||
batch = SimpleNamespace(
|
||||
do_classifier_free_guidance=False,
|
||||
prompt_embeds=[torch.zeros(1, 300, 4)],
|
||||
prompt_attention_mask=[torch.ones(1, 300)],
|
||||
condition_image=object() if conditioned else None,
|
||||
latents=torch.zeros(1, 2, 3, 1, 1),
|
||||
num_inference_steps=2,
|
||||
is_warmup=False,
|
||||
flow_shift=flow_shift,
|
||||
)
|
||||
stage.forward(batch, SimpleNamespace(pipeline_config=SanaVideo2PipelineConfig()))
|
||||
assert shifts == [expected]
|
||||
|
||||
|
||||
def test_denoising_stage_role():
|
||||
stage = object.__new__(sana_video2.SanaVideo2DenoisingStage)
|
||||
assert stage.role_affinity is RoleType.DENOISER
|
||||
|
||||
|
||||
def test_vae_statistics_are_inverted_for_decode():
|
||||
config = SanaVideo2PipelineConfig()
|
||||
config.vae_config.arch_config.scaling_factor = 2.0
|
||||
vae = SimpleNamespace(
|
||||
latents_mean=torch.tensor([1.0, -1.0]), latents_std=torch.tensor([2.0, 4.0])
|
||||
)
|
||||
scale, shift = config.get_decode_scale_and_shift(
|
||||
torch.device("cpu"), torch.float32, vae
|
||||
)
|
||||
normalized = torch.tensor([2.0, 3.0]).reshape(1, 2, 1, 1, 1)
|
||||
decoded = normalized / scale + shift
|
||||
torch.testing.assert_close(decoded.flatten(), torch.tensor([3.0, 5.0]))
|
||||
|
||||
|
||||
def test_prompt_window_preserves_case_and_only_positive_motion_suffix():
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import (
|
||||
SanaVideo2TextEncodingStage,
|
||||
)
|
||||
|
||||
stage = object.__new__(SanaVideo2TextEncodingStage)
|
||||
stage.instruction = "instruction: "
|
||||
stage.tokenizers = [SimpleNamespace(encode=lambda _: [1, 2, 3, 4, 5])]
|
||||
calls = []
|
||||
|
||||
def encode(text, server_args, **kwargs):
|
||||
calls.append((text, kwargs["max_length"]))
|
||||
length = kwargs["max_length"]
|
||||
values = torch.arange(length).reshape(1, length, 1)
|
||||
mask = torch.ones(1, length, dtype=torch.long)
|
||||
return [values], [mask], [], [mask.bool()], [[length]]
|
||||
|
||||
stage.encode_text = encode
|
||||
batch = SimpleNamespace(
|
||||
prompt=" A RED fox ",
|
||||
negative_prompt="Bad motion",
|
||||
max_sequence_length=4,
|
||||
extra={"motion_score": 10},
|
||||
do_classifier_free_guidance=True,
|
||||
prompt_embeds=[],
|
||||
pooled_embeds=[],
|
||||
prompt_attention_mask=None,
|
||||
negative_prompt_embeds=[],
|
||||
negative_pooled_embeds=[],
|
||||
negative_attention_mask=None,
|
||||
)
|
||||
stage.forward(batch, SimpleNamespace())
|
||||
assert calls == [
|
||||
(["instruction: A RED fox motion score: 10."], 7),
|
||||
("Bad motion", 4),
|
||||
]
|
||||
assert batch.prompt_embeds[0].flatten().tolist() == [0, 4, 5, 6]
|
||||
assert batch.negative_prompt_embeds[0].flatten().tolist() == [0, 1, 2, 3]
|
||||
assert batch.prompt_attention_mask[0].shape == (1, 4)
|
||||
|
||||
|
||||
def test_native_model_index_and_component_directory_contract(tmp_path):
|
||||
import yaml
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines.sana_video2 import SanaVideo2Pipeline
|
||||
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
yaml.safe_dump({"text_encoder": {"chi_prompt": ["first", "last"]}})
|
||||
)
|
||||
vae_root = tmp_path / "ltx"
|
||||
(vae_root / "vae").mkdir(parents=True)
|
||||
(vae_root / "vae/config.json").write_text("{}")
|
||||
pipeline = object.__new__(SanaVideo2Pipeline)
|
||||
pipeline.model_path = str(tmp_path)
|
||||
pipeline.server_args = SimpleNamespace(
|
||||
revision=None,
|
||||
pipeline_config=SanaVideo2PipelineConfig(),
|
||||
component_paths={"vae": str(vae_root), "text_encoder": str(tmp_path)},
|
||||
)
|
||||
index = pipeline._load_config()
|
||||
assert index["_class_name"] == "SanaVideo2Pipeline"
|
||||
assert "_diffusers_version" in index
|
||||
assert pipeline.server_args.pipeline_config.prompt_instruction == "first\nlast"
|
||||
assert pipeline._resolve_component_path(pipeline.server_args, "vae", "vae") == str(
|
||||
vae_root / "vae"
|
||||
)
|
||||
assert pipeline._resolve_component_path(
|
||||
pipeline.server_args, "tokenizer", "tokenizer"
|
||||
) == str(tmp_path)
|
||||
|
||||
|
||||
def test_released_pth_layout_loads_strictly_and_materializes_rope(tmp_path):
|
||||
import yaml
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.sana_video2 import SanaVideo2Config
|
||||
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
|
||||
SanaVideo2Transformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines.sana_video2 import (
|
||||
SanaVideo2TransformerLoader,
|
||||
)
|
||||
|
||||
arch = {
|
||||
"hidden_size": 32,
|
||||
"depth": 2,
|
||||
"num_heads": 4,
|
||||
"linear_head_dim": 8,
|
||||
"softmax_head_dim": 8,
|
||||
"softmax_ratio": 0.5,
|
||||
"in_channels": 4,
|
||||
"caption_channels": 12,
|
||||
"model_max_length": 5,
|
||||
"input_size": 2,
|
||||
}
|
||||
config = SanaVideo2Config()
|
||||
config.update_model_arch(arch)
|
||||
original = SanaVideo2Transformer3DModel(config)
|
||||
(tmp_path / "checkpoints").mkdir()
|
||||
torch.save(
|
||||
{"state_dict": original.state_dict()},
|
||||
tmp_path / "checkpoints/SANA_Video_2.0_5B_720p.pth",
|
||||
)
|
||||
(tmp_path / "config.yaml").write_text(
|
||||
yaml.safe_dump({"model": arch, "text_encoder": {}})
|
||||
)
|
||||
pipeline_config = SanaVideo2PipelineConfig()
|
||||
pipeline_config.dit_precision = "fp32"
|
||||
args = SimpleNamespace(
|
||||
pipeline_config=pipeline_config, component_precisions={}, model_paths={}
|
||||
)
|
||||
loaded = SanaVideo2TransformerLoader().load_customized(
|
||||
str(tmp_path), args, "transformer"
|
||||
)
|
||||
assert not any(value.is_meta for value in loaded.buffers())
|
||||
for name, value in original.state_dict().items():
|
||||
torch.testing.assert_close(loaded.state_dict()[name], value, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_local_native_root_does_not_require_diffusers_model_index(tmp_path):
|
||||
root = tmp_path / "sana-video2-5b"
|
||||
root.mkdir()
|
||||
(root / "config.yaml").write_text("model:\n model: SanaVideo2_5B\n")
|
||||
get_model_info.cache_clear()
|
||||
info = get_model_info(str(root))
|
||||
assert info.pipeline_config_cls is SanaVideo2PipelineConfig
|
||||
assert info.sampling_param_cls is SanaVideo2SamplingParams
|
||||
get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_diffusers_local_root_still_reads_model_index(tmp_path):
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.sana_video import (
|
||||
SanaVideoPipelineConfig,
|
||||
)
|
||||
|
||||
root = tmp_path / "sana-video-original"
|
||||
root.mkdir()
|
||||
(root / "transformer").mkdir()
|
||||
(root / "model_index.json").write_text(
|
||||
'{"_class_name": "SanaVideoPipeline", "_diffusers_version": "0.37.0", '
|
||||
'"transformer": ["diffusers", "SanaVideoTransformer3DModel"]}'
|
||||
)
|
||||
get_model_info.cache_clear()
|
||||
info = get_model_info(str(root))
|
||||
assert info.pipeline_config_cls is SanaVideoPipelineConfig
|
||||
get_model_info.cache_clear()
|
||||
|
||||
|
||||
def test_two_outputs_expand_with_independent_generators_before_latent_preparation():
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import (
|
||||
InputValidationStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import (
|
||||
SanaVideo2LatentPreparationStage,
|
||||
)
|
||||
|
||||
sampling = SanaVideo2SamplingParams(
|
||||
request_id="two-videos",
|
||||
prompt="A fox",
|
||||
height=32,
|
||||
width=32,
|
||||
num_frames=9,
|
||||
num_outputs_per_prompt=2,
|
||||
seed=[11, 12],
|
||||
generator_device="cpu",
|
||||
output_path="/tmp",
|
||||
output_file_name="video.mp4",
|
||||
)
|
||||
request = Req(sampling_params=sampling)
|
||||
args = SimpleNamespace(pipeline_config=SanaVideo2PipelineConfig())
|
||||
requests = list(InputValidationStage().iter_sequential_requests(request, args))
|
||||
assert len(requests) == 2
|
||||
assert [item.num_outputs_per_prompt for item in requests] == [1, 1]
|
||||
assert [item.seed for item in requests] == [11, 12]
|
||||
stage = SanaVideo2LatentPreparationStage(vae=None)
|
||||
latents = []
|
||||
for item in requests:
|
||||
item.prompt_embeds = [torch.zeros(1, 5, 12)]
|
||||
stage.forward(item, args)
|
||||
assert item.latents.shape == (1, 128, 2, 1, 1)
|
||||
expected = torch.randn(
|
||||
item.latents.shape, generator=torch.Generator().manual_seed(item.seed)
|
||||
)
|
||||
torch.testing.assert_close(item.latents, expected)
|
||||
latents.append(item.latents)
|
||||
assert not torch.equal(latents[0], latents[1])
|
||||
Reference in New Issue
Block a user