[diffusion][model] Add native SANA-Video 2.0 support (#41492)

Signed-off-by: cr-gao <gaochenrui@sjtu.edu.cn>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
vOv
2026-10-09 17:28:03 +08:00
committed by GitHub
co-authored by Xiaoyu Zhang
parent 4debae7534
commit 7cfecbef56
12 changed files with 1507 additions and 4 deletions
@@ -0,0 +1,52 @@
# SPDX-License-Identifier: Apache-2.0
from dataclasses import dataclass, field
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
@dataclass
class SanaVideo2ArchConfig(DiTArchConfig):
input_size: int = 15
patch_size: tuple[int, int, int] = (1, 1, 1)
in_channels: int = 128
hidden_size: int = 2560
depth: int = 32
num_heads: int = 20
linear_head_dim: int = 128
softmax_head_dim: int = 256
softmax_ratio: float = 0.25
mlp_ratio: float = 4.0
caption_channels: int = 2304
model_max_length: int = 300
qk_norm: bool = True
cross_norm: bool = True
y_norm: bool = True
y_norm_scale_factor: float = 0.01
norm_eps: float = 1e-5
attn_res_block_size: int = 8
timestep_norm_scale_factor: float = 1.0
fp32_attention: bool = True
def __post_init__(self):
super().__post_init__()
self.patch_size = tuple(self.patch_size)
if self.patch_size != (1, 1, 1):
raise ValueError("SANA-Video 2.0 requires patch_size=(1, 1, 1)")
if self.depth < 1 or self.attn_res_block_size < 1:
raise ValueError("depth and attn_res_block_size must be positive")
if not 0 < self.softmax_ratio <= 1:
raise ValueError("softmax_ratio must be in (0, 1]")
if any(
self.hidden_size % dim
for dim in (self.linear_head_dim, self.softmax_head_dim, self.num_heads)
):
raise ValueError("hidden_size must divide evenly into attention heads")
self.num_attention_heads = self.num_heads
self.num_channels_latents = self.in_channels
@dataclass
class SanaVideo2Config(DiTConfig):
arch_config: DiTArchConfig = field(default_factory=SanaVideo2ArchConfig)
prefix: str = "SanaVideo2"
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
"""Native SANA-Video 2.0 pipeline configuration."""
from dataclasses import dataclass, field
from sglang.multimodal_gen.configs.models.dits.sana_video2 import SanaVideo2Config
from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig
from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType
from sglang.multimodal_gen.configs.pipeline_configs.sana_video import (
SanaVideoPipelineConfig,
)
@dataclass
class SanaVideo2VAEConfig(LTXVideoVAEConfig):
def update_model_arch(self, source_model_dict):
config = dict(source_model_dict)
if "upsample_type" not in config and "decoder_upsample_type" in config:
config["upsample_type"] = list(reversed(config["decoder_upsample_type"]))
super().update_model_arch(config)
@dataclass
class SanaVideo2PipelineConfig(SanaVideoPipelineConfig):
task_type: ModelTaskType = ModelTaskType.TI2V
flow_shift: float | None = 12.0
skip_input_image_preprocess: bool = True
dit_config: SanaVideo2Config = field(default_factory=SanaVideo2Config)
vae_config: SanaVideo2VAEConfig = field(default_factory=SanaVideo2VAEConfig)
vae_precision: str = "bf16"
vae_decode_precision: str = "bf16"
prompt_instruction: str = ""
def __post_init__(self):
self.vae_config.load_encoder = True
self.vae_config.load_decoder = True
def supports_sequential_multi_output_inference(self):
return True
def get_decode_scale_and_shift(self, device, dtype, vae):
mean = vae.latents_mean.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1)
std = vae.latents_std.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1)
return self.vae_config.arch_config.scaling_factor / std, mean
def register():
from sglang.multimodal_gen.configs.sample.sana_video2 import (
SanaVideo2SamplingParams,
)
from sglang.multimodal_gen.registry import register_configs
register_configs(
sampling_param_cls=SanaVideo2SamplingParams,
pipeline_config_cls=SanaVideo2PipelineConfig,
hf_model_paths=["Efficient-Large-Model/SANA-Video_2.0_5B_720p"],
model_detectors=[
lambda name: any(
key in name.lower()
for key in (
"sana-video_2.0",
"sana-video2",
"sana_video2",
"sanavideo2pipeline",
)
)
],
)
@@ -0,0 +1,28 @@
# SPDX-License-Identifier: Apache-2.0
"""Sampling defaults for the SANA-Video 2.0 5B checkpoint."""
from dataclasses import dataclass
from sglang.multimodal_gen.configs.sample.sana_video import SanaVideoSamplingParams
@dataclass
class SanaVideo2SamplingParams(SanaVideoSamplingParams):
height: int = 736
width: int = 1280
num_frames: int = 193
fps: int = 24
guidance_scale: float = 8.0
motion_score: int = 10
high_motion: bool = False
def build_request_extra(self):
return {
**super().build_request_extra(),
"motion_score": self.motion_score,
"high_motion": self.high_motion,
}
@classmethod
def video_request_extra_fields(cls):
return super().video_request_extra_fields() | {"motion_score", "high_motion"}
+9 -3
View File
@@ -173,6 +173,9 @@ _MODEL_NAME_DETECTORS: List[Tuple[str, Callable[[str], bool]]] = []
# aliases next to the resolver that consumes them so CLI detection and
# pipeline selection cannot drift apart
KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = {
"sana-video_2.0": "SanaVideo2Pipeline",
"sana-video2": "SanaVideo2Pipeline",
"sana_video2": "SanaVideo2Pipeline",
"ming-image-0.1-design": "MingImagePipeline",
"minimaxai/minimax-h3": "MiniMaxH3Pipeline",
"minimax/minimax-h3": "MiniMaxH3Pipeline",
@@ -291,7 +294,7 @@ _configs_discovered: bool = False
# SANA-WM (register BEFORE generic SANA T2I to prevent "sana" detector false-match)
# SANA-Video (register before generic SANA to avoid detector overlap).
_CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video")
_CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video2", "sana_video")
def _discover_and_register_configs() -> None:
@@ -462,8 +465,11 @@ def _get_config_info(
return _CONFIG_REGISTRY.get(model_id)
# 3. Use detectors
config = maybe_download_model_index(model_path)
pipeline_name = config.get("_class_name", "").lower()
pipeline_name = get_non_diffusers_pipeline_name(model_path)
if pipeline_name is None:
config = maybe_download_model_index(model_path)
pipeline_name = config.get("_class_name", "")
pipeline_name = pipeline_name.lower()
matched_model_names = []
for model_id, detector in _MODEL_NAME_DETECTORS:
@@ -0,0 +1,353 @@
# Copyright 2026 NVIDIA CORPORATION & AFFILIATES
# SPDX-License-Identifier: Apache-2.0
"""SANA-Video 2.0 inference transformer with grouped Attention Residuals."""
import math
from typing import ClassVar
import torch
from torch import nn
from torch.nn import functional as F
from sglang.multimodal_gen.configs.models.dits.sana_video2 import SanaVideo2Config
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
from sglang.multimodal_gen.runtime.models.dits.sana_wm_components import (
CaptionEmbedder,
PatchEmbedMS3D,
T2IFinalLayer,
TimestepEmbedder,
WanRotaryPosEmbed,
_RMSNorm,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
def _apply_rope(x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
rotated = torch.view_as_complex(x.to(torch.float64).unflatten(-1, (-1, 2)))
return torch.view_as_real(rotated * freqs).flatten(-2).type_as(x)
class SanaVideo2RotaryPosEmbed(WanRotaryPosEmbed):
def _apply(self, fn, recurse=True):
# Keep complex128 frequencies; forward moves them to the input device.
return self
class ChannelRMSNorm(_RMSNorm):
def forward(self, x: torch.Tensor) -> torch.Tensor:
normalized = (
x.float() * (x.float().square().mean(-2, keepdim=True) + self.eps).rsqrt()
)
return (self.weight[None, None, :, None] * normalized).type_as(x)
class GatedLinearAttention(nn.Module):
def __init__(self, dim, head_dim, qk_norm=True, fp32_attention=True):
super().__init__()
self.heads = dim // head_dim
self.dim = head_dim
self.fp32_attention = fp32_attention
self.qkv = nn.Linear(dim, 3 * dim, bias=False)
self.proj = nn.Linear(dim, dim)
self.q_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity()
self.k_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity()
self.beta_proj = nn.Linear(dim, self.heads)
self.output_gate = nn.Linear(dim, dim)
self.o_norm = ChannelRMSNorm(head_dim, eps=1e-5)
def forward(self, x, rotary_emb=None):
batch, tokens, channels = x.shape
q, k, v = self.qkv(x).reshape(batch, tokens, 3, channels).unbind(2)
dtype = q.dtype
q = (
self.q_norm(q)
.transpose(-1, -2)
.reshape(batch, self.heads, self.dim, tokens)
)
k = (
self.k_norm(k)
.transpose(-1, -2)
.reshape(batch, self.heads, self.dim, tokens)
)
v = v.transpose(-1, -2).reshape(batch, self.heads, self.dim, tokens)
if rotary_emb is not None:
q = _apply_rope(q.transpose(-1, -2), rotary_emb).transpose(-1, -2)
k = _apply_rope(k.transpose(-1, -2), rotary_emb).transpose(-1, -2)
beta = self.beta_proj(x).sigmoid().transpose(1, 2).unsqueeze(2)
k = k * beta
if self.fp32_attention:
q, k, v = q.float(), k.float(), v.float()
out = (v @ k.transpose(-1, -2)) @ q
out = self.o_norm(out.to(dtype))
out = out.reshape(batch, channels, tokens).transpose(1, 2)
return self.proj(out * self.output_gate(x).sigmoid())
class GatedSoftmaxAttention(nn.Module):
def __init__(self, dim, head_dim, qk_norm=True, fp32_attention=True):
super().__init__()
self.heads = dim // head_dim
self.dim = head_dim
self.fp32_attention = fp32_attention
self.qkv = nn.Linear(dim, 3 * dim, bias=False)
self.proj = nn.Linear(dim, dim)
self.q_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity()
self.k_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity()
self.output_gate = nn.Linear(dim, dim)
def forward(self, x, rotary_emb=None):
batch, tokens, channels = x.shape
q, k, v = self.qkv(x).reshape(batch, tokens, 3, channels).unbind(2)
dtype = q.dtype
q = self.q_norm(q).reshape(batch, tokens, self.heads, self.dim).transpose(1, 2)
k = self.k_norm(k).reshape(batch, tokens, self.heads, self.dim).transpose(1, 2)
v = v.reshape(batch, tokens, self.heads, self.dim).transpose(1, 2)
if rotary_emb is not None:
q, k = _apply_rope(q, rotary_emb), _apply_rope(k, rotary_emb)
if self.fp32_attention:
q, k, v = q.float(), k.float(), v.float()
out = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0)
out = out.transpose(1, 2).reshape(batch, tokens, channels).to(dtype)
return self.proj(out * self.output_gate(x).sigmoid())
class CrossAttention(nn.Module):
def __init__(self, dim, num_heads, qk_norm=True):
super().__init__()
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.q_linear = nn.Linear(dim, dim)
self.kv_linear = nn.Linear(dim, 2 * dim)
self.proj = nn.Linear(dim, dim)
self.q_norm = _RMSNorm(dim, eps=1e-6) if qk_norm else nn.Identity()
self.k_norm = _RMSNorm(dim, eps=1e-6) if qk_norm else nn.Identity()
def forward(self, x, y, mask=None):
batch, tokens, channels = x.shape
q = self.q_norm(self.q_linear(x))
k, v = self.kv_linear(y).reshape(batch, -1, 2, channels).unbind(2)
q = q.reshape(batch, tokens, self.num_heads, self.head_dim).transpose(1, 2)
k = (
self.k_norm(k)
.reshape(batch, -1, self.num_heads, self.head_dim)
.transpose(1, 2)
)
v = v.reshape(batch, -1, self.num_heads, self.head_dim).transpose(1, 2)
if mask is not None:
mask = (1 - mask.to(q.dtype))[:, None, None, :] * -10000.0
out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0)
return self.proj(out.transpose(1, 2).reshape(batch, tokens, channels))
class SwiGLU(nn.Module):
def __init__(self, dim, hidden_dim):
super().__init__()
self.gate_proj = nn.Linear(dim, hidden_dim)
self.up_proj = nn.Linear(dim, hidden_dim)
self.down_proj = nn.Linear(hidden_dim, dim)
def forward(self, x):
return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
class SanaVideo2Block(nn.Module):
def __init__(self, config, attention):
super().__init__()
dim = config.hidden_size
self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
attention_cls = (
GatedSoftmaxAttention if attention == "softmax" else GatedLinearAttention
)
head_dim = (
config.softmax_head_dim
if attention == "softmax"
else config.linear_head_dim
)
self.attn = attention_cls(dim, head_dim, config.qk_norm, config.fp32_attention)
self.cross_attn = CrossAttention(dim, config.num_heads, config.cross_norm)
self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6)
self.mlp = SwiGLU(dim, int(dim * config.mlp_ratio))
self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5)
def _modulation(self, t, batch):
if t.ndim == 2:
return (self.scale_shift_table[None] + t.reshape(batch, 6, -1)).chunk(
6, dim=1
)
groups = t.shape[2]
return (
self.scale_shift_table[None, None] + t.reshape(batch, groups, 6, -1)
).chunk(6, dim=2)
def forward_attn_sublayer(self, x, y, t, mask=None, rotary_emb=None):
batch, _, channels = x.shape
shift, scale, gate, _, _, _ = self._modulation(t, batch)
grouped_shape = (
(batch, -1, channels) if t.ndim == 2 else (batch, t.shape[2], -1, channels)
)
attn_input = self.norm1(x).reshape(grouped_shape) * (1 + scale) + shift
delta = self.attn(attn_input.reshape_as(x), rotary_emb)
delta = (gate * delta.reshape(grouped_shape)).reshape_as(x)
return delta + self.cross_attn(x + delta, y, mask)
def forward_mlp_sublayer(self, x, t):
batch, _, channels = x.shape
_, _, _, shift, scale, gate = self._modulation(t, batch)
grouped_shape = (
(batch, -1, channels) if t.ndim == 2 else (batch, t.shape[2], -1, channels)
)
mlp_input = self.norm2(x).reshape(grouped_shape) * (1 + scale) + shift
delta = self.mlp(mlp_input.reshape_as(x)).reshape(grouped_shape)
return (gate * delta).reshape_as(x)
class DepthRMSNorm(nn.Module):
def forward(self, x):
return x * (x.square().mean(-1, keepdim=True) + 1e-6).rsqrt()
class BlockAttentionResidual(nn.Module):
def __init__(self, hidden_size):
super().__init__()
self.attn_proj = nn.Linear(hidden_size, 1, bias=False)
self.mlp_proj = nn.Linear(hidden_size, 1, bias=False)
self.final_proj = nn.Linear(hidden_size, 1, bias=False)
self.key_norm = DepthRMSNorm()
for projection in (self.attn_proj, self.mlp_proj, self.final_proj):
nn.init.zeros_(projection.weight)
def attend_buffer(self, projection, values, keys, active_count, partial):
source_count = active_count
if partial is not None:
values[active_count] = partial
keys[active_count] = self.key_norm(partial.unsqueeze(0)).squeeze(0)
source_count += 1
if source_count == 1:
return values[0]
values, keys = values[:source_count], keys[:source_count]
logits = torch.einsum("d,nbtd->nbt", projection.weight.squeeze(0), keys)
return torch.einsum("nbt,nbtd->btd", logits.softmax(0), values)
class SanaVideo2Transformer3DModel(BaseDiT):
_fsdp_shard_conditions: ClassVar[list] = []
_compile_conditions: ClassVar[list] = []
_supported_attention_backends: ClassVar[set] = {AttentionBackendEnum.TORCH_SDPA}
param_names_mapping: ClassVar[dict] = {}
reverse_param_names_mapping: ClassVar[dict] = {}
lora_param_names_mapping: ClassVar[dict] = {}
def __init__(self, config: SanaVideo2Config, hf_config=None, **kwargs):
super().__init__(config, hf_config=hf_config or {}, **kwargs)
c = self.config
self.hidden_size = c.hidden_size
self.num_attention_heads = c.num_heads
self.num_channels_latents = c.in_channels
self.register_buffer(
"pos_embed", torch.zeros(1, c.input_size**2, c.hidden_size)
)
self.x_embedder = PatchEmbedMS3D(c.patch_size, c.in_channels, c.hidden_size)
self.t_embedder = TimestepEmbedder(c.hidden_size)
self.t_block = nn.Sequential(
nn.SiLU(), nn.Linear(c.hidden_size, 6 * c.hidden_size)
)
self.y_embedder = CaptionEmbedder(
c.caption_channels, c.hidden_size, c.model_max_length
)
if c.y_norm:
self.attention_y_norm = _RMSNorm(
c.hidden_size, c.y_norm_scale_factor, c.norm_eps
)
self.rope_linear = SanaVideo2RotaryPosEmbed(c.linear_head_dim, c.patch_size)
self.rope_softmax = SanaVideo2RotaryPosEmbed(c.softmax_head_dim, c.patch_size)
anchor_count = max(1, int(c.depth * c.softmax_ratio))
self.softmax_layer_indices = [
int((i + 1) * c.depth / anchor_count) - 1 for i in range(anchor_count)
]
self.block_attention_types = [
"softmax" if i in self.softmax_layer_indices else "linear"
for i in range(c.depth)
]
self.blocks = nn.ModuleList(
[SanaVideo2Block(c, attention) for attention in self.block_attention_types]
)
self.final_layer = T2IFinalLayer(c.hidden_size, c.patch_size, c.in_channels)
self.attn_res = BlockAttentionResidual(c.hidden_size)
@property
def dtype(self):
return self.x_embedder.proj.weight.dtype
def post_load_weights(self):
for rope in (self.rope_linear, self.rope_softmax):
if rope._freqs.is_meta:
rope._init_freqs_buffer()
def forward(
self,
hidden_states,
timestep,
encoder_hidden_states,
encoder_attention_mask=None,
**kwargs,
):
c = self.config
batch, _, frames, height, width = hidden_states.shape
if timestep.ndim == 5:
timestep = timestep.reshape(batch, 1, frames)
if c.timestep_norm_scale_factor == 1.0:
timestep = timestep.long().float()
else:
timestep = timestep.float() / c.timestep_norm_scale_factor
x = self.x_embedder(hidden_states.to(self.dtype))
ropes = {
"linear": self.rope_linear((frames, height, width), x.device),
"softmax": self.rope_softmax((frames, height, width), x.device),
}
t = self.t_embedder(timestep.flatten())
t0 = self.t_block(t).unflatten(0, timestep.shape)
t = t.unflatten(0, timestep.shape)
y = self.y_embedder(encoder_hidden_states.to(self.dtype))
if c.y_norm:
y = self.attention_y_norm(y)
mask = encoder_attention_mask
if mask is not None:
mask = mask.reshape(batch, -1)
# Keep the grouped depth buffer layout stable across denoising steps.
count = math.ceil(c.depth / c.attn_res_block_size)
values = x.new_empty(count + 2, batch, x.shape[1], c.hidden_size)
keys = torch.empty_like(values)
values[0] = x
keys[0] = self.attn_res.key_norm(x.unsqueeze(0)).squeeze(0)
active_count, partial = 1, None
for index, block in enumerate(self.blocks):
hidden = self.attn_res.attend_buffer(
self.attn_res.attn_proj, values, keys, active_count, partial
)
delta = block.forward_attn_sublayer(
hidden, y, t0, mask, ropes[self.block_attention_types[index]]
)
partial = delta if partial is None else partial + delta
hidden = self.attn_res.attend_buffer(
self.attn_res.mlp_proj, values, keys, active_count, partial
)
partial = partial + block.forward_mlp_sublayer(hidden, t0)
if (index + 1) % c.attn_res_block_size == 0 or index + 1 == c.depth:
values[active_count] = partial
keys[active_count] = self.attn_res.key_norm(
partial.unsqueeze(0)
).squeeze(0)
active_count += 1
partial = None
x = self.attn_res.attend_buffer(
self.attn_res.final_proj, values, keys, active_count, None
)
return (
self.final_layer(x, t)
.transpose(1, 2)
.reshape(batch, c.in_channels, frames, height, width)
)
EntryClass = SanaVideo2Transformer3DModel
@@ -0,0 +1,132 @@
# SPDX-License-Identifier: Apache-2.0
"""Native SANA-Video 2.0 loading and T2V/TI2V pipeline."""
from pathlib import Path
from typing import ClassVar
import torch
import yaml
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
ComponentLoader,
)
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
SanaVideo2Transformer3DModel,
)
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
ComposedPipelineBase,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages import InputValidationStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import (
SanaVideo2DenoisingStage,
SanaVideo2LatentPreparationStage,
SanaVideo2TextEncodingStage,
)
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import maybe_download_model
from sglang.multimodal_gen.runtime.utils.precision import resolve_component_precision
class SanaVideo2TransformerLoader(ComponentLoader):
expected_library = "diffusers"
def component_load_precision(self, server_args, component_name):
return server_args.component_precisions.get(component_name)
def load_customized(self, component_model_path, server_args, component_name):
root = Path(component_model_path)
with (root / "config.yaml").open() as file:
source = yaml.safe_load(file)
checkpoint = torch.load(
root / "checkpoints/SANA_Video_2.0_5B_720p.pth",
map_location="cpu",
weights_only=True,
mmap=True,
)["state_dict"]
config = server_args.pipeline_config.dit_config
model_fields = config.arch_config.__dataclass_fields__
values = {
name: value
for section in (source["model"], source["text_encoder"])
for name, value in section.items()
if name in model_fields
}
if "pos_embed" in checkpoint:
values["input_size"] = int(checkpoint["pos_embed"].shape[1] ** 0.5)
config.update_model_arch(values)
dtype = resolve_component_precision(server_args, component_name)
with torch.device("meta"):
model = SanaVideo2Transformer3DModel(config)
model.load_state_dict(checkpoint, strict=True, assign=True)
model = model.to(dtype=dtype)
model.post_load_weights()
server_args.model_paths[component_name] = component_model_path
return model.eval().requires_grad_(False)
class SanaVideo2Pipeline(ComposedPipelineBase):
pipeline_name = "SanaVideo2Pipeline"
is_video_pipeline = True
_required_config_modules: ClassVar[list[str]] = [
"text_encoder",
"tokenizer",
"vae",
"transformer",
]
component_loaders: ClassVar[dict[str, type[ComponentLoader]]] = {
"transformer": SanaVideo2TransformerLoader
}
def _load_config(self):
self.model_path = maybe_download_model(
self.model_path,
revision=self.server_args.revision,
allow_patterns=["config.yaml", "checkpoints/SANA_Video_2.0_5B_720p.pth"],
)
with (Path(self.model_path) / "config.yaml").open() as file:
source = yaml.safe_load(file)
self.server_args.pipeline_config.prompt_instruction = "\n".join(
source["text_encoder"]["chi_prompt"]
)
return {
"_class_name": self.pipeline_name,
"_diffusers_version": "0.37.0",
"text_encoder": ["transformers", "Gemma2Model"],
"tokenizer": ["transformers", "AutoTokenizer"],
"vae": ["diffusers", "AutoencoderKLLTX2Video"],
"transformer": ["diffusers", "SanaVideo2Transformer3DModel"],
}
def _resolve_component_path(self, server_args, module_name, load_module_name):
path = server_args.component_paths.get(module_name)
if path is None:
if module_name == "transformer":
return self.model_path
if module_name == "vae":
root = maybe_download_model(
"Efficient-Large-Model/LTX-2.3-Diffusers", allow_patterns=["vae/**"]
)
return str(Path(root) / "vae")
path = server_args.component_paths.get(
"text_encoder", "Efficient-Large-Model/gemma-2-2b-it"
)
path = maybe_download_model(path)
if module_name == "vae" and (Path(path) / "vae/config.json").is_file():
path = str(Path(path) / "vae")
return path
def create_pipeline_stages(self, server_args):
self.add_stage(InputValidationStage())
self.add_stage(
SanaVideo2TextEncodingStage(
[self.get_module("text_encoder")],
[self.get_module("tokenizer")],
server_args.pipeline_config.prompt_instruction,
),
"prompt_encoding_stage_primary",
)
self.add_stage(SanaVideo2LatentPreparationStage(self.get_module("vae")))
self.add_stage(SanaVideo2DenoisingStage(self.get_module("transformer")))
self.add_standard_decoding_stage()
EntryClass = SanaVideo2Pipeline
@@ -0,0 +1,280 @@
# Copyright 2024-2026 NVIDIA CORPORATION & AFFILIATES
# SPDX-License-Identifier: Apache-2.0
"""Prompt conditioning and request-local flow solvers for SANA-Video 2.0."""
import numpy as np
import torch
from diffusers import FlowMatchEulerDiscreteScheduler
from diffusers.utils.torch_utils import randn_tensor
from PIL import Image
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines.sana_video import (
select_sana_video_prompt_window,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
TextEncodingStage,
)
def sample_flow_dpm(predict_noise, latents, steps, shift, callback=None):
"""Second-order multistep DPM-Solver++ with first-order endpoint steps."""
if steps < 2:
raise ValueError("Flow DPM-Solver requires at least two inference steps")
betas = torch.linspace(1.0, 0.001, steps + 1, device="cpu").to(latents.device)
sigmas = 1.0 - betas
times = (shift * sigmas / (1 + (shift - 1) * sigmas)).flip(0)
previous_time = previous_prediction = None
x = latents
for index in range(steps):
s = times[index].reshape((1,) * x.ndim)
t = times[index + 1].reshape((1,) * x.ndim)
noise = predict_noise(x, times[index])
prediction = (x - s * noise) / (1 - s)
lambda_s = torch.log(1 - s) - torch.log(s)
lambda_t = torch.log(1 - t) - torch.log(t)
h = lambda_t - lambda_s
alpha_t = torch.exp(torch.log(1 - t))
phi = torch.expm1(-h)
if index == 0 or index == steps - 1:
x = t / s * x - alpha_t * phi * prediction
else:
lambda_previous = torch.log(1 - previous_time) - torch.log(previous_time)
ratio = (lambda_s - lambda_previous) / h
derivative = (1.0 / ratio) * (prediction - previous_prediction)
x = (
t / s * x
- alpha_t * phi * prediction
- 0.5 * (alpha_t * phi) * derivative
)
previous_time, previous_prediction = s, prediction
if callback is not None:
callback(index, times[index + 1], x)
return x
def sample_ltx_euler(predict_flow, latents, steps, shift, callback=None):
"""Flow Euler with a clean, fixed first latent frame."""
with torch.device("cpu"):
scheduler = FlowMatchEulerDiscreteScheduler(shift=shift)
scheduler.set_timesteps(steps, device=latents.device)
condition_mask = torch.zeros_like(latents, dtype=torch.float32)
condition_mask[:, :, 0] = 1
for index, time in enumerate(scheduler.timesteps):
timestep = torch.minimum(
time.expand(latents.shape).float(), (1 - condition_mask) * 1000.0
)
prediction = predict_flow(latents, timestep[:, :1, :, :1, :1])
batch, channels = latents.shape[:2]
updated = (
scheduler.step(
-prediction.reshape(batch, channels, -1).transpose(1, 2),
time,
latents.reshape(batch, channels, -1).transpose(1, 2),
per_token_timesteps=timestep.reshape(batch, channels, -1)[:, 0],
return_dict=False,
)[0]
.transpose(1, 2)
.reshape(latents.shape)
)
denoise_mask = time / 1000 - 1e-6 < (1.0 - condition_mask)
latents = torch.where(denoise_mask, updated, latents).to(latents.dtype)
if callback is not None:
callback(index, time, latents)
return latents
class SanaVideo2TextEncodingStage(TextEncodingStage):
def __init__(self, text_encoders, tokenizers, instruction):
super().__init__(text_encoders, tokenizers)
self.instruction = instruction
@torch.no_grad()
def forward(self, batch, server_args):
self.tokenizers[0].padding_side = "right"
length = batch.max_sequence_length or 300
prompts = [batch.prompt] if isinstance(batch.prompt, str) else batch.prompt
motion = batch.extra.get("motion_score", 10)
suffix = (
f" motion score: {int(motion)}."
if motion > 0
else ""
if motion < 0
else " high motion"
if batch.extra.get("high_motion", False)
else " low motion"
)
prompts = [self.instruction + prompt.strip() + suffix for prompt in prompts]
encoded_length = (
len(self.tokenizers[0].encode(self.instruction)) + length - 2
if self.instruction
else length
)
outputs = list(
self.encode_text(
prompts,
server_args,
return_attention_mask=True,
max_length=encoded_length,
)
)
for index in (0, 1, 3):
outputs[index] = [
select_sana_video_prompt_window(value, length)
for value in outputs[index]
]
outputs[4] = [
[int(value) for value in mask.sum(dim=1).tolist()] for mask in outputs[1]
]
self._append_positive_text_outputs(batch, *outputs)
if batch.do_classifier_free_guidance:
negative = self.encode_text(
batch.negative_prompt,
server_args,
return_attention_mask=True,
max_length=length,
)
self._append_negative_text_outputs(batch, outputs[0], *negative)
return batch
def prepare_image(image, height, width):
image = image.convert("RGB")
w, h = image.size
scale = max(height / h, width / w)
resized = image.resize(
(round(w * scale), round(h * scale)), Image.Resampling.BICUBIC
)
top = round((resized.height - height) / 2.0)
left = round((resized.width - width) / 2.0)
pixels = np.array(resized)[top : top + height, left : left + width].copy()
return (
torch.from_numpy(pixels).permute(2, 0, 1).float().div_(255).sub_(0.5).div_(0.5)
)
class SanaVideo2LatentPreparationStage(PipelineStage):
def __init__(self, vae):
super().__init__()
self.vae = vae
def component_uses(self, server_args, stage_name=None):
return [ComponentUse(self._component_stage_name(stage_name), "vae")]
@torch.no_grad()
def forward(self, batch, server_args):
config = server_args.pipeline_config
if batch.height % 32 or batch.width % 32:
raise ValueError("SANA-Video 2.0 height and width must be divisible by 32")
batch.num_frames = config.adjust_num_frames(batch.num_frames)
device = batch.prompt_embeds[0].device
batch_size = batch.prompt_embeds[0].shape[0]
shape = config.prepare_latent_shape(
batch, batch_size, (batch.num_frames - 1) // 8 + 1
)
if batch.latents is None:
batch.latents = randn_tensor(
shape, generator=batch.generator, device=device, dtype=torch.float32
)
else:
if tuple(batch.latents.shape) != shape:
raise ValueError(
f"Expected latents with shape {shape}, got {tuple(batch.latents.shape)}"
)
batch.latents = batch.latents.to(device=device, dtype=torch.float32).clone()
if batch.condition_image is not None:
images = (
batch.condition_image
if isinstance(batch.condition_image, list)
else [batch.condition_image]
)
if len(images) == 1:
images = images * batch_size
if len(images) != batch_size:
raise ValueError("TI2V requires one conditioning image per prompt")
self.begin_declared_component_use(component_name="vae", module=self.vae)
pixels = torch.stack(
[prepare_image(image, batch.height, batch.width) for image in images]
)[:, :, None]
pixels = pixels.to(device=device, dtype=next(self.vae.parameters()).dtype)
image_latents = self.vae.encode(pixels).latent_dist.mode()
scale, mean = config.get_decode_scale_and_shift(
device, image_latents.dtype, self.vae
)
batch.latents[:, :, :1] = (image_latents - mean) * scale
return batch
class SanaVideo2DenoisingStage(PipelineStage):
def __init__(self, transformer):
super().__init__()
self.transformer = transformer
@property
def role_affinity(self) -> RoleType:
return RoleType.DENOISER
def component_uses(self, server_args, stage_name=None):
return [ComponentUse(self._component_stage_name(stage_name), "transformer")]
def default_workload_iterations(self, batch, num_inference_steps):
return num_inference_steps
@torch.no_grad()
def forward(self, batch, server_args):
self.begin_declared_component_use(
component_name="transformer", module=self.transformer
)
cfg = batch.do_classifier_free_guidance
embeds = batch.prompt_embeds[0]
mask = batch.prompt_attention_mask[0]
if cfg:
embeds = torch.cat([batch.negative_prompt_embeds[0], embeds])
mask = torch.cat([batch.negative_attention_mask[0], mask])
is_ti2v = batch.condition_image is not None
def predict(x, time):
inputs = torch.cat([x, x]) if cfg else x
if time.ndim == 0:
timestep = (time * 1000).expand(inputs.shape[0])
else:
timestep = torch.cat([time, time]) if cfg else time
prediction = self.transformer(
hidden_states=inputs,
timestep=timestep,
encoder_hidden_states=embeds,
encoder_attention_mask=mask,
)
if not is_ti2v:
# Keep flow-to-noise conversion before CFG to preserve FP32 rounding.
sigma = time.reshape((1,) * x.ndim).to(inputs)
prediction = (1 - sigma) * prediction + inputs
if cfg:
uncond, cond = prediction.chunk(2)
prediction = uncond + batch.guidance_scale * (cond - uncond)
return prediction
steps = (
max(2, batch.num_inference_steps)
if batch.is_warmup
else batch.num_inference_steps
)
sampler = sample_ltx_euler if is_ti2v else sample_flow_dpm
flow_shift = (
batch.flow_shift
if batch.flow_shift is not None
else server_args.pipeline_config.flow_shift
)
with self.progress_bar(total=steps, batch=batch) as progress:
batch.latents = sampler(
predict,
batch.latents,
steps,
flow_shift,
callback=lambda *_: progress.update(),
)
return batch
@@ -0,0 +1,158 @@
# SPDX-License-Identifier: Apache-2.0
import unittest
import torch
class TestSanaVideo2(unittest.TestCase):
def test_linear_attention_matches_explicit_token_sum(self):
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
GatedLinearAttention,
)
torch.manual_seed(12)
module = GatedLinearAttention(24, 12, fp32_attention=True).eval()
x = torch.randn(2, 5, 24)
q, k, v = module.qkv(x).chunk(3, dim=-1)
q = module.q_norm(q).reshape(2, 5, 2, 12)
k = module.k_norm(k).reshape(2, 5, 2, 12)
v = v.reshape(2, 5, 2, 12)
beta = module.beta_proj(x).sigmoid()
scores = torch.einsum("bihd,bjhd->bhij", q, k)
out = torch.einsum("bhij,bjh,bjhd->bihd", scores, beta, v)
out = out.float() * (out.float().square().mean(-1, keepdim=True) + 1e-5).rsqrt()
out = out * module.o_norm.weight
out = out.reshape_as(x) * module.output_gate(x).sigmoid()
expected = module.proj(out)
torch.testing.assert_close(module(x), expected, atol=2e-6, rtol=2e-5)
def test_softmax_attention_matches_dense_equation(self):
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
GatedSoftmaxAttention,
)
torch.manual_seed(13)
module = GatedSoftmaxAttention(24, 12, fp32_attention=True).eval()
x = torch.randn(2, 5, 24)
q, k, v = module.qkv(x).chunk(3, -1)
q = module.q_norm(q).reshape(2, 5, 2, 12)
k = module.k_norm(k).reshape(2, 5, 2, 12)
v = v.reshape(2, 5, 2, 12)
scores = torch.einsum("bihd,bjhd->bhij", q, k) / 12**0.5
out = torch.einsum("bhij,bjhd->bihd", scores.softmax(-1), v)
expected = module.proj(out.reshape_as(x) * module.output_gate(x).sigmoid())
torch.testing.assert_close(module(x), expected)
def test_attention_residual_weights_depth_per_token(self):
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
BlockAttentionResidual,
)
torch.manual_seed(14)
module = BlockAttentionResidual(4)
with torch.no_grad():
module.attn_proj.weight.copy_(torch.tensor([[0.3, -0.4, 0.5, 0.1]]))
values = torch.randn(4, 2, 3, 4)
keys = values * (values.square().mean(-1, keepdim=True) + 1e-6).rsqrt()
scores = (keys * module.attn_proj.weight).sum(-1).softmax(0)
expected = (scores.unsqueeze(-1) * values).sum(0)
actual = module.attend_buffer(
module.attn_proj, values.clone(), keys.clone(), 3, values[3]
)
torch.testing.assert_close(actual, expected)
@staticmethod
def small_model():
from sglang.multimodal_gen.configs.models.dits.sana_video2 import (
SanaVideo2ArchConfig,
SanaVideo2Config,
)
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
SanaVideo2Transformer3DModel,
)
return SanaVideo2Transformer3DModel(
SanaVideo2Config(
arch_config=SanaVideo2ArchConfig(
hidden_size=24,
depth=4,
num_heads=2,
linear_head_dim=12,
softmax_head_dim=24,
in_channels=4,
caption_channels=8,
model_max_length=5,
mlp_ratio=2,
attn_res_block_size=2,
)
)
).eval()
def test_uniform_frame_timesteps_equal_batch_timesteps(self):
torch.manual_seed(15)
model = self.small_model()
x, y = torch.randn(2, 4, 3, 2, 2), torch.randn(2, 5, 8)
t = torch.tensor([42.9, 123.9])
mask = torch.tensor([[1, 1, 1, 0, 0], [1, 1, 1, 1, 0]])
with torch.no_grad():
batch = model(x, t, y, mask)
frames = model(
x, t[:, None, None, None, None].expand(-1, 1, 3, 1, 1), y[:, None], mask
)
changed_t = t[:, None, None, None, None].expand(-1, 1, 3, 1, 1).clone()
changed_t[:, :, 0] = 0
changed = model(x, changed_t, y, mask)
self.assertEqual(batch.shape, x.shape)
torch.testing.assert_close(batch, frames)
self.assertFalse(torch.allclose(batch, changed))
self.assertEqual(
model.block_attention_types, ["linear", "linear", "linear", "softmax"]
)
def test_padding_tokens_do_not_change_flow_and_calls_do_not_share_shape(self):
torch.manual_seed(16)
model = self.small_model()
x, y, t = torch.randn(1, 4, 2, 2, 2), torch.randn(1, 5, 8), torch.tensor([20.0])
mask = torch.tensor([[1, 1, 1, 0, 0]])
with torch.no_grad():
original = model(x, t, y, mask)
y[:, 3:] = 1000
model(torch.randn(1, 4, 1, 1, 3), t, y, mask)
actual = model(x, t, y, mask)
torch.testing.assert_close(actual, original)
def test_dtype_cast_preserves_complex_rotary_frequencies(self):
model = self.small_model()
before = model.rope_linear((2, 2, 2), torch.device("cpu"))
model.to(dtype=torch.bfloat16)
after = model.rope_linear((2, 2, 2), torch.device("cpu"))
self.assertTrue(after.is_complex())
torch.testing.assert_close(after, before, rtol=0, atol=0)
def test_official_checkpoint_names_and_output_sign(self):
torch.manual_seed(17)
model = self.small_model()
state = model.state_dict()
for key in (
"pos_embed",
"y_embedder.y_embedding",
"blocks.0.attn.beta_proj.weight",
"blocks.3.attn.output_gate.weight",
"blocks.0.cross_attn.kv_linear.weight",
"blocks.0.mlp.gate_proj.weight",
"attn_res.final_proj.weight",
):
self.assertIn(key, state)
model.load_state_dict(state, strict=True)
with torch.no_grad():
model.final_layer.linear.weight.zero_()
model.final_layer.linear.bias.fill_(2)
output = model(
torch.randn(1, 4, 2, 2, 2), torch.tensor([1.0]), torch.randn(1, 5, 8)
)
torch.testing.assert_close(output, torch.full_like(output, 2))
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,323 @@
from contextlib import nullcontext
from itertools import pairwise
from types import SimpleNamespace
import pytest
import torch
from sglang.multimodal_gen.configs.pipeline_configs.sana_video2 import (
SanaVideo2PipelineConfig,
)
from sglang.multimodal_gen.configs.sample.sana_video2 import SanaVideo2SamplingParams
from sglang.multimodal_gen.registry import get_model_info
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages import (
sana_video2,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import (
sample_flow_dpm,
sample_ltx_euler,
)
def test_native_registry_precedes_sana_video(monkeypatch):
monkeypatch.setattr(
"sglang.multimodal_gen.registry.maybe_download_model_index",
lambda _: None,
)
get_model_info.cache_clear()
info = get_model_info("Efficient-Large-Model/SANA-Video_2.0_5B_720p")
assert info.pipeline_config_cls is SanaVideo2PipelineConfig
assert info.sampling_param_cls is SanaVideo2SamplingParams
get_model_info.cache_clear()
def test_latent_layout_and_frame_alignment():
config = SanaVideo2PipelineConfig()
assert config.adjust_num_frames(193) == 193
assert config.adjust_num_frames(192) == 185
assert config.prepare_latent_shape(
SimpleNamespace(height=736, width=1280), 2, 25
) == (2, 128, 25, 23, 40)
assert config.get_latent_dtype(torch.bfloat16) == torch.float32
defaults = SanaVideo2SamplingParams()
assert (defaults.height, defaults.width, defaults.num_frames, defaults.fps) == (
736,
1280,
193,
24,
)
assert defaults.num_inference_steps == 50
def test_dpm_constant_data_prediction_reaches_clean_endpoint():
initial = torch.tensor([[[[[0.4, -0.7]]]]], dtype=torch.float32)
clean = torch.tensor([[[[[0.2, 0.9]]]]], dtype=torch.float32)
times = []
def predict_noise(x, time):
times.append(time.item())
return (x - (1 - time) * clean) / time
actual = sample_flow_dpm(predict_noise, initial, steps=5, shift=12.0)
torch.testing.assert_close(actual, clean, atol=1e-5, rtol=1e-5)
assert len(times) == 5
assert all(a > b for a, b in pairwise(times))
assert actual.dtype == torch.float32
def test_ti2v_euler_preserves_condition_and_uses_frame_timestep():
initial = torch.zeros(1, 2, 3, 1, 1)
initial[:, :, 0] = 7
seen = []
def predict_flow(x, time):
seen.append(time.clone())
return torch.ones_like(x)
actual = sample_ltx_euler(predict_flow, initial.clone(), steps=4, shift=12.0)
torch.testing.assert_close(actual[:, :, 0], initial[:, :, 0], atol=0, rtol=0)
torch.testing.assert_close(actual[:, :, 1:], -torch.ones_like(actual[:, :, 1:]))
assert len(seen) == 4
assert all(time.shape == (1, 1, 3, 1, 1) for time in seen)
assert all(torch.count_nonzero(time[:, :, 0]) == 0 for time in seen)
assert all(torch.all(time[:, :, 1:] > 0) for time in seen)
@pytest.mark.parametrize("conditioned", [False, True])
@pytest.mark.parametrize("flow_shift, expected", [(None, 12.0), (3.0, 3.0)])
def test_denoising_uses_request_flow_shift(
monkeypatch, conditioned, flow_shift, expected
):
shifts = []
def sample(predict, latents, steps, shift, callback):
shifts.append(shift)
return latents
monkeypatch.setattr(
sana_video2, "sample_ltx_euler" if conditioned else "sample_flow_dpm", sample
)
stage = object.__new__(sana_video2.SanaVideo2DenoisingStage)
stage.transformer = None
stage.begin_declared_component_use = lambda **_: None
stage.progress_bar = lambda **_: nullcontext()
batch = SimpleNamespace(
do_classifier_free_guidance=False,
prompt_embeds=[torch.zeros(1, 300, 4)],
prompt_attention_mask=[torch.ones(1, 300)],
condition_image=object() if conditioned else None,
latents=torch.zeros(1, 2, 3, 1, 1),
num_inference_steps=2,
is_warmup=False,
flow_shift=flow_shift,
)
stage.forward(batch, SimpleNamespace(pipeline_config=SanaVideo2PipelineConfig()))
assert shifts == [expected]
def test_denoising_stage_role():
stage = object.__new__(sana_video2.SanaVideo2DenoisingStage)
assert stage.role_affinity is RoleType.DENOISER
def test_vae_statistics_are_inverted_for_decode():
config = SanaVideo2PipelineConfig()
config.vae_config.arch_config.scaling_factor = 2.0
vae = SimpleNamespace(
latents_mean=torch.tensor([1.0, -1.0]), latents_std=torch.tensor([2.0, 4.0])
)
scale, shift = config.get_decode_scale_and_shift(
torch.device("cpu"), torch.float32, vae
)
normalized = torch.tensor([2.0, 3.0]).reshape(1, 2, 1, 1, 1)
decoded = normalized / scale + shift
torch.testing.assert_close(decoded.flatten(), torch.tensor([3.0, 5.0]))
def test_prompt_window_preserves_case_and_only_positive_motion_suffix():
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import (
SanaVideo2TextEncodingStage,
)
stage = object.__new__(SanaVideo2TextEncodingStage)
stage.instruction = "instruction: "
stage.tokenizers = [SimpleNamespace(encode=lambda _: [1, 2, 3, 4, 5])]
calls = []
def encode(text, server_args, **kwargs):
calls.append((text, kwargs["max_length"]))
length = kwargs["max_length"]
values = torch.arange(length).reshape(1, length, 1)
mask = torch.ones(1, length, dtype=torch.long)
return [values], [mask], [], [mask.bool()], [[length]]
stage.encode_text = encode
batch = SimpleNamespace(
prompt=" A RED fox ",
negative_prompt="Bad motion",
max_sequence_length=4,
extra={"motion_score": 10},
do_classifier_free_guidance=True,
prompt_embeds=[],
pooled_embeds=[],
prompt_attention_mask=None,
negative_prompt_embeds=[],
negative_pooled_embeds=[],
negative_attention_mask=None,
)
stage.forward(batch, SimpleNamespace())
assert calls == [
(["instruction: A RED fox motion score: 10."], 7),
("Bad motion", 4),
]
assert batch.prompt_embeds[0].flatten().tolist() == [0, 4, 5, 6]
assert batch.negative_prompt_embeds[0].flatten().tolist() == [0, 1, 2, 3]
assert batch.prompt_attention_mask[0].shape == (1, 4)
def test_native_model_index_and_component_directory_contract(tmp_path):
import yaml
from sglang.multimodal_gen.runtime.pipelines.sana_video2 import SanaVideo2Pipeline
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"text_encoder": {"chi_prompt": ["first", "last"]}})
)
vae_root = tmp_path / "ltx"
(vae_root / "vae").mkdir(parents=True)
(vae_root / "vae/config.json").write_text("{}")
pipeline = object.__new__(SanaVideo2Pipeline)
pipeline.model_path = str(tmp_path)
pipeline.server_args = SimpleNamespace(
revision=None,
pipeline_config=SanaVideo2PipelineConfig(),
component_paths={"vae": str(vae_root), "text_encoder": str(tmp_path)},
)
index = pipeline._load_config()
assert index["_class_name"] == "SanaVideo2Pipeline"
assert "_diffusers_version" in index
assert pipeline.server_args.pipeline_config.prompt_instruction == "first\nlast"
assert pipeline._resolve_component_path(pipeline.server_args, "vae", "vae") == str(
vae_root / "vae"
)
assert pipeline._resolve_component_path(
pipeline.server_args, "tokenizer", "tokenizer"
) == str(tmp_path)
def test_released_pth_layout_loads_strictly_and_materializes_rope(tmp_path):
import yaml
from sglang.multimodal_gen.configs.models.dits.sana_video2 import SanaVideo2Config
from sglang.multimodal_gen.runtime.models.dits.sana_video2 import (
SanaVideo2Transformer3DModel,
)
from sglang.multimodal_gen.runtime.pipelines.sana_video2 import (
SanaVideo2TransformerLoader,
)
arch = {
"hidden_size": 32,
"depth": 2,
"num_heads": 4,
"linear_head_dim": 8,
"softmax_head_dim": 8,
"softmax_ratio": 0.5,
"in_channels": 4,
"caption_channels": 12,
"model_max_length": 5,
"input_size": 2,
}
config = SanaVideo2Config()
config.update_model_arch(arch)
original = SanaVideo2Transformer3DModel(config)
(tmp_path / "checkpoints").mkdir()
torch.save(
{"state_dict": original.state_dict()},
tmp_path / "checkpoints/SANA_Video_2.0_5B_720p.pth",
)
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"model": arch, "text_encoder": {}})
)
pipeline_config = SanaVideo2PipelineConfig()
pipeline_config.dit_precision = "fp32"
args = SimpleNamespace(
pipeline_config=pipeline_config, component_precisions={}, model_paths={}
)
loaded = SanaVideo2TransformerLoader().load_customized(
str(tmp_path), args, "transformer"
)
assert not any(value.is_meta for value in loaded.buffers())
for name, value in original.state_dict().items():
torch.testing.assert_close(loaded.state_dict()[name], value, rtol=0, atol=0)
def test_local_native_root_does_not_require_diffusers_model_index(tmp_path):
root = tmp_path / "sana-video2-5b"
root.mkdir()
(root / "config.yaml").write_text("model:\n model: SanaVideo2_5B\n")
get_model_info.cache_clear()
info = get_model_info(str(root))
assert info.pipeline_config_cls is SanaVideo2PipelineConfig
assert info.sampling_param_cls is SanaVideo2SamplingParams
get_model_info.cache_clear()
def test_diffusers_local_root_still_reads_model_index(tmp_path):
from sglang.multimodal_gen.configs.pipeline_configs.sana_video import (
SanaVideoPipelineConfig,
)
root = tmp_path / "sana-video-original"
root.mkdir()
(root / "transformer").mkdir()
(root / "model_index.json").write_text(
'{"_class_name": "SanaVideoPipeline", "_diffusers_version": "0.37.0", '
'"transformer": ["diffusers", "SanaVideoTransformer3DModel"]}'
)
get_model_info.cache_clear()
info = get_model_info(str(root))
assert info.pipeline_config_cls is SanaVideoPipelineConfig
get_model_info.cache_clear()
def test_two_outputs_expand_with_independent_generators_before_latent_preparation():
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import (
InputValidationStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import (
SanaVideo2LatentPreparationStage,
)
sampling = SanaVideo2SamplingParams(
request_id="two-videos",
prompt="A fox",
height=32,
width=32,
num_frames=9,
num_outputs_per_prompt=2,
seed=[11, 12],
generator_device="cpu",
output_path="/tmp",
output_file_name="video.mp4",
)
request = Req(sampling_params=sampling)
args = SimpleNamespace(pipeline_config=SanaVideo2PipelineConfig())
requests = list(InputValidationStage().iter_sequential_requests(request, args))
assert len(requests) == 2
assert [item.num_outputs_per_prompt for item in requests] == [1, 1]
assert [item.seed for item in requests] == [11, 12]
stage = SanaVideo2LatentPreparationStage(vae=None)
latents = []
for item in requests:
item.prompt_embeds = [torch.zeros(1, 5, 12)]
stage.forward(item, args)
assert item.latents.shape == (1, 128, 2, 1, 1)
expected = torch.randn(
item.latents.shape, generator=torch.Generator().manual_seed(item.seed)
)
torch.testing.assert_close(item.latents, expected)
latents.append(item.latents)
assert not torch.equal(latents[0], latents[1])