From 7cfecbef567c9d5b4e0656bc8b34c27454f344b5 Mon Sep 17 00:00:00 2001 From: vOv <124694036+cr-gao@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:28:03 -0500 Subject: [PATCH] [diffusion][model] Add native SANA-Video 2.0 support (#41492) Signed-off-by: cr-gao Co-authored-by: Xiaoyu Zhang <1182563586@qq.com> --- .../diffusion/SANA-Video/SANA-Video.mdx | 32 +- docs/src/snippets/diffusion/model-catalog.jsx | 5 + .../configs/models/dits/sana_video2.py | 52 +++ .../configs/pipeline_configs/sana_video2.py | 68 ++++ .../configs/sample/sana_video2.py | 28 ++ python/sglang/multimodal_gen/registry.py | 12 +- .../runtime/models/dits/sana_video2.py | 353 ++++++++++++++++++ .../runtime/pipelines/sana_video2.py | 132 +++++++ .../model_specific_stages/sana_video2.py | 280 ++++++++++++++ .../test/unit/test_sana_video2.py | 158 ++++++++ .../test/unit/test_sana_video2_pipeline.py | 323 ++++++++++++++++ .../e2e/diffusion/test_sana_video2.py | 68 ++++ 12 files changed, 1507 insertions(+), 4 deletions(-) create mode 100644 python/sglang/multimodal_gen/configs/models/dits/sana_video2.py create mode 100644 python/sglang/multimodal_gen/configs/pipeline_configs/sana_video2.py create mode 100644 python/sglang/multimodal_gen/configs/sample/sana_video2.py create mode 100644 python/sglang/multimodal_gen/runtime/models/dits/sana_video2.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines/sana_video2.py create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_video2.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_sana_video2.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_sana_video2_pipeline.py create mode 100644 test/registered/e2e/diffusion/test_sana_video2.py diff --git a/docs/cookbook/diffusion/SANA-Video/SANA-Video.mdx b/docs/cookbook/diffusion/SANA-Video/SANA-Video.mdx index f560569583..66006723e7 100644 --- a/docs/cookbook/diffusion/SANA-Video/SANA-Video.mdx +++ b/docs/cookbook/diffusion/SANA-Video/SANA-Video.mdx @@ -1,6 +1,6 @@ --- title: SANA-Video -description: Serve the native SANA-Video 2B 480p text-to-video model with SGLang Diffusion. +description: Generate videos with native SANA-Video 2B and SANA-Video 2.0 5B pipelines. metatags: description: "Run Efficient-Large-Model/SANA-Video_2B_480p_diffusers text-to-video generation with SGLang Diffusion." --- @@ -95,3 +95,33 @@ Path("sana_video.mp4").write_bytes(video.content) - Use width and height values divisible by 16. - The prompt supports an optional `motion score: N.` suffix to express the desired amount of motion. + +## 6. SANA-Video 2.0 + +The separate `Efficient-Large-Model/SANA-Video_2.0_5B_720p` checkpoint supports +text-to-video and first-frame image conditioning. Its default profile is +1280×736, 193 frames at 24 FPS, 50 steps, and guidance scale 8. This integration +loads the official checkpoint directly and uses Gemma 2 and the LTX 2.3 VAE. + +On a single NVIDIA CUDA GPU, generate a short video with: + +```bash Command +sglang generate \ + --model-path Efficient-Large-Model/SANA-Video_2.0_5B_720p \ + --prompt "A red tram moves slowly through a sunlit city square." \ + --width 832 --height 480 --num-frames 17 --fps 24 \ + --num-inference-steps 50 --guidance-scale 8 --seed 42 \ + --output-file-path sana_video2.mp4 +``` + +Add `--image-path first_frame.png` to condition the video on an image. Frame +counts align to `8n+1`; width and height must be divisible by 32. The 4-step +preview is a different checkpoint and is not supported by this pipeline. + +For offline weights, keep the model directory name, such as +`/path/to/SANA-Video_2.0_5B_720p`, and pass it as `--model-path`. That directory +must contain `config.yaml` and `checkpoints/SANA_Video_2.0_5B_720p.pth`. Set +`--component-paths.text_encoder /path/to/gemma-2-2b-it` and +`--component-paths.vae /path/to/ltx-2.3-vae`. The latter directory may contain +the `vae` subdirectory or the VAE files directly. Without these overrides, +the component checkpoints are downloaded automatically. diff --git a/docs/src/snippets/diffusion/model-catalog.jsx b/docs/src/snippets/diffusion/model-catalog.jsx index 74184acc60..2bea4493f3 100644 --- a/docs/src/snippets/diffusion/model-catalog.jsx +++ b/docs/src/snippets/diffusion/model-catalog.jsx @@ -244,6 +244,11 @@ export const DiffusionModelCatalog = ({ category }) => { modelIds: ["Efficient-Large-Model/SANA-Video_2B_480p_diffusers"], cookbook: "/cookbook/diffusion/SANA-Video/SANA-Video", }, + { + name: "SANA-Video 2.0", + modelIds: ["Efficient-Large-Model/SANA-Video_2.0_5B_720p"], + cookbook: "/cookbook/diffusion/SANA-Video/SANA-Video#6-sana-video-20", + }, { name: "LingBot Video MoE", modelIds: ["robbyant/lingbot-video-moe-30b-a3b"], diff --git a/python/sglang/multimodal_gen/configs/models/dits/sana_video2.py b/python/sglang/multimodal_gen/configs/models/dits/sana_video2.py new file mode 100644 index 0000000000..d424a45bc2 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/models/dits/sana_video2.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig + + +@dataclass +class SanaVideo2ArchConfig(DiTArchConfig): + input_size: int = 15 + patch_size: tuple[int, int, int] = (1, 1, 1) + in_channels: int = 128 + hidden_size: int = 2560 + depth: int = 32 + num_heads: int = 20 + linear_head_dim: int = 128 + softmax_head_dim: int = 256 + softmax_ratio: float = 0.25 + mlp_ratio: float = 4.0 + caption_channels: int = 2304 + model_max_length: int = 300 + qk_norm: bool = True + cross_norm: bool = True + y_norm: bool = True + y_norm_scale_factor: float = 0.01 + norm_eps: float = 1e-5 + attn_res_block_size: int = 8 + timestep_norm_scale_factor: float = 1.0 + fp32_attention: bool = True + + def __post_init__(self): + super().__post_init__() + self.patch_size = tuple(self.patch_size) + if self.patch_size != (1, 1, 1): + raise ValueError("SANA-Video 2.0 requires patch_size=(1, 1, 1)") + if self.depth < 1 or self.attn_res_block_size < 1: + raise ValueError("depth and attn_res_block_size must be positive") + if not 0 < self.softmax_ratio <= 1: + raise ValueError("softmax_ratio must be in (0, 1]") + if any( + self.hidden_size % dim + for dim in (self.linear_head_dim, self.softmax_head_dim, self.num_heads) + ): + raise ValueError("hidden_size must divide evenly into attention heads") + self.num_attention_heads = self.num_heads + self.num_channels_latents = self.in_channels + + +@dataclass +class SanaVideo2Config(DiTConfig): + arch_config: DiTArchConfig = field(default_factory=SanaVideo2ArchConfig) + prefix: str = "SanaVideo2" diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/sana_video2.py b/python/sglang/multimodal_gen/configs/pipeline_configs/sana_video2.py new file mode 100644 index 0000000000..fc174a8eb6 --- /dev/null +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/sana_video2.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Native SANA-Video 2.0 pipeline configuration.""" + +from dataclasses import dataclass, field + +from sglang.multimodal_gen.configs.models.dits.sana_video2 import SanaVideo2Config +from sglang.multimodal_gen.configs.models.vaes.ltx_video import LTXVideoVAEConfig +from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType +from sglang.multimodal_gen.configs.pipeline_configs.sana_video import ( + SanaVideoPipelineConfig, +) + + +@dataclass +class SanaVideo2VAEConfig(LTXVideoVAEConfig): + def update_model_arch(self, source_model_dict): + config = dict(source_model_dict) + if "upsample_type" not in config and "decoder_upsample_type" in config: + config["upsample_type"] = list(reversed(config["decoder_upsample_type"])) + super().update_model_arch(config) + + +@dataclass +class SanaVideo2PipelineConfig(SanaVideoPipelineConfig): + task_type: ModelTaskType = ModelTaskType.TI2V + flow_shift: float | None = 12.0 + skip_input_image_preprocess: bool = True + dit_config: SanaVideo2Config = field(default_factory=SanaVideo2Config) + vae_config: SanaVideo2VAEConfig = field(default_factory=SanaVideo2VAEConfig) + vae_precision: str = "bf16" + vae_decode_precision: str = "bf16" + prompt_instruction: str = "" + + def __post_init__(self): + self.vae_config.load_encoder = True + self.vae_config.load_decoder = True + + def supports_sequential_multi_output_inference(self): + return True + + def get_decode_scale_and_shift(self, device, dtype, vae): + mean = vae.latents_mean.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1) + std = vae.latents_std.to(device=device, dtype=dtype).view(1, -1, 1, 1, 1) + return self.vae_config.arch_config.scaling_factor / std, mean + + +def register(): + from sglang.multimodal_gen.configs.sample.sana_video2 import ( + SanaVideo2SamplingParams, + ) + from sglang.multimodal_gen.registry import register_configs + + register_configs( + sampling_param_cls=SanaVideo2SamplingParams, + pipeline_config_cls=SanaVideo2PipelineConfig, + hf_model_paths=["Efficient-Large-Model/SANA-Video_2.0_5B_720p"], + model_detectors=[ + lambda name: any( + key in name.lower() + for key in ( + "sana-video_2.0", + "sana-video2", + "sana_video2", + "sanavideo2pipeline", + ) + ) + ], + ) diff --git a/python/sglang/multimodal_gen/configs/sample/sana_video2.py b/python/sglang/multimodal_gen/configs/sample/sana_video2.py new file mode 100644 index 0000000000..6b2f5306ed --- /dev/null +++ b/python/sglang/multimodal_gen/configs/sample/sana_video2.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Sampling defaults for the SANA-Video 2.0 5B checkpoint.""" + +from dataclasses import dataclass + +from sglang.multimodal_gen.configs.sample.sana_video import SanaVideoSamplingParams + + +@dataclass +class SanaVideo2SamplingParams(SanaVideoSamplingParams): + height: int = 736 + width: int = 1280 + num_frames: int = 193 + fps: int = 24 + guidance_scale: float = 8.0 + motion_score: int = 10 + high_motion: bool = False + + def build_request_extra(self): + return { + **super().build_request_extra(), + "motion_score": self.motion_score, + "high_motion": self.high_motion, + } + + @classmethod + def video_request_extra_fields(cls): + return super().video_request_extra_fields() | {"motion_score", "high_motion"} diff --git a/python/sglang/multimodal_gen/registry.py b/python/sglang/multimodal_gen/registry.py index 0f2a025a48..6ef80039ad 100644 --- a/python/sglang/multimodal_gen/registry.py +++ b/python/sglang/multimodal_gen/registry.py @@ -173,6 +173,9 @@ _MODEL_NAME_DETECTORS: List[Tuple[str, Callable[[str], bool]]] = [] # aliases next to the resolver that consumes them so CLI detection and # pipeline selection cannot drift apart KNOWN_NON_DIFFUSERS_DIFFUSION_MODEL_PATTERNS: Dict[str, str] = { + "sana-video_2.0": "SanaVideo2Pipeline", + "sana-video2": "SanaVideo2Pipeline", + "sana_video2": "SanaVideo2Pipeline", "ming-image-0.1-design": "MingImagePipeline", "minimaxai/minimax-h3": "MiniMaxH3Pipeline", "minimax/minimax-h3": "MiniMaxH3Pipeline", @@ -291,7 +294,7 @@ _configs_discovered: bool = False # SANA-WM (register BEFORE generic SANA T2I to prevent "sana" detector false-match) # SANA-Video (register before generic SANA to avoid detector overlap). -_CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video") +_CONFIG_REGISTER_PRIORITY: Tuple[str, ...] = ("sana_wm", "sana_video2", "sana_video") def _discover_and_register_configs() -> None: @@ -462,8 +465,11 @@ def _get_config_info( return _CONFIG_REGISTRY.get(model_id) # 3. Use detectors - config = maybe_download_model_index(model_path) - pipeline_name = config.get("_class_name", "").lower() + pipeline_name = get_non_diffusers_pipeline_name(model_path) + if pipeline_name is None: + config = maybe_download_model_index(model_path) + pipeline_name = config.get("_class_name", "") + pipeline_name = pipeline_name.lower() matched_model_names = [] for model_id, detector in _MODEL_NAME_DETECTORS: diff --git a/python/sglang/multimodal_gen/runtime/models/dits/sana_video2.py b/python/sglang/multimodal_gen/runtime/models/dits/sana_video2.py new file mode 100644 index 0000000000..3b9af97682 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/dits/sana_video2.py @@ -0,0 +1,353 @@ +# Copyright 2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 + +"""SANA-Video 2.0 inference transformer with grouped Attention Residuals.""" + +import math +from typing import ClassVar + +import torch +from torch import nn +from torch.nn import functional as F + +from sglang.multimodal_gen.configs.models.dits.sana_video2 import SanaVideo2Config +from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT +from sglang.multimodal_gen.runtime.models.dits.sana_wm_components import ( + CaptionEmbedder, + PatchEmbedMS3D, + T2IFinalLayer, + TimestepEmbedder, + WanRotaryPosEmbed, + _RMSNorm, +) +from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum + + +def _apply_rope(x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor: + rotated = torch.view_as_complex(x.to(torch.float64).unflatten(-1, (-1, 2))) + return torch.view_as_real(rotated * freqs).flatten(-2).type_as(x) + + +class SanaVideo2RotaryPosEmbed(WanRotaryPosEmbed): + def _apply(self, fn, recurse=True): + # Keep complex128 frequencies; forward moves them to the input device. + return self + + +class ChannelRMSNorm(_RMSNorm): + def forward(self, x: torch.Tensor) -> torch.Tensor: + normalized = ( + x.float() * (x.float().square().mean(-2, keepdim=True) + self.eps).rsqrt() + ) + return (self.weight[None, None, :, None] * normalized).type_as(x) + + +class GatedLinearAttention(nn.Module): + def __init__(self, dim, head_dim, qk_norm=True, fp32_attention=True): + super().__init__() + self.heads = dim // head_dim + self.dim = head_dim + self.fp32_attention = fp32_attention + self.qkv = nn.Linear(dim, 3 * dim, bias=False) + self.proj = nn.Linear(dim, dim) + self.q_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity() + self.k_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity() + self.beta_proj = nn.Linear(dim, self.heads) + self.output_gate = nn.Linear(dim, dim) + self.o_norm = ChannelRMSNorm(head_dim, eps=1e-5) + + def forward(self, x, rotary_emb=None): + batch, tokens, channels = x.shape + q, k, v = self.qkv(x).reshape(batch, tokens, 3, channels).unbind(2) + dtype = q.dtype + q = ( + self.q_norm(q) + .transpose(-1, -2) + .reshape(batch, self.heads, self.dim, tokens) + ) + k = ( + self.k_norm(k) + .transpose(-1, -2) + .reshape(batch, self.heads, self.dim, tokens) + ) + v = v.transpose(-1, -2).reshape(batch, self.heads, self.dim, tokens) + if rotary_emb is not None: + q = _apply_rope(q.transpose(-1, -2), rotary_emb).transpose(-1, -2) + k = _apply_rope(k.transpose(-1, -2), rotary_emb).transpose(-1, -2) + beta = self.beta_proj(x).sigmoid().transpose(1, 2).unsqueeze(2) + k = k * beta + if self.fp32_attention: + q, k, v = q.float(), k.float(), v.float() + out = (v @ k.transpose(-1, -2)) @ q + out = self.o_norm(out.to(dtype)) + out = out.reshape(batch, channels, tokens).transpose(1, 2) + return self.proj(out * self.output_gate(x).sigmoid()) + + +class GatedSoftmaxAttention(nn.Module): + def __init__(self, dim, head_dim, qk_norm=True, fp32_attention=True): + super().__init__() + self.heads = dim // head_dim + self.dim = head_dim + self.fp32_attention = fp32_attention + self.qkv = nn.Linear(dim, 3 * dim, bias=False) + self.proj = nn.Linear(dim, dim) + self.q_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity() + self.k_norm = _RMSNorm(dim, eps=1e-5) if qk_norm else nn.Identity() + self.output_gate = nn.Linear(dim, dim) + + def forward(self, x, rotary_emb=None): + batch, tokens, channels = x.shape + q, k, v = self.qkv(x).reshape(batch, tokens, 3, channels).unbind(2) + dtype = q.dtype + q = self.q_norm(q).reshape(batch, tokens, self.heads, self.dim).transpose(1, 2) + k = self.k_norm(k).reshape(batch, tokens, self.heads, self.dim).transpose(1, 2) + v = v.reshape(batch, tokens, self.heads, self.dim).transpose(1, 2) + if rotary_emb is not None: + q, k = _apply_rope(q, rotary_emb), _apply_rope(k, rotary_emb) + if self.fp32_attention: + q, k, v = q.float(), k.float(), v.float() + out = F.scaled_dot_product_attention(q, k, v, dropout_p=0.0) + out = out.transpose(1, 2).reshape(batch, tokens, channels).to(dtype) + return self.proj(out * self.output_gate(x).sigmoid()) + + +class CrossAttention(nn.Module): + def __init__(self, dim, num_heads, qk_norm=True): + super().__init__() + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.q_linear = nn.Linear(dim, dim) + self.kv_linear = nn.Linear(dim, 2 * dim) + self.proj = nn.Linear(dim, dim) + self.q_norm = _RMSNorm(dim, eps=1e-6) if qk_norm else nn.Identity() + self.k_norm = _RMSNorm(dim, eps=1e-6) if qk_norm else nn.Identity() + + def forward(self, x, y, mask=None): + batch, tokens, channels = x.shape + q = self.q_norm(self.q_linear(x)) + k, v = self.kv_linear(y).reshape(batch, -1, 2, channels).unbind(2) + q = q.reshape(batch, tokens, self.num_heads, self.head_dim).transpose(1, 2) + k = ( + self.k_norm(k) + .reshape(batch, -1, self.num_heads, self.head_dim) + .transpose(1, 2) + ) + v = v.reshape(batch, -1, self.num_heads, self.head_dim).transpose(1, 2) + if mask is not None: + mask = (1 - mask.to(q.dtype))[:, None, None, :] * -10000.0 + out = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, dropout_p=0.0) + return self.proj(out.transpose(1, 2).reshape(batch, tokens, channels)) + + +class SwiGLU(nn.Module): + def __init__(self, dim, hidden_dim): + super().__init__() + self.gate_proj = nn.Linear(dim, hidden_dim) + self.up_proj = nn.Linear(dim, hidden_dim) + self.down_proj = nn.Linear(hidden_dim, dim) + + def forward(self, x): + return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x)) + + +class SanaVideo2Block(nn.Module): + def __init__(self, config, attention): + super().__init__() + dim = config.hidden_size + self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + attention_cls = ( + GatedSoftmaxAttention if attention == "softmax" else GatedLinearAttention + ) + head_dim = ( + config.softmax_head_dim + if attention == "softmax" + else config.linear_head_dim + ) + self.attn = attention_cls(dim, head_dim, config.qk_norm, config.fp32_attention) + self.cross_attn = CrossAttention(dim, config.num_heads, config.cross_norm) + self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.mlp = SwiGLU(dim, int(dim * config.mlp_ratio)) + self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5) + + def _modulation(self, t, batch): + if t.ndim == 2: + return (self.scale_shift_table[None] + t.reshape(batch, 6, -1)).chunk( + 6, dim=1 + ) + groups = t.shape[2] + return ( + self.scale_shift_table[None, None] + t.reshape(batch, groups, 6, -1) + ).chunk(6, dim=2) + + def forward_attn_sublayer(self, x, y, t, mask=None, rotary_emb=None): + batch, _, channels = x.shape + shift, scale, gate, _, _, _ = self._modulation(t, batch) + grouped_shape = ( + (batch, -1, channels) if t.ndim == 2 else (batch, t.shape[2], -1, channels) + ) + attn_input = self.norm1(x).reshape(grouped_shape) * (1 + scale) + shift + delta = self.attn(attn_input.reshape_as(x), rotary_emb) + delta = (gate * delta.reshape(grouped_shape)).reshape_as(x) + return delta + self.cross_attn(x + delta, y, mask) + + def forward_mlp_sublayer(self, x, t): + batch, _, channels = x.shape + _, _, _, shift, scale, gate = self._modulation(t, batch) + grouped_shape = ( + (batch, -1, channels) if t.ndim == 2 else (batch, t.shape[2], -1, channels) + ) + mlp_input = self.norm2(x).reshape(grouped_shape) * (1 + scale) + shift + delta = self.mlp(mlp_input.reshape_as(x)).reshape(grouped_shape) + return (gate * delta).reshape_as(x) + + +class DepthRMSNorm(nn.Module): + def forward(self, x): + return x * (x.square().mean(-1, keepdim=True) + 1e-6).rsqrt() + + +class BlockAttentionResidual(nn.Module): + def __init__(self, hidden_size): + super().__init__() + self.attn_proj = nn.Linear(hidden_size, 1, bias=False) + self.mlp_proj = nn.Linear(hidden_size, 1, bias=False) + self.final_proj = nn.Linear(hidden_size, 1, bias=False) + self.key_norm = DepthRMSNorm() + for projection in (self.attn_proj, self.mlp_proj, self.final_proj): + nn.init.zeros_(projection.weight) + + def attend_buffer(self, projection, values, keys, active_count, partial): + source_count = active_count + if partial is not None: + values[active_count] = partial + keys[active_count] = self.key_norm(partial.unsqueeze(0)).squeeze(0) + source_count += 1 + if source_count == 1: + return values[0] + values, keys = values[:source_count], keys[:source_count] + logits = torch.einsum("d,nbtd->nbt", projection.weight.squeeze(0), keys) + return torch.einsum("nbt,nbtd->btd", logits.softmax(0), values) + + +class SanaVideo2Transformer3DModel(BaseDiT): + _fsdp_shard_conditions: ClassVar[list] = [] + _compile_conditions: ClassVar[list] = [] + _supported_attention_backends: ClassVar[set] = {AttentionBackendEnum.TORCH_SDPA} + param_names_mapping: ClassVar[dict] = {} + reverse_param_names_mapping: ClassVar[dict] = {} + lora_param_names_mapping: ClassVar[dict] = {} + + def __init__(self, config: SanaVideo2Config, hf_config=None, **kwargs): + super().__init__(config, hf_config=hf_config or {}, **kwargs) + c = self.config + self.hidden_size = c.hidden_size + self.num_attention_heads = c.num_heads + self.num_channels_latents = c.in_channels + self.register_buffer( + "pos_embed", torch.zeros(1, c.input_size**2, c.hidden_size) + ) + self.x_embedder = PatchEmbedMS3D(c.patch_size, c.in_channels, c.hidden_size) + self.t_embedder = TimestepEmbedder(c.hidden_size) + self.t_block = nn.Sequential( + nn.SiLU(), nn.Linear(c.hidden_size, 6 * c.hidden_size) + ) + self.y_embedder = CaptionEmbedder( + c.caption_channels, c.hidden_size, c.model_max_length + ) + if c.y_norm: + self.attention_y_norm = _RMSNorm( + c.hidden_size, c.y_norm_scale_factor, c.norm_eps + ) + self.rope_linear = SanaVideo2RotaryPosEmbed(c.linear_head_dim, c.patch_size) + self.rope_softmax = SanaVideo2RotaryPosEmbed(c.softmax_head_dim, c.patch_size) + anchor_count = max(1, int(c.depth * c.softmax_ratio)) + self.softmax_layer_indices = [ + int((i + 1) * c.depth / anchor_count) - 1 for i in range(anchor_count) + ] + self.block_attention_types = [ + "softmax" if i in self.softmax_layer_indices else "linear" + for i in range(c.depth) + ] + self.blocks = nn.ModuleList( + [SanaVideo2Block(c, attention) for attention in self.block_attention_types] + ) + self.final_layer = T2IFinalLayer(c.hidden_size, c.patch_size, c.in_channels) + self.attn_res = BlockAttentionResidual(c.hidden_size) + + @property + def dtype(self): + return self.x_embedder.proj.weight.dtype + + def post_load_weights(self): + for rope in (self.rope_linear, self.rope_softmax): + if rope._freqs.is_meta: + rope._init_freqs_buffer() + + def forward( + self, + hidden_states, + timestep, + encoder_hidden_states, + encoder_attention_mask=None, + **kwargs, + ): + c = self.config + batch, _, frames, height, width = hidden_states.shape + if timestep.ndim == 5: + timestep = timestep.reshape(batch, 1, frames) + if c.timestep_norm_scale_factor == 1.0: + timestep = timestep.long().float() + else: + timestep = timestep.float() / c.timestep_norm_scale_factor + x = self.x_embedder(hidden_states.to(self.dtype)) + ropes = { + "linear": self.rope_linear((frames, height, width), x.device), + "softmax": self.rope_softmax((frames, height, width), x.device), + } + t = self.t_embedder(timestep.flatten()) + t0 = self.t_block(t).unflatten(0, timestep.shape) + t = t.unflatten(0, timestep.shape) + y = self.y_embedder(encoder_hidden_states.to(self.dtype)) + if c.y_norm: + y = self.attention_y_norm(y) + mask = encoder_attention_mask + if mask is not None: + mask = mask.reshape(batch, -1) + # Keep the grouped depth buffer layout stable across denoising steps. + count = math.ceil(c.depth / c.attn_res_block_size) + values = x.new_empty(count + 2, batch, x.shape[1], c.hidden_size) + keys = torch.empty_like(values) + values[0] = x + keys[0] = self.attn_res.key_norm(x.unsqueeze(0)).squeeze(0) + active_count, partial = 1, None + for index, block in enumerate(self.blocks): + hidden = self.attn_res.attend_buffer( + self.attn_res.attn_proj, values, keys, active_count, partial + ) + delta = block.forward_attn_sublayer( + hidden, y, t0, mask, ropes[self.block_attention_types[index]] + ) + partial = delta if partial is None else partial + delta + hidden = self.attn_res.attend_buffer( + self.attn_res.mlp_proj, values, keys, active_count, partial + ) + partial = partial + block.forward_mlp_sublayer(hidden, t0) + if (index + 1) % c.attn_res_block_size == 0 or index + 1 == c.depth: + values[active_count] = partial + keys[active_count] = self.attn_res.key_norm( + partial.unsqueeze(0) + ).squeeze(0) + active_count += 1 + partial = None + x = self.attn_res.attend_buffer( + self.attn_res.final_proj, values, keys, active_count, None + ) + return ( + self.final_layer(x, t) + .transpose(1, 2) + .reshape(batch, c.in_channels, frames, height, width) + ) + + +EntryClass = SanaVideo2Transformer3DModel diff --git a/python/sglang/multimodal_gen/runtime/pipelines/sana_video2.py b/python/sglang/multimodal_gen/runtime/pipelines/sana_video2.py new file mode 100644 index 0000000000..10db7a72a7 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines/sana_video2.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Native SANA-Video 2.0 loading and T2V/TI2V pipeline.""" + +from pathlib import Path +from typing import ClassVar + +import torch +import yaml + +from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import ( + ComponentLoader, +) +from sglang.multimodal_gen.runtime.models.dits.sana_video2 import ( + SanaVideo2Transformer3DModel, +) +from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import ( + ComposedPipelineBase, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages import InputValidationStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import ( + SanaVideo2DenoisingStage, + SanaVideo2LatentPreparationStage, + SanaVideo2TextEncodingStage, +) +from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import maybe_download_model +from sglang.multimodal_gen.runtime.utils.precision import resolve_component_precision + + +class SanaVideo2TransformerLoader(ComponentLoader): + expected_library = "diffusers" + + def component_load_precision(self, server_args, component_name): + return server_args.component_precisions.get(component_name) + + def load_customized(self, component_model_path, server_args, component_name): + root = Path(component_model_path) + with (root / "config.yaml").open() as file: + source = yaml.safe_load(file) + checkpoint = torch.load( + root / "checkpoints/SANA_Video_2.0_5B_720p.pth", + map_location="cpu", + weights_only=True, + mmap=True, + )["state_dict"] + config = server_args.pipeline_config.dit_config + model_fields = config.arch_config.__dataclass_fields__ + values = { + name: value + for section in (source["model"], source["text_encoder"]) + for name, value in section.items() + if name in model_fields + } + if "pos_embed" in checkpoint: + values["input_size"] = int(checkpoint["pos_embed"].shape[1] ** 0.5) + config.update_model_arch(values) + dtype = resolve_component_precision(server_args, component_name) + with torch.device("meta"): + model = SanaVideo2Transformer3DModel(config) + model.load_state_dict(checkpoint, strict=True, assign=True) + model = model.to(dtype=dtype) + model.post_load_weights() + server_args.model_paths[component_name] = component_model_path + return model.eval().requires_grad_(False) + + +class SanaVideo2Pipeline(ComposedPipelineBase): + pipeline_name = "SanaVideo2Pipeline" + is_video_pipeline = True + _required_config_modules: ClassVar[list[str]] = [ + "text_encoder", + "tokenizer", + "vae", + "transformer", + ] + component_loaders: ClassVar[dict[str, type[ComponentLoader]]] = { + "transformer": SanaVideo2TransformerLoader + } + + def _load_config(self): + self.model_path = maybe_download_model( + self.model_path, + revision=self.server_args.revision, + allow_patterns=["config.yaml", "checkpoints/SANA_Video_2.0_5B_720p.pth"], + ) + with (Path(self.model_path) / "config.yaml").open() as file: + source = yaml.safe_load(file) + self.server_args.pipeline_config.prompt_instruction = "\n".join( + source["text_encoder"]["chi_prompt"] + ) + return { + "_class_name": self.pipeline_name, + "_diffusers_version": "0.37.0", + "text_encoder": ["transformers", "Gemma2Model"], + "tokenizer": ["transformers", "AutoTokenizer"], + "vae": ["diffusers", "AutoencoderKLLTX2Video"], + "transformer": ["diffusers", "SanaVideo2Transformer3DModel"], + } + + def _resolve_component_path(self, server_args, module_name, load_module_name): + path = server_args.component_paths.get(module_name) + if path is None: + if module_name == "transformer": + return self.model_path + if module_name == "vae": + root = maybe_download_model( + "Efficient-Large-Model/LTX-2.3-Diffusers", allow_patterns=["vae/**"] + ) + return str(Path(root) / "vae") + path = server_args.component_paths.get( + "text_encoder", "Efficient-Large-Model/gemma-2-2b-it" + ) + path = maybe_download_model(path) + if module_name == "vae" and (Path(path) / "vae/config.json").is_file(): + path = str(Path(path) / "vae") + return path + + def create_pipeline_stages(self, server_args): + self.add_stage(InputValidationStage()) + self.add_stage( + SanaVideo2TextEncodingStage( + [self.get_module("text_encoder")], + [self.get_module("tokenizer")], + server_args.pipeline_config.prompt_instruction, + ), + "prompt_encoding_stage_primary", + ) + self.add_stage(SanaVideo2LatentPreparationStage(self.get_module("vae"))) + self.add_stage(SanaVideo2DenoisingStage(self.get_module("transformer"))) + self.add_standard_decoding_stage() + + +EntryClass = SanaVideo2Pipeline diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_video2.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_video2.py new file mode 100644 index 0000000000..7df4778975 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/sana_video2.py @@ -0,0 +1,280 @@ +# Copyright 2024-2026 NVIDIA CORPORATION & AFFILIATES +# SPDX-License-Identifier: Apache-2.0 +"""Prompt conditioning and request-local flow solvers for SANA-Video 2.0.""" + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.utils.torch_utils import randn_tensor +from PIL import Image + +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( + ComponentUse, +) +from sglang.multimodal_gen.runtime.pipelines.sana_video import ( + select_sana_video_prompt_window, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import ( + TextEncodingStage, +) + + +def sample_flow_dpm(predict_noise, latents, steps, shift, callback=None): + """Second-order multistep DPM-Solver++ with first-order endpoint steps.""" + if steps < 2: + raise ValueError("Flow DPM-Solver requires at least two inference steps") + betas = torch.linspace(1.0, 0.001, steps + 1, device="cpu").to(latents.device) + sigmas = 1.0 - betas + times = (shift * sigmas / (1 + (shift - 1) * sigmas)).flip(0) + previous_time = previous_prediction = None + x = latents + for index in range(steps): + s = times[index].reshape((1,) * x.ndim) + t = times[index + 1].reshape((1,) * x.ndim) + noise = predict_noise(x, times[index]) + prediction = (x - s * noise) / (1 - s) + lambda_s = torch.log(1 - s) - torch.log(s) + lambda_t = torch.log(1 - t) - torch.log(t) + h = lambda_t - lambda_s + alpha_t = torch.exp(torch.log(1 - t)) + phi = torch.expm1(-h) + if index == 0 or index == steps - 1: + x = t / s * x - alpha_t * phi * prediction + else: + lambda_previous = torch.log(1 - previous_time) - torch.log(previous_time) + ratio = (lambda_s - lambda_previous) / h + derivative = (1.0 / ratio) * (prediction - previous_prediction) + x = ( + t / s * x + - alpha_t * phi * prediction + - 0.5 * (alpha_t * phi) * derivative + ) + previous_time, previous_prediction = s, prediction + if callback is not None: + callback(index, times[index + 1], x) + return x + + +def sample_ltx_euler(predict_flow, latents, steps, shift, callback=None): + """Flow Euler with a clean, fixed first latent frame.""" + with torch.device("cpu"): + scheduler = FlowMatchEulerDiscreteScheduler(shift=shift) + scheduler.set_timesteps(steps, device=latents.device) + condition_mask = torch.zeros_like(latents, dtype=torch.float32) + condition_mask[:, :, 0] = 1 + for index, time in enumerate(scheduler.timesteps): + timestep = torch.minimum( + time.expand(latents.shape).float(), (1 - condition_mask) * 1000.0 + ) + prediction = predict_flow(latents, timestep[:, :1, :, :1, :1]) + batch, channels = latents.shape[:2] + updated = ( + scheduler.step( + -prediction.reshape(batch, channels, -1).transpose(1, 2), + time, + latents.reshape(batch, channels, -1).transpose(1, 2), + per_token_timesteps=timestep.reshape(batch, channels, -1)[:, 0], + return_dict=False, + )[0] + .transpose(1, 2) + .reshape(latents.shape) + ) + denoise_mask = time / 1000 - 1e-6 < (1.0 - condition_mask) + latents = torch.where(denoise_mask, updated, latents).to(latents.dtype) + if callback is not None: + callback(index, time, latents) + return latents + + +class SanaVideo2TextEncodingStage(TextEncodingStage): + def __init__(self, text_encoders, tokenizers, instruction): + super().__init__(text_encoders, tokenizers) + self.instruction = instruction + + @torch.no_grad() + def forward(self, batch, server_args): + self.tokenizers[0].padding_side = "right" + length = batch.max_sequence_length or 300 + prompts = [batch.prompt] if isinstance(batch.prompt, str) else batch.prompt + motion = batch.extra.get("motion_score", 10) + suffix = ( + f" motion score: {int(motion)}." + if motion > 0 + else "" + if motion < 0 + else " high motion" + if batch.extra.get("high_motion", False) + else " low motion" + ) + prompts = [self.instruction + prompt.strip() + suffix for prompt in prompts] + encoded_length = ( + len(self.tokenizers[0].encode(self.instruction)) + length - 2 + if self.instruction + else length + ) + outputs = list( + self.encode_text( + prompts, + server_args, + return_attention_mask=True, + max_length=encoded_length, + ) + ) + for index in (0, 1, 3): + outputs[index] = [ + select_sana_video_prompt_window(value, length) + for value in outputs[index] + ] + outputs[4] = [ + [int(value) for value in mask.sum(dim=1).tolist()] for mask in outputs[1] + ] + self._append_positive_text_outputs(batch, *outputs) + if batch.do_classifier_free_guidance: + negative = self.encode_text( + batch.negative_prompt, + server_args, + return_attention_mask=True, + max_length=length, + ) + self._append_negative_text_outputs(batch, outputs[0], *negative) + return batch + + +def prepare_image(image, height, width): + image = image.convert("RGB") + w, h = image.size + scale = max(height / h, width / w) + resized = image.resize( + (round(w * scale), round(h * scale)), Image.Resampling.BICUBIC + ) + top = round((resized.height - height) / 2.0) + left = round((resized.width - width) / 2.0) + pixels = np.array(resized)[top : top + height, left : left + width].copy() + return ( + torch.from_numpy(pixels).permute(2, 0, 1).float().div_(255).sub_(0.5).div_(0.5) + ) + + +class SanaVideo2LatentPreparationStage(PipelineStage): + def __init__(self, vae): + super().__init__() + self.vae = vae + + def component_uses(self, server_args, stage_name=None): + return [ComponentUse(self._component_stage_name(stage_name), "vae")] + + @torch.no_grad() + def forward(self, batch, server_args): + config = server_args.pipeline_config + if batch.height % 32 or batch.width % 32: + raise ValueError("SANA-Video 2.0 height and width must be divisible by 32") + batch.num_frames = config.adjust_num_frames(batch.num_frames) + device = batch.prompt_embeds[0].device + batch_size = batch.prompt_embeds[0].shape[0] + shape = config.prepare_latent_shape( + batch, batch_size, (batch.num_frames - 1) // 8 + 1 + ) + if batch.latents is None: + batch.latents = randn_tensor( + shape, generator=batch.generator, device=device, dtype=torch.float32 + ) + else: + if tuple(batch.latents.shape) != shape: + raise ValueError( + f"Expected latents with shape {shape}, got {tuple(batch.latents.shape)}" + ) + batch.latents = batch.latents.to(device=device, dtype=torch.float32).clone() + if batch.condition_image is not None: + images = ( + batch.condition_image + if isinstance(batch.condition_image, list) + else [batch.condition_image] + ) + if len(images) == 1: + images = images * batch_size + if len(images) != batch_size: + raise ValueError("TI2V requires one conditioning image per prompt") + self.begin_declared_component_use(component_name="vae", module=self.vae) + pixels = torch.stack( + [prepare_image(image, batch.height, batch.width) for image in images] + )[:, :, None] + pixels = pixels.to(device=device, dtype=next(self.vae.parameters()).dtype) + image_latents = self.vae.encode(pixels).latent_dist.mode() + scale, mean = config.get_decode_scale_and_shift( + device, image_latents.dtype, self.vae + ) + batch.latents[:, :, :1] = (image_latents - mean) * scale + return batch + + +class SanaVideo2DenoisingStage(PipelineStage): + def __init__(self, transformer): + super().__init__() + self.transformer = transformer + + @property + def role_affinity(self) -> RoleType: + return RoleType.DENOISER + + def component_uses(self, server_args, stage_name=None): + return [ComponentUse(self._component_stage_name(stage_name), "transformer")] + + def default_workload_iterations(self, batch, num_inference_steps): + return num_inference_steps + + @torch.no_grad() + def forward(self, batch, server_args): + self.begin_declared_component_use( + component_name="transformer", module=self.transformer + ) + cfg = batch.do_classifier_free_guidance + embeds = batch.prompt_embeds[0] + mask = batch.prompt_attention_mask[0] + if cfg: + embeds = torch.cat([batch.negative_prompt_embeds[0], embeds]) + mask = torch.cat([batch.negative_attention_mask[0], mask]) + is_ti2v = batch.condition_image is not None + + def predict(x, time): + inputs = torch.cat([x, x]) if cfg else x + if time.ndim == 0: + timestep = (time * 1000).expand(inputs.shape[0]) + else: + timestep = torch.cat([time, time]) if cfg else time + prediction = self.transformer( + hidden_states=inputs, + timestep=timestep, + encoder_hidden_states=embeds, + encoder_attention_mask=mask, + ) + if not is_ti2v: + # Keep flow-to-noise conversion before CFG to preserve FP32 rounding. + sigma = time.reshape((1,) * x.ndim).to(inputs) + prediction = (1 - sigma) * prediction + inputs + if cfg: + uncond, cond = prediction.chunk(2) + prediction = uncond + batch.guidance_scale * (cond - uncond) + return prediction + + steps = ( + max(2, batch.num_inference_steps) + if batch.is_warmup + else batch.num_inference_steps + ) + sampler = sample_ltx_euler if is_ti2v else sample_flow_dpm + flow_shift = ( + batch.flow_shift + if batch.flow_shift is not None + else server_args.pipeline_config.flow_shift + ) + with self.progress_bar(total=steps, batch=batch) as progress: + batch.latents = sampler( + predict, + batch.latents, + steps, + flow_shift, + callback=lambda *_: progress.update(), + ) + return batch diff --git a/python/sglang/multimodal_gen/test/unit/test_sana_video2.py b/python/sglang/multimodal_gen/test/unit/test_sana_video2.py new file mode 100644 index 0000000000..83d6d30b00 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_sana_video2.py @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +import torch + + +class TestSanaVideo2(unittest.TestCase): + def test_linear_attention_matches_explicit_token_sum(self): + from sglang.multimodal_gen.runtime.models.dits.sana_video2 import ( + GatedLinearAttention, + ) + + torch.manual_seed(12) + module = GatedLinearAttention(24, 12, fp32_attention=True).eval() + x = torch.randn(2, 5, 24) + q, k, v = module.qkv(x).chunk(3, dim=-1) + q = module.q_norm(q).reshape(2, 5, 2, 12) + k = module.k_norm(k).reshape(2, 5, 2, 12) + v = v.reshape(2, 5, 2, 12) + beta = module.beta_proj(x).sigmoid() + scores = torch.einsum("bihd,bjhd->bhij", q, k) + out = torch.einsum("bhij,bjh,bjhd->bihd", scores, beta, v) + out = out.float() * (out.float().square().mean(-1, keepdim=True) + 1e-5).rsqrt() + out = out * module.o_norm.weight + out = out.reshape_as(x) * module.output_gate(x).sigmoid() + expected = module.proj(out) + torch.testing.assert_close(module(x), expected, atol=2e-6, rtol=2e-5) + + def test_softmax_attention_matches_dense_equation(self): + from sglang.multimodal_gen.runtime.models.dits.sana_video2 import ( + GatedSoftmaxAttention, + ) + + torch.manual_seed(13) + module = GatedSoftmaxAttention(24, 12, fp32_attention=True).eval() + x = torch.randn(2, 5, 24) + q, k, v = module.qkv(x).chunk(3, -1) + q = module.q_norm(q).reshape(2, 5, 2, 12) + k = module.k_norm(k).reshape(2, 5, 2, 12) + v = v.reshape(2, 5, 2, 12) + scores = torch.einsum("bihd,bjhd->bhij", q, k) / 12**0.5 + out = torch.einsum("bhij,bjhd->bihd", scores.softmax(-1), v) + expected = module.proj(out.reshape_as(x) * module.output_gate(x).sigmoid()) + torch.testing.assert_close(module(x), expected) + + def test_attention_residual_weights_depth_per_token(self): + from sglang.multimodal_gen.runtime.models.dits.sana_video2 import ( + BlockAttentionResidual, + ) + + torch.manual_seed(14) + module = BlockAttentionResidual(4) + with torch.no_grad(): + module.attn_proj.weight.copy_(torch.tensor([[0.3, -0.4, 0.5, 0.1]])) + values = torch.randn(4, 2, 3, 4) + keys = values * (values.square().mean(-1, keepdim=True) + 1e-6).rsqrt() + scores = (keys * module.attn_proj.weight).sum(-1).softmax(0) + expected = (scores.unsqueeze(-1) * values).sum(0) + actual = module.attend_buffer( + module.attn_proj, values.clone(), keys.clone(), 3, values[3] + ) + torch.testing.assert_close(actual, expected) + + @staticmethod + def small_model(): + from sglang.multimodal_gen.configs.models.dits.sana_video2 import ( + SanaVideo2ArchConfig, + SanaVideo2Config, + ) + from sglang.multimodal_gen.runtime.models.dits.sana_video2 import ( + SanaVideo2Transformer3DModel, + ) + + return SanaVideo2Transformer3DModel( + SanaVideo2Config( + arch_config=SanaVideo2ArchConfig( + hidden_size=24, + depth=4, + num_heads=2, + linear_head_dim=12, + softmax_head_dim=24, + in_channels=4, + caption_channels=8, + model_max_length=5, + mlp_ratio=2, + attn_res_block_size=2, + ) + ) + ).eval() + + def test_uniform_frame_timesteps_equal_batch_timesteps(self): + torch.manual_seed(15) + model = self.small_model() + x, y = torch.randn(2, 4, 3, 2, 2), torch.randn(2, 5, 8) + t = torch.tensor([42.9, 123.9]) + mask = torch.tensor([[1, 1, 1, 0, 0], [1, 1, 1, 1, 0]]) + with torch.no_grad(): + batch = model(x, t, y, mask) + frames = model( + x, t[:, None, None, None, None].expand(-1, 1, 3, 1, 1), y[:, None], mask + ) + changed_t = t[:, None, None, None, None].expand(-1, 1, 3, 1, 1).clone() + changed_t[:, :, 0] = 0 + changed = model(x, changed_t, y, mask) + self.assertEqual(batch.shape, x.shape) + torch.testing.assert_close(batch, frames) + self.assertFalse(torch.allclose(batch, changed)) + self.assertEqual( + model.block_attention_types, ["linear", "linear", "linear", "softmax"] + ) + + def test_padding_tokens_do_not_change_flow_and_calls_do_not_share_shape(self): + torch.manual_seed(16) + model = self.small_model() + x, y, t = torch.randn(1, 4, 2, 2, 2), torch.randn(1, 5, 8), torch.tensor([20.0]) + mask = torch.tensor([[1, 1, 1, 0, 0]]) + with torch.no_grad(): + original = model(x, t, y, mask) + y[:, 3:] = 1000 + model(torch.randn(1, 4, 1, 1, 3), t, y, mask) + actual = model(x, t, y, mask) + torch.testing.assert_close(actual, original) + + def test_dtype_cast_preserves_complex_rotary_frequencies(self): + model = self.small_model() + before = model.rope_linear((2, 2, 2), torch.device("cpu")) + model.to(dtype=torch.bfloat16) + after = model.rope_linear((2, 2, 2), torch.device("cpu")) + self.assertTrue(after.is_complex()) + torch.testing.assert_close(after, before, rtol=0, atol=0) + + def test_official_checkpoint_names_and_output_sign(self): + torch.manual_seed(17) + model = self.small_model() + state = model.state_dict() + for key in ( + "pos_embed", + "y_embedder.y_embedding", + "blocks.0.attn.beta_proj.weight", + "blocks.3.attn.output_gate.weight", + "blocks.0.cross_attn.kv_linear.weight", + "blocks.0.mlp.gate_proj.weight", + "attn_res.final_proj.weight", + ): + self.assertIn(key, state) + model.load_state_dict(state, strict=True) + with torch.no_grad(): + model.final_layer.linear.weight.zero_() + model.final_layer.linear.bias.fill_(2) + output = model( + torch.randn(1, 4, 2, 2, 2), torch.tensor([1.0]), torch.randn(1, 5, 8) + ) + torch.testing.assert_close(output, torch.full_like(output, 2)) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/sglang/multimodal_gen/test/unit/test_sana_video2_pipeline.py b/python/sglang/multimodal_gen/test/unit/test_sana_video2_pipeline.py new file mode 100644 index 0000000000..fc1f9e4ed1 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_sana_video2_pipeline.py @@ -0,0 +1,323 @@ +from contextlib import nullcontext +from itertools import pairwise +from types import SimpleNamespace + +import pytest +import torch + +from sglang.multimodal_gen.configs.pipeline_configs.sana_video2 import ( + SanaVideo2PipelineConfig, +) +from sglang.multimodal_gen.configs.sample.sana_video2 import SanaVideo2SamplingParams +from sglang.multimodal_gen.registry import get_model_info +from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages import ( + sana_video2, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import ( + sample_flow_dpm, + sample_ltx_euler, +) + + +def test_native_registry_precedes_sana_video(monkeypatch): + monkeypatch.setattr( + "sglang.multimodal_gen.registry.maybe_download_model_index", + lambda _: None, + ) + get_model_info.cache_clear() + info = get_model_info("Efficient-Large-Model/SANA-Video_2.0_5B_720p") + assert info.pipeline_config_cls is SanaVideo2PipelineConfig + assert info.sampling_param_cls is SanaVideo2SamplingParams + get_model_info.cache_clear() + + +def test_latent_layout_and_frame_alignment(): + config = SanaVideo2PipelineConfig() + assert config.adjust_num_frames(193) == 193 + assert config.adjust_num_frames(192) == 185 + assert config.prepare_latent_shape( + SimpleNamespace(height=736, width=1280), 2, 25 + ) == (2, 128, 25, 23, 40) + assert config.get_latent_dtype(torch.bfloat16) == torch.float32 + defaults = SanaVideo2SamplingParams() + assert (defaults.height, defaults.width, defaults.num_frames, defaults.fps) == ( + 736, + 1280, + 193, + 24, + ) + assert defaults.num_inference_steps == 50 + + +def test_dpm_constant_data_prediction_reaches_clean_endpoint(): + initial = torch.tensor([[[[[0.4, -0.7]]]]], dtype=torch.float32) + clean = torch.tensor([[[[[0.2, 0.9]]]]], dtype=torch.float32) + times = [] + + def predict_noise(x, time): + times.append(time.item()) + return (x - (1 - time) * clean) / time + + actual = sample_flow_dpm(predict_noise, initial, steps=5, shift=12.0) + torch.testing.assert_close(actual, clean, atol=1e-5, rtol=1e-5) + assert len(times) == 5 + assert all(a > b for a, b in pairwise(times)) + assert actual.dtype == torch.float32 + + +def test_ti2v_euler_preserves_condition_and_uses_frame_timestep(): + initial = torch.zeros(1, 2, 3, 1, 1) + initial[:, :, 0] = 7 + seen = [] + + def predict_flow(x, time): + seen.append(time.clone()) + return torch.ones_like(x) + + actual = sample_ltx_euler(predict_flow, initial.clone(), steps=4, shift=12.0) + torch.testing.assert_close(actual[:, :, 0], initial[:, :, 0], atol=0, rtol=0) + torch.testing.assert_close(actual[:, :, 1:], -torch.ones_like(actual[:, :, 1:])) + assert len(seen) == 4 + assert all(time.shape == (1, 1, 3, 1, 1) for time in seen) + assert all(torch.count_nonzero(time[:, :, 0]) == 0 for time in seen) + assert all(torch.all(time[:, :, 1:] > 0) for time in seen) + + +@pytest.mark.parametrize("conditioned", [False, True]) +@pytest.mark.parametrize("flow_shift, expected", [(None, 12.0), (3.0, 3.0)]) +def test_denoising_uses_request_flow_shift( + monkeypatch, conditioned, flow_shift, expected +): + shifts = [] + + def sample(predict, latents, steps, shift, callback): + shifts.append(shift) + return latents + + monkeypatch.setattr( + sana_video2, "sample_ltx_euler" if conditioned else "sample_flow_dpm", sample + ) + stage = object.__new__(sana_video2.SanaVideo2DenoisingStage) + stage.transformer = None + stage.begin_declared_component_use = lambda **_: None + stage.progress_bar = lambda **_: nullcontext() + batch = SimpleNamespace( + do_classifier_free_guidance=False, + prompt_embeds=[torch.zeros(1, 300, 4)], + prompt_attention_mask=[torch.ones(1, 300)], + condition_image=object() if conditioned else None, + latents=torch.zeros(1, 2, 3, 1, 1), + num_inference_steps=2, + is_warmup=False, + flow_shift=flow_shift, + ) + stage.forward(batch, SimpleNamespace(pipeline_config=SanaVideo2PipelineConfig())) + assert shifts == [expected] + + +def test_denoising_stage_role(): + stage = object.__new__(sana_video2.SanaVideo2DenoisingStage) + assert stage.role_affinity is RoleType.DENOISER + + +def test_vae_statistics_are_inverted_for_decode(): + config = SanaVideo2PipelineConfig() + config.vae_config.arch_config.scaling_factor = 2.0 + vae = SimpleNamespace( + latents_mean=torch.tensor([1.0, -1.0]), latents_std=torch.tensor([2.0, 4.0]) + ) + scale, shift = config.get_decode_scale_and_shift( + torch.device("cpu"), torch.float32, vae + ) + normalized = torch.tensor([2.0, 3.0]).reshape(1, 2, 1, 1, 1) + decoded = normalized / scale + shift + torch.testing.assert_close(decoded.flatten(), torch.tensor([3.0, 5.0])) + + +def test_prompt_window_preserves_case_and_only_positive_motion_suffix(): + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import ( + SanaVideo2TextEncodingStage, + ) + + stage = object.__new__(SanaVideo2TextEncodingStage) + stage.instruction = "instruction: " + stage.tokenizers = [SimpleNamespace(encode=lambda _: [1, 2, 3, 4, 5])] + calls = [] + + def encode(text, server_args, **kwargs): + calls.append((text, kwargs["max_length"])) + length = kwargs["max_length"] + values = torch.arange(length).reshape(1, length, 1) + mask = torch.ones(1, length, dtype=torch.long) + return [values], [mask], [], [mask.bool()], [[length]] + + stage.encode_text = encode + batch = SimpleNamespace( + prompt=" A RED fox ", + negative_prompt="Bad motion", + max_sequence_length=4, + extra={"motion_score": 10}, + do_classifier_free_guidance=True, + prompt_embeds=[], + pooled_embeds=[], + prompt_attention_mask=None, + negative_prompt_embeds=[], + negative_pooled_embeds=[], + negative_attention_mask=None, + ) + stage.forward(batch, SimpleNamespace()) + assert calls == [ + (["instruction: A RED fox motion score: 10."], 7), + ("Bad motion", 4), + ] + assert batch.prompt_embeds[0].flatten().tolist() == [0, 4, 5, 6] + assert batch.negative_prompt_embeds[0].flatten().tolist() == [0, 1, 2, 3] + assert batch.prompt_attention_mask[0].shape == (1, 4) + + +def test_native_model_index_and_component_directory_contract(tmp_path): + import yaml + + from sglang.multimodal_gen.runtime.pipelines.sana_video2 import SanaVideo2Pipeline + + (tmp_path / "config.yaml").write_text( + yaml.safe_dump({"text_encoder": {"chi_prompt": ["first", "last"]}}) + ) + vae_root = tmp_path / "ltx" + (vae_root / "vae").mkdir(parents=True) + (vae_root / "vae/config.json").write_text("{}") + pipeline = object.__new__(SanaVideo2Pipeline) + pipeline.model_path = str(tmp_path) + pipeline.server_args = SimpleNamespace( + revision=None, + pipeline_config=SanaVideo2PipelineConfig(), + component_paths={"vae": str(vae_root), "text_encoder": str(tmp_path)}, + ) + index = pipeline._load_config() + assert index["_class_name"] == "SanaVideo2Pipeline" + assert "_diffusers_version" in index + assert pipeline.server_args.pipeline_config.prompt_instruction == "first\nlast" + assert pipeline._resolve_component_path(pipeline.server_args, "vae", "vae") == str( + vae_root / "vae" + ) + assert pipeline._resolve_component_path( + pipeline.server_args, "tokenizer", "tokenizer" + ) == str(tmp_path) + + +def test_released_pth_layout_loads_strictly_and_materializes_rope(tmp_path): + import yaml + + from sglang.multimodal_gen.configs.models.dits.sana_video2 import SanaVideo2Config + from sglang.multimodal_gen.runtime.models.dits.sana_video2 import ( + SanaVideo2Transformer3DModel, + ) + from sglang.multimodal_gen.runtime.pipelines.sana_video2 import ( + SanaVideo2TransformerLoader, + ) + + arch = { + "hidden_size": 32, + "depth": 2, + "num_heads": 4, + "linear_head_dim": 8, + "softmax_head_dim": 8, + "softmax_ratio": 0.5, + "in_channels": 4, + "caption_channels": 12, + "model_max_length": 5, + "input_size": 2, + } + config = SanaVideo2Config() + config.update_model_arch(arch) + original = SanaVideo2Transformer3DModel(config) + (tmp_path / "checkpoints").mkdir() + torch.save( + {"state_dict": original.state_dict()}, + tmp_path / "checkpoints/SANA_Video_2.0_5B_720p.pth", + ) + (tmp_path / "config.yaml").write_text( + yaml.safe_dump({"model": arch, "text_encoder": {}}) + ) + pipeline_config = SanaVideo2PipelineConfig() + pipeline_config.dit_precision = "fp32" + args = SimpleNamespace( + pipeline_config=pipeline_config, component_precisions={}, model_paths={} + ) + loaded = SanaVideo2TransformerLoader().load_customized( + str(tmp_path), args, "transformer" + ) + assert not any(value.is_meta for value in loaded.buffers()) + for name, value in original.state_dict().items(): + torch.testing.assert_close(loaded.state_dict()[name], value, rtol=0, atol=0) + + +def test_local_native_root_does_not_require_diffusers_model_index(tmp_path): + root = tmp_path / "sana-video2-5b" + root.mkdir() + (root / "config.yaml").write_text("model:\n model: SanaVideo2_5B\n") + get_model_info.cache_clear() + info = get_model_info(str(root)) + assert info.pipeline_config_cls is SanaVideo2PipelineConfig + assert info.sampling_param_cls is SanaVideo2SamplingParams + get_model_info.cache_clear() + + +def test_diffusers_local_root_still_reads_model_index(tmp_path): + from sglang.multimodal_gen.configs.pipeline_configs.sana_video import ( + SanaVideoPipelineConfig, + ) + + root = tmp_path / "sana-video-original" + root.mkdir() + (root / "transformer").mkdir() + (root / "model_index.json").write_text( + '{"_class_name": "SanaVideoPipeline", "_diffusers_version": "0.37.0", ' + '"transformer": ["diffusers", "SanaVideoTransformer3DModel"]}' + ) + get_model_info.cache_clear() + info = get_model_info(str(root)) + assert info.pipeline_config_cls is SanaVideoPipelineConfig + get_model_info.cache_clear() + + +def test_two_outputs_expand_with_independent_generators_before_latent_preparation(): + from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req + from sglang.multimodal_gen.runtime.pipelines_core.stages.input_validation import ( + InputValidationStage, + ) + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.sana_video2 import ( + SanaVideo2LatentPreparationStage, + ) + + sampling = SanaVideo2SamplingParams( + request_id="two-videos", + prompt="A fox", + height=32, + width=32, + num_frames=9, + num_outputs_per_prompt=2, + seed=[11, 12], + generator_device="cpu", + output_path="/tmp", + output_file_name="video.mp4", + ) + request = Req(sampling_params=sampling) + args = SimpleNamespace(pipeline_config=SanaVideo2PipelineConfig()) + requests = list(InputValidationStage().iter_sequential_requests(request, args)) + assert len(requests) == 2 + assert [item.num_outputs_per_prompt for item in requests] == [1, 1] + assert [item.seed for item in requests] == [11, 12] + stage = SanaVideo2LatentPreparationStage(vae=None) + latents = [] + for item in requests: + item.prompt_embeds = [torch.zeros(1, 5, 12)] + stage.forward(item, args) + assert item.latents.shape == (1, 128, 2, 1, 1) + expected = torch.randn( + item.latents.shape, generator=torch.Generator().manual_seed(item.seed) + ) + torch.testing.assert_close(item.latents, expected) + latents.append(item.latents) + assert not torch.equal(latents[0], latents[1]) diff --git a/test/registered/e2e/diffusion/test_sana_video2.py b/test/registered/e2e/diffusion/test_sana_video2.py new file mode 100644 index 0000000000..bf7b1d1c6b --- /dev/null +++ b/test/registered/e2e/diffusion/test_sana_video2.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +from pathlib import Path + +import av +import pytest +import torch +from PIL import Image + +from sglang.multimodal_gen import DiffGenerator +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=60, stage="base-b", runner_config="diffusion-1-gpu-h100") + +pytestmark = [ + pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA"), + pytest.mark.skipif( + os.environ.get("DIFFUSION_PARTITION_ID", "0") != "0", + reason="model smoke runs on diffusion partition 0", + ), +] + + +@pytest.fixture(scope="module") +def generator(): + generator = DiffGenerator.from_pretrained( + model_path="Efficient-Large-Model/SANA-Video_2.0_5B_720p", + warmup_mode="off", + ) + try: + yield generator + finally: + generator.shutdown() + + +@pytest.mark.parametrize("mode", ["t2v", "ti2v"]) +def test_sana_video2_smoke(generator, tmp_path, mode): + params = dict( + prompt="A red cube on a white table.", + width=832, + height=480, + num_frames=17, + fps=24, + num_inference_steps=8, + guidance_scale=8.0, + flow_shift=3.0, + seed=42, + save_output=True, + output_path=str(tmp_path), + output_file_name=f"{mode}.mp4", + ) + if mode == "ti2v": + image_path = tmp_path / "first_frame.png" + Image.new("RGB", (832, 480), "red").save(image_path) + params["image_path"] = str(image_path) + result = generator.generate(params) + assert Path(result.output_file_path).is_file() + with av.open(result.output_file_path) as video: + stream = video.streams.video[0] + assert (stream.width, stream.height) == (832, 480) + assert stream.average_rate == 24 + assert sum(1 for _ in video.decode(video=0)) == 17 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v", "-s"]))