[diffusion] refactor: simplify Kandinsky 6 models, SR sampling and tests (#42768)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-07 08:48:16 +08:00
committed by GitHub
co-authored by Mick Qian
parent f1065423d3
commit 1351f32bb4
47 changed files with 1792 additions and 4620 deletions
@@ -4,7 +4,7 @@
Official out_visual_dim is the total DX head width; n_grid lives in the scheduler.
Unsupported architecture options and unknown fields are rejected."""
from dataclasses import dataclass, field
from dataclasses import dataclass, field, fields
from typing import Any
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
@@ -16,13 +16,6 @@ ATTRIBUTE_OVERRIDE_WHITELIST = ("instruct_type", "visual_cond", "attention_param
# Bookkeeping keys the loader leaves in the component config.
_TOLERATED_EXTRA_KEYS = frozenset({"_class_name", "_diffusers_version"})
_SPARSE_ATTENTION_TYPES = ("nabla", "nabla_framewise_causal")
_PIFLOW_FIELDS = (
"piflow_nfe",
"piflow_num_policy_substeps",
"piflow_final_step_size_scale",
"piflow_shift",
"piflow_eps",
)
_LQ_NOISE_TYPES = ("ddpm", "linear")
@@ -40,7 +33,7 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
}
)
# --- DiffusionTransformer3D constructor keys (trained architecture) ---
# trained architecture, before inference overrides
in_visual_dim: int = 4
in_text_dim: int = 3584
in_text_dim2: int = 768
@@ -61,30 +54,13 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
zero_lq_in_main_path: bool = False
use_adapter: bool = False
adapter_gamma: float = 0.0
# The reference default is True; the SR release is text-free and its config says
# so explicitly.
use_text: bool = False
use_lq_noise_cond: bool = False
# --- optional legacy / test-only DX head / pi-Flow fields ---
# An official checkpoint stores the TOTAL head width in ``out_visual_dim`` and the
# pi-Flow values in its scheduler config; with ``n_grid > 1`` here the head is
# ``out_visual_dim * n_grid`` wide instead (``out_visual_dim`` being the base dim).
n_grid: int = 1
piflow_nfe: int | None = None
piflow_num_policy_substeps: int | None = None
piflow_final_step_size_scale: float | None = None
piflow_shift: float | None = None
piflow_eps: float | None = None
# Post-load attribute overrides (whitelist: instruct_type, visual_cond,
# attention_params); the architecture above stays as trained. The official
# config writes ``null`` when there are none.
# inference-only overrides must not change the trained layer dimensions
attribute_overrides: dict[str, Any] | None = field(default_factory=dict)
# --- SR sampling parameters of the training config ---
# As stored in ``transformer/config.json`` (``sr_params``); expanded into the
# flat ``sr_*`` fields below in ``__post_init__``.
# checkpoint sr_params expands into the flat sr_* fields
sr_params: dict[str, Any] | None = None
sr_visual_size: list[int] = field(default_factory=lambda: [512])
sr_scale_factor: dict[str, list[float]] | None = None
@@ -104,77 +80,18 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
if self.attribute_overrides is None:
self.attribute_overrides = {}
if self.sr_params is not None:
self._expand_sr_params(self.sr_params)
self._reject_unknown_keys()
self._reject_unsupported_flags()
self._normalize_and_validate_shapes()
self._validate_overrides()
self._validate_sampler_fields()
self.hidden_size = self.model_dim
self.num_attention_heads = self.model_dim // sum(self.axes_dims)
self.in_channels = self.in_visual_dim
self.out_channels = self.out_visual_dim
self.num_channels_latents = self.in_visual_dim
@property
def is_piflow(self) -> bool:
return self.piflow_nfe is not None
@property
def head_width(self) -> int:
"""Channels the DiT head emits per latent position (all grids of a DX head)."""
return self.out_visual_dim * self.n_grid
@property
def base_out_visual_dim(self) -> int:
"""Channels per latent grid; legacy n_grid configs store the base output width."""
return self.out_visual_dim if self.n_grid > 1 else self.in_visual_dim
@property
def trained_wide_input(self) -> bool:
"""Whether the input layer was trained on ``[x | cond | mask]`` channels."""
return bool(self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES)
def effective_override(self, name: str) -> Any:
"""Post-load value of ``name``: the override if present, else as trained."""
if name in self.attribute_overrides:
return self.attribute_overrides[name]
return getattr(self, name)
def requested_sparse_attention(self) -> str | None:
"""Sparse attention type asked for by ``attention_params``, if any."""
params = self.effective_override("attention_params")
if not isinstance(params, dict):
return None
for value in params.values():
attention_type = value.get("type") if isinstance(value, dict) else None
if attention_type in _SPARSE_ATTENTION_TYPES:
return attention_type
return None
_SR_PARAM_FIELDS = {
"visual_size": "sr_visual_size",
"scale_factor": "sr_scale_factor",
"scheduler_scale": "sr_scheduler_scale",
"lq_noise_scale": "sr_lq_noise_scale",
"lq_noise_type": "sr_lq_noise_type",
"lq_channel_noise_scale": "sr_lq_channel_noise_scale",
"cap_noise_timestep": "sr_cap_noise_timestep",
"fps": "sr_fps",
}
def _expand_sr_params(self, sr_params: dict[str, Any]) -> None:
"""Nested ``sr_params`` of ``transformer/config.json`` -> the flat ``sr_*`` fields."""
unknown = sorted(set(sr_params) - set(self._SR_PARAM_FIELDS))
if unknown:
raise ValueError(
f"Kandinsky6SR sr_params has unsupported keys {unknown}; accepted "
f"keys: {sorted(self._SR_PARAM_FIELDS)}"
)
for key, value in sr_params.items():
setattr(self, self._SR_PARAM_FIELDS[key], value)
def _reject_unknown_keys(self) -> None:
known = {
item.name.removeprefix("sr_")
for item in fields(self)
if item.name.startswith("sr_") and item.name != "sr_params"
}
unknown = sorted(set(self.sr_params) - known)
if unknown:
raise ValueError(
f"Kandinsky6SR sr_params has unsupported keys {unknown}; accepted keys: {sorted(known)}"
)
for key, value in self.sr_params.items():
setattr(self, f"sr_{key}", value)
unknown = sorted(set(self.extra_attrs) - _TOLERATED_EXTRA_KEYS)
if unknown:
raise ValueError(
@@ -182,23 +99,15 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
f"{unknown}; declare them on Kandinsky6SRArchConfig or fix the checkpoint config."
)
def _reject_unsupported_flags(self) -> None:
if self.use_text:
raise NotImplementedError(
"Kandinsky6SR: use_text=True (text-conditioned SR DiT) is not "
"supported; only the text-free SR DiT (use_text=False) is implemented."
)
if self.use_adapter:
raise NotImplementedError(
"Kandinsky6SR: use_adapter=True (VideoAdapter) is not supported."
)
if self.use_lq_modulation:
raise NotImplementedError(
"Kandinsky6SR: use_lq_modulation=True (per-block LQ modulation) is "
"not supported."
)
def _normalize_and_validate_shapes(self) -> None:
for name, enabled in (
("use_text", self.use_text),
("use_adapter", self.use_adapter),
("use_lq_modulation", self.use_lq_modulation),
):
if enabled:
raise NotImplementedError(
f"Kandinsky6SR: {name}=True is not supported."
)
self.patch_size = tuple(self.patch_size)
self.axes_dims = tuple(self.axes_dims)
if len(self.patch_size) != 3 or self.patch_size[0] != 1:
@@ -218,10 +127,7 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
)
if self.model_dim % 2 != 0:
raise ValueError(f"model_dim must be even, got {self.model_dim}")
if self.n_grid < 1:
raise ValueError(f"n_grid must be >= 1, got {self.n_grid}")
def _validate_overrides(self) -> None:
unknown = sorted(
set(self.attribute_overrides) - set(ATTRIBUTE_OVERRIDE_WHITELIST)
)
@@ -239,19 +145,37 @@ class Kandinsky6SRArchConfig(DiTArchConfig):
visual_cond = self.effective_override("visual_cond")
if not isinstance(visual_cond, bool):
raise ValueError(f"visual_cond must be a bool, got {visual_cond!r}")
def _validate_sampler_fields(self) -> None:
present = [name for name in _PIFLOW_FIELDS if getattr(self, name) is not None]
if present and len(present) != len(_PIFLOW_FIELDS):
missing = sorted(set(_PIFLOW_FIELDS) - set(present))
raise ValueError(
f"pi-Flow config is incomplete: {missing} missing next to {present}"
)
if self.sr_lq_noise_type not in _LQ_NOISE_TYPES:
raise ValueError(
f"sr_lq_noise_type must be one of {list(_LQ_NOISE_TYPES)}, "
f"got {self.sr_lq_noise_type!r}"
)
self.hidden_size = self.model_dim
self.num_attention_heads = self.model_dim // head_dim
self.in_channels = self.in_visual_dim
self.out_channels = self.out_visual_dim
self.num_channels_latents = self.in_visual_dim
@property
def trained_wide_input(self) -> bool:
"""Whether the input layer was trained on ``[x | cond | mask]`` channels."""
return bool(self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES)
def effective_override(self, name: str) -> Any:
"""Post-load value of ``name``: the override if present, else as trained."""
return self.attribute_overrides.get(name, self.__dict__[name])
def requested_sparse_attention(self) -> str | None:
"""Sparse attention type asked for by ``attention_params``, if any."""
params = self.effective_override("attention_params")
if isinstance(params, dict):
for value in params.values():
if (
isinstance(value, dict)
and value.get("type") in _SPARSE_ATTENTION_TYPES
):
return value["type"]
return None
@dataclass
@@ -157,21 +157,15 @@ class Kandinsky6TI2VAPipelineConfig(PipelineConfig):
audio_downsample_factor: int = 1024
def __post_init__(self) -> None:
if len(self.text_encoder_configs) != 2:
raise ValueError(
"Kandinsky6 pipeline requires exactly 2 text encoders "
f"(Reason1 and CLIP), but got {len(self.text_encoder_configs)}."
)
if len(self.text_encoder_precisions) != 2:
raise ValueError(
"Kandinsky6 pipeline requires exactly 2 text encoder "
f"precisions, but got {len(self.text_encoder_precisions)}."
)
if len(self.text_encoder_extra_args) != 2:
raise ValueError(
"Kandinsky6 pipeline requires exactly 2 text encoder extra "
f"tokenizer arg dicts, but got {len(self.text_encoder_extra_args)}."
)
for name, values in (
("text encoders (Reason1 and CLIP)", self.text_encoder_configs),
("text encoder precisions", self.text_encoder_precisions),
("text encoder extra tokenizer arg dicts", self.text_encoder_extra_args),
):
if len(values) != 2:
raise ValueError(
f"Kandinsky6 pipeline requires exactly 2 {name}, but got {len(values)}."
)
# keep the video VAE encoder available for optional image conditioning
self.vae_config.load_encoder = True
@@ -10,11 +10,14 @@ NABLA attention is not supported.
import math
from dataclasses import dataclass
from functools import partial
from typing import Any
import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
from torch.utils.checkpoint import checkpoint
from sglang.kernels.ops.diffusion import apply_matrix_rope, residual_gate_fp32
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
@@ -69,6 +72,28 @@ def _is_kandinsky6_transformer_block(name: str, module: object) -> bool:
return is_module_list_entry_in(name, _KANDINSKY6_BLOCK_CONTAINERS)
def _validate_parallelism(num_heads: int, **tp_dimensions: int) -> None:
"""TP splits heads/FFNs; Ulysses splits video heads remaining after TP."""
tp_size = get_tp_world_size()
ulysses_size, _ = get_ulysses_ctx()
ring_size, _ = get_ring_ctx()
for name, size in (("TP", tp_size), ("Ulysses", ulysses_size), ("ring", ring_size)):
if size <= 0:
raise ValueError(f"Kandinsky6 {name} size must be positive.")
for name, value in dict(num_attention_heads=num_heads, **tp_dimensions).items():
if value % tp_size:
raise ValueError(
f"Kandinsky6 {name}={value} must be divisible by TP size {tp_size}."
)
# ring rotates complete K/V shards, so it imposes no head divisibility constraint
local_heads = num_heads // tp_size
if local_heads % ulysses_size:
raise ValueError(
f"Kandinsky6 TP-local video attention heads {local_heads} must be "
f"divisible by Ulysses size {ulysses_size} (total heads={num_heads}, TP={tp_size})."
)
def _build_rotary_freqs(dim: int, max_period: float) -> torch.Tensor:
return torch.exp(
-math.log(max_period)
@@ -146,20 +171,13 @@ class Kandinsky6VisualEmbeddings(nn.Module):
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
batch_size, duration, height, width, dim = x.shape
x = (
x.view(
batch_size,
duration // self.patch_size[0],
self.patch_size[0],
height // self.patch_size[1],
self.patch_size[1],
width // self.patch_size[2],
self.patch_size[2],
dim,
)
.permute(0, 1, 3, 5, 2, 4, 6, 7)
.flatten(4, 7)
pt, ph, pw = self.patch_size
x = rearrange(
x,
"b (t pt) (h ph) (w pw) c -> b t h w (pt ph pw c)",
pt=pt,
ph=ph,
pw=pw,
)
x, _ = self.in_layer(x)
return x
@@ -321,30 +339,22 @@ class Kandinsky6Attention(nn.Module):
tp_size = get_tp_world_size()
self.local_num_heads = divide(self.num_heads, tp_size)
self.to_query = ColumnParallelLinear(
num_channels,
num_channels,
bias=True,
gather_output=False,
quant_config=quant_config,
prefix=add_prefix("to_query", prefix),
)
self.to_key = ColumnParallelLinear(
kv_dim,
num_channels,
bias=True,
gather_output=False,
quant_config=quant_config,
prefix=add_prefix("to_key", prefix),
)
self.to_value = ColumnParallelLinear(
kv_dim,
num_channels,
bias=True,
gather_output=False,
quant_config=quant_config,
prefix=add_prefix("to_value", prefix),
)
for name, width in (
("to_query", num_channels),
("to_key", kv_dim),
("to_value", kv_dim),
):
self.add_module(
name,
ColumnParallelLinear(
width,
num_channels,
bias=True,
gather_output=False,
quant_config=quant_config,
prefix=add_prefix(name, prefix),
),
)
self.query_norm = Kandinsky6QKNorm(head_dim)
self.key_norm = Kandinsky6QKNorm(head_dim)
self.out_layer = RowParallelLinear(
@@ -414,48 +424,20 @@ class Kandinsky6Attention(nn.Module):
key = gather_seq(key, context_seq_len)
value = gather_seq(value, context_seq_len)
try:
hidden_states = self.attention(
query,
key,
value,
attn_mask_meta=attn_mask_meta,
skip_sequence_parallel_override=skip_sequence_parallel,
)
except AssertionError as exc:
# standalone single-rank parity uses SDPA without a pipeline forward context
if "Forward context is not set" not in str(exc):
raise
query_shape = query.shape[:-2]
key_shape = key.shape[:-2]
query = query.reshape(
query_shape[0], -1, self.local_num_heads, query.shape[-1]
).transpose(1, 2)
key = key.reshape(
key_shape[0], -1, self.local_num_heads, key.shape[-1]
).transpose(1, 2)
value = value.reshape(
key_shape[0], -1, self.local_num_heads, value.shape[-1]
).transpose(1, 2)
hidden_states = F.scaled_dot_product_attention(
query,
key,
value,
attn_mask=None,
is_causal=False,
)
hidden_states = hidden_states.transpose(1, 2).reshape(
*query_shape, self.local_num_heads, -1
)
hidden_states = self.attention(
query,
key,
value,
attn_mask_meta=attn_mask_meta,
skip_sequence_parallel_override=skip_sequence_parallel,
)
hidden_states = hidden_states.flatten(-2, -1)
hidden_states, _ = self.out_layer(hidden_states)
return hidden_states
class _Kandinsky6MLP(nn.Module):
"""Bias-free TP MLP with checkpoint-mapped fc_in/fc_out names.
class Kandinsky6FeedForward(nn.Module):
"""Bias-free TP MLP with checkpoint-mapped mlp.fc_in/fc_out names.
The shared MLP currently hardcodes bias=True, so it cannot be used here."""
@@ -467,7 +449,9 @@ class _Kandinsky6MLP(nn.Module):
quant_config: QuantizationConfig | None = None,
):
super().__init__()
self.fc_in = ColumnParallelLinear(
prefix = add_prefix("mlp", prefix)
self.mlp = nn.Module()
self.mlp.fc_in = ColumnParallelLinear(
dim,
ff_dim,
bias=False,
@@ -475,8 +459,8 @@ class _Kandinsky6MLP(nn.Module):
quant_config=quant_config,
prefix=add_prefix("fc_in", prefix),
)
self.act = get_act_fn("gelu")
self.fc_out = RowParallelLinear(
self.mlp.act = get_act_fn("gelu")
self.mlp.fc_out = RowParallelLinear(
ff_dim,
dim,
bias=False,
@@ -486,29 +470,12 @@ class _Kandinsky6MLP(nn.Module):
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x, _ = self.fc_in(x)
x = self.act(x)
x, _ = self.fc_out(x)
x, _ = self.mlp.fc_in(x)
x = self.mlp.act(x)
x, _ = self.mlp.fc_out(x)
return x
class Kandinsky6FeedForward(nn.Module):
def __init__(
self,
dim: int,
ff_dim: int,
prefix: str = "",
quant_config: QuantizationConfig | None = None,
):
super().__init__()
self.mlp = _Kandinsky6MLP(
dim, ff_dim, prefix=add_prefix("mlp", prefix), quant_config=quant_config
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.mlp(x)
def _norm_scale_shift(
norm: LayerNormScaleShift, x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor
) -> torch.Tensor:
@@ -535,7 +502,11 @@ class Kandinsky6OutLayer(nn.Module):
)
def forward(
self, visual_embed: torch.Tensor, time_embed: torch.Tensor
self,
visual_embed: torch.Tensor,
time_embed: torch.Tensor,
*,
compute_dtype: torch.dtype | None = None,
) -> torch.Tensor:
shift, scale = torch.chunk(
self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1
@@ -543,28 +514,18 @@ class Kandinsky6OutLayer(nn.Module):
visual_embed = (
self.norm(visual_embed.float()) * (scale.float()[:, None, None] + 1.0)
+ shift.float()[:, None, None]
).type_as(visual_embed)
).to(dtype=compute_dtype or visual_embed.dtype)
x, _ = self.out_layer(visual_embed)
batch_size, duration, height, width, _ = x.shape
x = (
x.view(
batch_size,
duration,
height,
width,
-1,
self.patch_size[0],
self.patch_size[1],
self.patch_size[2],
)
.permute(0, 1, 5, 2, 6, 3, 7, 4)
.flatten(1, 2)
.flatten(2, 3)
.flatten(3, 4)
pt, ph, pw = self.patch_size
return rearrange(
x,
"b t h w (c pt ph pw) -> b (t pt) (h ph) (w pw) c",
pt=pt,
ph=ph,
pw=pw,
)
return x
class Kandinsky6OutLayerAudio(nn.Module):
@@ -591,8 +552,12 @@ class Kandinsky6OutLayerAudio(nn.Module):
return out
class Kandinsky6TransformerEncoderBlock(nn.Module):
"""Text-only self-attention + feed-forward block (video/audio text towers)."""
class Kandinsky6TransformerBlock(nn.Module):
"""Shared parameter layout for text, video and SR transformer blocks."""
modulation_name = "visual_modulation"
with_cross_attention = False
skip_sequence_parallel = False
def __init__(
self,
@@ -605,24 +570,38 @@ class Kandinsky6TransformerEncoderBlock(nn.Module):
quant_config: QuantizationConfig | None = None,
):
super().__init__()
self.text_modulation = Kandinsky6Modulation(time_dim, model_dim, 6)
self.self_attention_norm = LayerNormScaleShift(
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
self.add_module(
self.modulation_name,
Kandinsky6Modulation(
time_dim, model_dim, 9 if self.with_cross_attention else 6
),
)
self.self_attention = Kandinsky6Attention(
model_dim,
head_dim,
supported_attention_backends=supported_attention_backends,
prefix=add_prefix("self_attention", prefix),
quant_config=quant_config,
# text is replicated across SP ranks, so skip Ulysses even for self-attention
skip_sequence_parallel=True,
)
self.feed_forward_norm = LayerNormScaleShift(
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
attention_names = (
("self_attention", "cross_attention")
if self.with_cross_attention
else ("self_attention",)
)
for name in (*attention_names, "feed_forward"):
self.add_module(
f"{name}_norm",
LayerNormScaleShift(
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
),
)
for name in attention_names:
self.add_module(
name,
Kandinsky6Attention(
model_dim,
head_dim,
supported_attention_backends,
prefix=add_prefix(name, prefix),
quant_config=quant_config,
is_cross_attention=name == "cross_attention",
skip_sequence_parallel=self.skip_sequence_parallel
or name == "cross_attention",
),
)
self.feed_forward = Kandinsky6FeedForward(
model_dim,
ff_dim,
@@ -630,6 +609,13 @@ class Kandinsky6TransformerEncoderBlock(nn.Module):
quant_config=quant_config,
)
class Kandinsky6TransformerEncoderBlock(Kandinsky6TransformerBlock):
"""Replicated text self-attention and FFN."""
modulation_name = "text_modulation"
skip_sequence_parallel = True
def forward(
self, x: torch.Tensor, time_embed: torch.Tensor, rope: torch.Tensor
) -> torch.Tensor:
@@ -649,59 +635,10 @@ class Kandinsky6TransformerEncoderBlock(nn.Module):
return x
class Kandinsky6TransformerDecoderBlock(nn.Module):
"""Self-attention, text cross-attention and FFN.
class Kandinsky6TransformerDecoderBlock(Kandinsky6TransformerBlock):
"""Video self-attention, text cross-attention and FFN."""
The joint decoder invokes these sublayers separately to insert cross-modal attention."""
def __init__(
self,
model_dim: int,
time_dim: int,
ff_dim: int,
head_dim: int,
supported_attention_backends: set[AttentionBackendEnum] | None = None,
prefix: str = "",
quant_config: QuantizationConfig | None = None,
):
super().__init__()
self.visual_modulation = Kandinsky6Modulation(time_dim, model_dim, 9)
self.self_attention_norm = LayerNormScaleShift(
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
)
self.self_attention = Kandinsky6Attention(
model_dim,
head_dim,
supported_attention_backends=supported_attention_backends,
prefix=add_prefix("self_attention", prefix),
quant_config=quant_config,
# Video self-attention over the (long) visual token sequence --
# the one role this port's sequence parallelism actually shards.
)
self.cross_attention_norm = LayerNormScaleShift(
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
)
self.cross_attention = Kandinsky6Attention(
model_dim,
head_dim,
supported_attention_backends=supported_attention_backends,
prefix=add_prefix("cross_attention", prefix),
quant_config=quant_config,
# video queries attend to replicated text K/V
is_cross_attention=True,
)
self.feed_forward_norm = LayerNormScaleShift(
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
)
self.feed_forward = Kandinsky6FeedForward(
model_dim,
ff_dim,
prefix=add_prefix("feed_forward", prefix),
quant_config=quant_config,
)
with_cross_attention = True
def forward(
self,
@@ -749,12 +686,6 @@ class Kandinsky6TransformerDecoderBlock(nn.Module):
return visual_embed
def _apply_gate_sum(
x: torch.Tensor, out: torch.Tensor, gate: torch.Tensor
) -> torch.Tensor:
return residual_gate_fp32(x, out, gate)
def _apply_scale_shift(
norm: nn.LayerNorm, x: torch.Tensor, scale: torch.Tensor, shift: torch.Tensor
) -> torch.Tensor:
@@ -856,7 +787,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
self.videoT.visual_modulation(t_v).unsqueeze(dim=1), 3, dim=-1
)
shift, scale, gate = torch.chunk(sa_p, 3, dim=-1)
vis = _apply_gate_sum(
vis = residual_gate_fp32(
vis,
self.videoT.self_attention(
_norm_scale_shift(self.videoT.self_attention_norm, vis, shift, scale),
@@ -877,7 +808,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
self.audioT.visual_modulation(t_a).unsqueeze(dim=1), 3, dim=-1
)
shift, scale, gate = torch.chunk(sa_p, 3, dim=-1)
aud = _apply_gate_sum(
aud = residual_gate_fp32(
aud,
self.audioT.self_attention(
_norm_scale_shift(self.audioT.self_attention_norm, aud, shift, scale),
@@ -893,7 +824,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
aud_out_t = self.audioT.cross_attention(
aud_pre_ca, encoder_hidden_states=text_a
)
aud = _apply_gate_sum(aud, aud_out_t, gate_a)
aud = residual_gate_fp32(aud, aud_out_t, gate_a)
t_va_mod = t_a if not self.fix_modulation else t_v
t_av_mod = t_v if not self.fix_modulation else t_a
@@ -910,7 +841,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
va_shift, va_scale, va_gate = torch.chunk(va_params, 3, dim=-1)
av_shift, av_scale, av_gate = torch.chunk(av_params, 3, dim=-1)
vis = _apply_gate_sum(vis, vis_out_t, gate_v)
vis = residual_gate_fp32(vis, vis_out_t, gate_v)
vis_for_va = _apply_scale_shift(self.va_normalization, vis, va_scale, va_shift)
aud_for_av = _apply_scale_shift(self.av_normalization, aud, av_scale, av_shift)
rq_v = vis_rope if self.ca_rope else None
@@ -928,19 +859,19 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
rotary_emb_kv=rq_v,
context_seq_len=video_seq_len,
)
vis = _apply_gate_sum(
vis = residual_gate_fp32(
vis,
vis_from_aud,
(va_gate if not self.cross_gates else av_gate) * va_gate_scale,
)
aud = _apply_gate_sum(
aud = residual_gate_fp32(
aud,
aud_from_vis,
(av_gate if not self.cross_gates else va_gate) * av_gate_scale,
)
shift, scale, gate = torch.chunk(ff_p, 3, dim=-1)
vis = _apply_gate_sum(
vis = residual_gate_fp32(
vis,
self.videoT.feed_forward(
_norm_scale_shift(self.videoT.feed_forward_norm, vis, shift, scale)
@@ -948,7 +879,7 @@ class Kandinsky6FusedTransformerDecoderBlock(nn.Module):
gate,
)
shift, scale, gate = torch.chunk(ff_p_a, 3, dim=-1)
aud = _apply_gate_sum(
aud = residual_gate_fp32(
aud,
self.audioT.feed_forward(
_norm_scale_shift(self.audioT.feed_forward_norm, aud, shift, scale)
@@ -977,42 +908,6 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
AttentionBackendEnum.TORCH_SDPA,
}
@staticmethod
def _validate_tp_config(
*, arch: Kandinsky6ArchConfig, tp_size: int, num_heads: int, num_heads_a: int
) -> None:
if tp_size <= 0:
raise ValueError("Kandinsky6 TP size must be positive.")
for name, value in (
("num_attention_heads (video)", num_heads),
("num_attention_heads_a (audio)", num_heads_a),
("ff_dim", arch.ff_dim),
("ff_dim_a", arch.ff_dim_a),
):
if value % tp_size:
raise ValueError(
f"Kandinsky6 {name}={value} must be divisible by TP size {tp_size}."
)
@staticmethod
def _validate_sequence_parallel_config(
*, tp_size: int, num_heads: int, ulysses_size: int, ring_size: int
) -> None:
if ulysses_size <= 0:
raise ValueError("Kandinsky6 Ulysses size must be positive.")
if ring_size <= 0:
raise ValueError("Kandinsky6 ring size must be positive.")
if ulysses_size == 1 and ring_size == 1:
return
# only Ulysses splits heads; ring rotates complete K/V shards
local_heads = num_heads // tp_size
if local_heads % ulysses_size:
raise ValueError(
f"Kandinsky6 TP-local video attention heads {local_heads} must be "
f"divisible by Ulysses size {ulysses_size} (total video heads="
f"{num_heads}, TP={tp_size})."
)
def __init__(
self,
config: Kandinsky6VideoAudioConfig,
@@ -1034,20 +929,11 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
head_dim = sum(arch.axes_dims)
head_dim_a = sum(arch.axes_dims_a)
tp_size = get_tp_world_size()
ulysses_size, _ = get_ulysses_ctx()
ring_size, _ = get_ring_ctx()
self._validate_tp_config(
arch=arch,
tp_size=tp_size,
num_heads=arch.model_dim // head_dim,
num_heads_a=arch.model_dim_a // head_dim_a,
)
self._validate_sequence_parallel_config(
tp_size=tp_size,
num_heads=arch.model_dim // head_dim,
ulysses_size=ulysses_size,
ring_size=ring_size,
_validate_parallelism(
arch.model_dim // head_dim,
num_attention_heads_a=arch.model_dim_a // head_dim_a,
ff_dim=arch.ff_dim,
ff_dim_a=arch.ff_dim_a,
)
self.in_visual_dim = arch.in_visual_dim
self.in_audio_dim = arch.in_audio_dim
@@ -1079,46 +965,10 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
)
if not self.is_multimodal:
self.time_embeddings = Kandinsky6TimeEmbeddings(
arch.model_dim, arch.time_dim
)
self.text_embeddings = Kandinsky6TextEmbeddings(
arch.in_text_dim, arch.model_dim
)
self.pooled_text_embeddings = Kandinsky6TextEmbeddings(
arch.in_text_dim2, arch.time_dim
)
self.text_rope_embeddings = Kandinsky6RoPE1D(head_dim)
self.text_transformer_blocks = nn.ModuleList(
[
Kandinsky6TransformerEncoderBlock(
arch.model_dim,
arch.time_dim,
arch.ff_dim,
head_dim,
self._supported_attention_backends,
prefix=add_prefix(f"text_transformer_blocks.{i}", self.prefix),
quant_config=quant_config,
)
for i in range(arch.num_text_blocks)
]
)
self.visual_transformer_blocks = nn.ModuleList(
[
Kandinsky6TransformerDecoderBlock(
arch.model_dim,
arch.time_dim,
arch.ff_dim,
head_dim,
self._supported_attention_backends,
prefix=add_prefix(
f"visual_transformer_blocks.{i}", self.prefix
),
quant_config=quant_config,
)
for i in range(arch.num_visual_blocks)
]
self._build_text_tower(
"", arch.model_dim, arch.time_dim, arch.ff_dim, head_dim
)
block_cls = Kandinsky6TransformerDecoderBlock
else:
self.audio_embeddings = Kandinsky6TextEmbeddings(
arch.in_audio_dim, arch.model_dim_a
@@ -1132,81 +982,35 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
arch.out_audio_dim or arch.in_audio_dim,
)
for (
tower_prefix,
tower_model_dim,
tower_time_dim,
tower_head_dim,
tower_ff_dim,
) in (
("video", arch.model_dim, arch.time_dim, head_dim, arch.ff_dim),
("audio", arch.model_dim_a, arch.time_dim_a, head_dim_a, arch.ff_dim_a),
):
setattr(
self,
f"{tower_prefix}_time_embeddings",
Kandinsky6TimeEmbeddings(tower_model_dim, tower_time_dim),
)
setattr(
self,
f"{tower_prefix}_text_embeddings",
Kandinsky6TextEmbeddings(arch.in_text_dim, tower_model_dim),
)
setattr(
self,
f"{tower_prefix}_pooled_text_embeddings",
Kandinsky6TextEmbeddings(arch.in_text_dim2, tower_time_dim),
)
setattr(
self,
f"{tower_prefix}_text_rope_embeddings",
Kandinsky6RoPE1D(tower_head_dim),
)
setattr(
self,
f"{tower_prefix}_text_transformer_blocks",
nn.ModuleList(
[
Kandinsky6TransformerEncoderBlock(
tower_model_dim,
tower_time_dim,
tower_ff_dim,
tower_head_dim,
self._supported_attention_backends,
prefix=add_prefix(
f"{tower_prefix}_text_transformer_blocks.{i}",
self.prefix,
),
quant_config=quant_config,
)
for i in range(arch.num_text_blocks)
]
),
)
self.visual_transformer_blocks = nn.ModuleList(
[
Kandinsky6FusedTransformerDecoderBlock(
arch.model_dim,
arch.time_dim,
arch.ff_dim,
head_dim,
arch.model_dim_a,
arch.time_dim_a,
arch.ff_dim_a,
head_dim_a,
self._supported_attention_backends,
prefix=add_prefix(
f"visual_transformer_blocks.{i}", self.prefix
),
ca_rope=arch.ca_rope,
cross_gates=arch.cross_gates,
fix_modulation=arch.fix_modulation,
quant_config=quant_config,
)
for i in range(arch.num_visual_blocks)
]
self._build_text_tower(
"video_", arch.model_dim, arch.time_dim, arch.ff_dim, head_dim
)
self._build_text_tower(
"audio_", arch.model_dim_a, arch.time_dim_a, arch.ff_dim_a, head_dim_a
)
block_cls = partial(
Kandinsky6FusedTransformerDecoderBlock,
model_dim_a=arch.model_dim_a,
time_dim_a=arch.time_dim_a,
ff_dim_a=arch.ff_dim_a,
head_dim_a=head_dim_a,
ca_rope=arch.ca_rope,
cross_gates=arch.cross_gates,
fix_modulation=arch.fix_modulation,
)
self.visual_transformer_blocks = nn.ModuleList(
block_cls(
arch.model_dim,
arch.time_dim,
arch.ff_dim,
head_dim,
supported_attention_backends=self._supported_attention_backends,
prefix=add_prefix(f"visual_transformer_blocks.{i}", self.prefix),
quant_config=quant_config,
)
for i in range(arch.num_visual_blocks)
)
self.gradient_checkpointing = False
self.hidden_size = arch.hidden_size
@@ -1219,6 +1023,38 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
) + ["visual_transformer_blocks"]
self.__post_init__()
def _build_text_tower(self, prefix, model_dim, time_dim, ff_dim, head_dim):
arch = self.config
self.add_module(
f"{prefix}time_embeddings", Kandinsky6TimeEmbeddings(model_dim, time_dim)
)
self.add_module(
f"{prefix}text_embeddings",
Kandinsky6TextEmbeddings(arch.in_text_dim, model_dim),
)
self.add_module(
f"{prefix}pooled_text_embeddings",
Kandinsky6TextEmbeddings(arch.in_text_dim2, time_dim),
)
self.add_module(f"{prefix}text_rope_embeddings", Kandinsky6RoPE1D(head_dim))
self.add_module(
f"{prefix}text_transformer_blocks",
nn.ModuleList(
Kandinsky6TransformerEncoderBlock(
model_dim,
time_dim,
ff_dim,
head_dim,
self._supported_attention_backends,
prefix=add_prefix(
f"{prefix}text_transformer_blocks.{i}", self.prefix
),
quant_config=self.quant_config,
)
for i in range(arch.num_text_blocks)
),
)
def _encode_text(
self,
prefix: str | None,
@@ -1237,11 +1073,8 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
text_rope = rope_embeddings(text_rope_pos).unsqueeze(dim=0)
for block in blocks:
if torch.is_grad_enabled() and self.gradient_checkpointing:
te = torch.utils.checkpoint.checkpoint(
block, te, tm, text_rope, use_reentrant=False
)
else:
te = block(te, tm, text_rope)
block = partial(checkpoint, block, use_reentrant=False)
te = block(te, tm, text_rope)
return te, tm
def forward(
@@ -1332,51 +1165,27 @@ class Kandinsky6Transformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
audio_rope = self.audio_rope_embeddings(audio_rope_pos).unsqueeze(dim=0)
for block in self.visual_transformer_blocks:
if torch.is_grad_enabled() and self.gradient_checkpointing:
visual_embed, audio_embed = torch.utils.checkpoint.checkpoint(
block,
visual_embed,
audio_embed,
video_te,
audio_te,
(video_tm, audio_tm),
visual_rope,
audio_rope,
va_gate_scale,
av_gate_scale,
video_seq_len=shard.orig_len,
video_attn_meta=attn_meta,
use_reentrant=False,
)
else:
visual_embed, audio_embed = block(
visual_embed,
audio_embed,
video_te,
audio_te,
(video_tm, audio_tm),
visual_rope,
audio_rope,
va_gate_scale,
av_gate_scale,
video_seq_len=shard.orig_len,
video_attn_meta=attn_meta,
)
block = partial(checkpoint, block, use_reentrant=False)
visual_embed, audio_embed = block(
visual_embed,
audio_embed,
video_te,
audio_te,
(video_tm, audio_tm),
visual_rope,
audio_rope,
va_gate_scale,
av_gate_scale,
video_seq_len=shard.orig_len,
video_attn_meta=attn_meta,
)
else:
for block in self.visual_transformer_blocks:
if torch.is_grad_enabled() and self.gradient_checkpointing:
visual_embed = torch.utils.checkpoint.checkpoint(
block,
visual_embed,
video_te,
video_tm,
visual_rope,
attn_mask_meta=attn_meta,
use_reentrant=False,
)
else:
visual_embed = block(
visual_embed, video_te, video_tm, visual_rope, attn_meta
)
block = partial(checkpoint, block, use_reentrant=False)
visual_embed = block(
visual_embed, video_te, video_tm, visual_rope, attn_meta
)
visual_embed = gather_seq(visual_embed, shard.orig_len).reshape(
*visual_shape, -1
@@ -12,16 +12,11 @@ import torch
import torch.nn as nn
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
ATTRIBUTE_OVERRIDE_WHITELIST,
WIDE_INPUT_INSTRUCT_TYPES,
Kandinsky6SRArchConfig,
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_ring_ctx,
get_ulysses_ctx,
)
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import (
gather_seq,
shard_like,
@@ -38,14 +33,13 @@ from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload im
)
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
Kandinsky6Attention,
Kandinsky6FeedForward,
Kandinsky6Modulation,
Kandinsky6OutLayer,
Kandinsky6RoPE3D,
Kandinsky6TimeEmbeddings,
Kandinsky6TransformerBlock,
Kandinsky6VisualEmbeddings,
_build_rotary_freqs,
_validate_parallelism,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
@@ -80,40 +74,8 @@ class Kandinsky6SRRoPE3D(Kandinsky6RoPE3D):
return self
class Kandinsky6SRDecoderBlock(nn.Module):
"""Modulated self-attention and FFN with fp32 residuals and parameter-dtype linears."""
def __init__(
self,
model_dim: int,
time_dim: int,
ff_dim: int,
head_dim: int,
supported_attention_backends: set[AttentionBackendEnum] | None = None,
prefix: str = "",
quant_config: QuantizationConfig | None = None,
):
super().__init__()
self.visual_modulation = Kandinsky6Modulation(time_dim, model_dim, 6)
self.self_attention_norm = LayerNormScaleShift(
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
)
self.self_attention = Kandinsky6Attention(
model_dim,
head_dim,
supported_attention_backends=supported_attention_backends,
prefix=add_prefix("self_attention", prefix),
quant_config=quant_config,
)
self.feed_forward_norm = LayerNormScaleShift(
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
)
self.feed_forward = Kandinsky6FeedForward(
model_dim,
ff_dim,
prefix=add_prefix("feed_forward", prefix),
quant_config=quant_config,
)
class Kandinsky6SRDecoderBlock(Kandinsky6TransformerBlock):
"""Modulated self-attention and FFN with fp32 residuals."""
def forward(
self,
@@ -139,32 +101,6 @@ class Kandinsky6SRDecoderBlock(nn.Module):
return visual_embed + gate.float() * out.float()
class Kandinsky6SROutLayer(Kandinsky6OutLayer):
"""K6 ``OutLayer`` for an fp32 residual stream (norm in fp32, linear in weight dtype)."""
def forward(
self, visual_embed: torch.Tensor, time_embed: torch.Tensor
) -> torch.Tensor:
shift, scale = torch.chunk(
self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1
)
x = (
self.norm(visual_embed.float()) * (scale.float()[:, None, None] + 1.0)
+ shift.float()[:, None, None]
)
x, _ = self.out_layer(x.to(self.out_layer.weight.dtype))
batch_size, duration, height, width, _ = x.shape
pt, ph, pw = self.patch_size
return (
x.view(batch_size, duration, height, width, -1, pt, ph, pw)
.permute(0, 1, 5, 2, 6, 3, 7, 4)
.flatten(1, 2)
.flatten(2, 3)
.flatten(3, 4)
)
class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
"""Text-free Kandinsky 6 SR DiT (optionally with a DX / pi-Flow head)."""
@@ -179,39 +115,6 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
AttentionBackendEnum.TORCH_SDPA,
}
@staticmethod
def _validate_tp_config(*, arch: Kandinsky6SRArchConfig, tp_size: int) -> None:
if tp_size <= 0:
raise ValueError("Kandinsky6SR TP size must be positive.")
for name, value in (
("num_attention_heads", arch.num_attention_heads),
("ff_dim", arch.ff_dim),
):
if value % tp_size:
raise ValueError(
f"Kandinsky6SR {name}={value} must be divisible by TP size {tp_size}."
)
@staticmethod
def _validate_sequence_parallel_config(
*, arch: Kandinsky6SRArchConfig, tp_size: int, ulysses_size: int, ring_size: int
) -> None:
if ulysses_size <= 0:
raise ValueError("Kandinsky6SR Ulysses size must be positive.")
if ring_size <= 0:
raise ValueError("Kandinsky6SR ring size must be positive.")
if ulysses_size == 1 and ring_size == 1:
return
# Ring Attention rotates whole K/V shards between ranks rather than
# splitting heads, so only Ulysses constrains head divisibility.
local_heads = arch.num_attention_heads // tp_size
if local_heads % ulysses_size:
raise ValueError(
f"Kandinsky6SR TP-local attention heads {local_heads} must be "
f"divisible by Ulysses size {ulysses_size} (total heads="
f"{arch.num_attention_heads}, TP={tp_size})."
)
def __init__(
self,
config: Kandinsky6SRDitConfig,
@@ -223,45 +126,19 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
self.quant_config = quant_config
head_dim = sum(arch.axes_dims)
tp_size = get_tp_world_size()
ulysses_size, _ = get_ulysses_ctx()
ring_size, _ = get_ring_ctx()
self._validate_tp_config(arch=arch, tp_size=tp_size)
self._validate_sequence_parallel_config(
arch=arch, tp_size=tp_size, ulysses_size=ulysses_size, ring_size=ring_size
)
_validate_parallelism(arch.num_attention_heads, ff_dim=arch.ff_dim)
self.in_visual_dim = arch.in_visual_dim
self.base_out_visual_dim = arch.base_out_visual_dim
self.n_grid = arch.n_grid
self.model_dim = arch.model_dim
self.patch_size = arch.patch_size
self.use_motion_score = arch.use_motion_score
self.use_lq_noise_cond = arch.use_lq_noise_cond
# The input width is fixed by the *trained* configuration ...
# overrides change inference behavior, never the trained input width
self.visual_embed_dim = (
2 * arch.in_visual_dim + 1
if arch.trained_wide_input
else arch.in_visual_dim
)
self._build_layers(arch, head_dim=head_dim, quant_config=quant_config)
self.hidden_size = arch.hidden_size
self.num_attention_heads = arch.num_attention_heads
self.num_channels_latents = arch.num_channels_latents
self.layer_names = list(_BLOCK_CONTAINERS)
# ... while inference behaviour comes from the post-load overrides.
self._apply_attribute_overrides(arch)
self._require_all_checkpoint_keys()
self.__post_init__()
def _build_layers(
self,
arch: Kandinsky6SRArchConfig,
*,
head_dim: int,
quant_config: QuantizationConfig | None,
) -> None:
self.time_embeddings = Kandinsky6TimeEmbeddings(arch.model_dim, arch.time_dim)
if arch.use_motion_score:
self.motion_embeddings = Kandinsky6TimeEmbeddings(
@@ -295,35 +172,17 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
for i in range(arch.num_visual_blocks)
]
)
self.out_layer = Kandinsky6SROutLayer(
self.out_layer = Kandinsky6OutLayer(
arch.model_dim,
arch.time_dim,
arch.head_width,
arch.out_visual_dim,
arch.patch_size,
)
def _apply_attribute_overrides(self, arch: Kandinsky6SRArchConfig) -> None:
self.instruct_type = arch.instruct_type
self.visual_cond = arch.visual_cond
self.attention_params = arch.attention_params
for name in ATTRIBUTE_OVERRIDE_WHITELIST:
if name in arch.attribute_overrides:
setattr(self, name, arch.attribute_overrides[name])
self._check_input_width_matches_overrides()
sparse_type = arch.requested_sparse_attention()
if sparse_type is not None:
logger.warning(
"Kandinsky6SR: attention_params request %r sparse attention; this port "
"runs dense attention, so outputs differ from the sparse reference.",
sparse_type,
)
def _check_input_width_matches_overrides(self) -> None:
run_wide = self.visual_cond or self.instruct_type in (
"channel",
"hybrid",
"hybrid_anchor",
)
self.instruct_type = arch.effective_override("instruct_type")
self.visual_cond = arch.effective_override("visual_cond")
self.attention_params = arch.effective_override("attention_params")
run_wide = self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES
run_width = 2 * self.in_visual_dim + 1 if run_wide else self.in_visual_dim
if run_width != self.visual_embed_dim:
raise ValueError(
@@ -334,10 +193,21 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
"under instruct_type='noise'."
)
def _require_all_checkpoint_keys(self) -> None:
"""A missing tensor must fail loudly instead of being zero-filled by the loader."""
sparse_type = arch.requested_sparse_attention()
if sparse_type is not None:
logger.warning(
"Kandinsky6SR: attention_params request %r sparse attention; this port "
"runs dense attention, so outputs differ from the sparse reference.",
sparse_type,
)
self.hidden_size = arch.hidden_size
self.num_attention_heads = arch.num_attention_heads
self.num_channels_latents = arch.num_channels_latents
self.layer_names = list(_BLOCK_CONTAINERS)
# missing checkpoint tensors must fail, not be zero-filled
for param in self.parameters():
param.missing_param_init = "error"
self.__post_init__()
def preprocess_loaded_state_dict(
self, weight_iterator: Iterable[tuple[str, torch.Tensor]]
@@ -371,7 +241,16 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
timesteps are scaled by 1000. DX channels are ordered grid-major;
the scheduler reshapes them into n_grid velocity predictions."""
self._check_latent_shape(hidden_states)
if hidden_states.ndim != 5 or hidden_states.shape[-1] != self.visual_embed_dim:
raise ValueError(
"Kandinsky6SR expects a [B, T, H, W, "
f"{self.visual_embed_dim}] latent, got {tuple(hidden_states.shape)}"
)
height, width = hidden_states.shape[2:4]
if height % self.patch_size[1] or width % self.patch_size[2]:
raise ValueError(
f"latent {height}x{width} is not divisible by patch {self.patch_size}"
)
compute_dtype = self.visual_embeddings.in_layer.weight.dtype
time_embed = self.time_embeddings(timestep)
if motion_score is not None and self.use_motion_score:
@@ -397,19 +276,11 @@ class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
attn_mask_meta=attn_meta,
)
visual_embed = gather_seq(visual_embed, shard.orig_len)
return self.out_layer(visual_embed.reshape(*visual_shape, -1), time_embed)
def _check_latent_shape(self, hidden_states: torch.Tensor) -> None:
if hidden_states.ndim != 5 or hidden_states.shape[-1] != self.visual_embed_dim:
raise ValueError(
"Kandinsky6SR expects a [B, T, H, W, "
f"{self.visual_embed_dim}] latent, got {tuple(hidden_states.shape)}"
)
height, width = hidden_states.shape[2:4]
if height % self.patch_size[1] or width % self.patch_size[2]:
raise ValueError(
f"latent {height}x{width} is not divisible by patch {self.patch_size}"
)
return self.out_layer(
visual_embed.reshape(*visual_shape, -1),
time_embed,
compute_dtype=self.out_layer.out_layer.weight.dtype,
)
def post_load_weights(self) -> None:
"""Rebuild RoPE / time-embedding tables that a meta-device init left on meta."""
@@ -25,119 +25,10 @@ from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler
)
class DXPolicy:
def __init__( # noqa: PLR0913
self,
denoising_output: torch.Tensor,
x_t_src: torch.Tensor,
sigma_t_src: torch.Tensor,
segment_size: float | torch.Tensor = 1.0,
shift: float = 1.0,
mode: str = "grid",
eps: float = 1e-4,
) -> None:
self.x_t_src = x_t_src
self.ndim = x_t_src.dim()
self.shift = shift
self.eps = eps
if mode not in ("grid", "polynomial"):
raise ValueError(f"Unknown mode: {mode}")
self.mode = mode
self.sigma_t_src = sigma_t_src.reshape(
*sigma_t_src.size(), *((self.ndim - sigma_t_src.dim()) * [1])
)
self.raw_t_src = self._unwarp_t(self.sigma_t_src)
segment = segment_size
if isinstance(segment, torch.Tensor) and segment.dim() < self.raw_t_src.dim():
segment = segment.reshape(
*segment.size(), *((self.raw_t_src.dim() - segment.dim()) * [1])
)
self.raw_t_dst = (self.raw_t_src - segment).clamp(min=0)
self.segment_size = (self.raw_t_src - self.raw_t_dst).clamp(min=eps)
self.denoising_output_x_0 = self._u_to_x_0(
denoising_output, self.x_t_src, self.sigma_t_src
)
@staticmethod
def _interpolate(x: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
n = x.size(1)
if n < 2: # noqa: PLR2004
return x.squeeze(1)
t = t.clamp(min=0, max=1) * (n - 1)
t0 = t.floor().to(torch.long).clamp(min=0, max=n - 2)
t1 = t0 + 1
indices = torch.stack([t0, t1], dim=1)
values = torch.gather(x, dim=1, index=indices.expand(-1, -1, *x.shape[2:]))
return (t1 - t) * values[:, 0] + (t - t0) * values[:, 1]
def _unwarp_t(self, sigma_t: torch.Tensor) -> torch.Tensor:
return sigma_t / (self.shift + (1 - self.shift) * sigma_t)
@staticmethod
def _u_to_x_0(
denoising_output: torch.Tensor,
x_t: torch.Tensor,
sigma_t: torch.Tensor,
) -> torch.Tensor:
return x_t.unsqueeze(1) - sigma_t.unsqueeze(1) * denoising_output
def pi(self, x_t: torch.Tensor, sigma_t: torch.Tensor) -> torch.Tensor:
sigma_t = sigma_t.reshape(*sigma_t.size(), *((self.ndim - sigma_t.dim()) * [1]))
raw_t = self._unwarp_t(sigma_t)
if self.mode == "grid":
x_0 = self._interpolate(
self.denoising_output_x_0,
(raw_t - self.raw_t_dst) / self.segment_size,
)
else:
p_order = self.denoising_output_x_0.size(1)
diff_t = self.raw_t_src - raw_t
basis = torch.stack([diff_t**i for i in range(p_order)], dim=1)
x_0 = torch.sum(basis * self.denoising_output_x_0, dim=1)
return (x_t - x_0) / sigma_t.clamp(min=self.eps)
def shift_timesteps(t: torch.Tensor, shift: float) -> torch.Tensor:
return shift * t / (1 + (shift - 1) * t)
def policy_rollout_fm( # noqa: PLR0913
x_t_start: torch.Tensor,
sigma_t_start: torch.Tensor,
raw_t_start: torch.Tensor,
raw_t_end: torch.Tensor,
total_substeps: int,
policy: DXPolicy,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
num_batches = x_t_start.size(0)
ndim = x_t_start.dim()
shape = (num_batches, *((ndim - 1) * [1]))
raw_t_start = raw_t_start.reshape(shape)
raw_t_end = raw_t_end.reshape(shape)
sigma_t = sigma_t_start.reshape(shape)
delta_raw_t = raw_t_start - raw_t_end
num_substeps = (delta_raw_t * total_substeps).round().to(torch.long).clamp(min=1)
substep_size = delta_raw_t / num_substeps
max_num_substeps = num_substeps.max()
raw_t = raw_t_start
x_t = x_t_start
for substep_id in range(max_num_substeps.item()):
velocity = policy.pi(x_t, sigma_t)
raw_t_minus = (raw_t - substep_size).clamp(min=0)
sigma_t_minus = shift_timesteps(raw_t_minus, policy.shift)
x_t_minus = x_t + velocity * (sigma_t_minus - sigma_t)
active_mask = num_substeps > substep_id
x_t = torch.where(active_mask, x_t_minus, x_t)
sigma_t = torch.where(active_mask, sigma_t_minus, sigma_t)
raw_t = torch.where(active_mask, raw_t_minus, raw_t)
return x_t, sigma_t, sigma_t.flatten() * 1_000
class PiflowScheduler(FlowMatchEulerDiscreteScheduler):
is_piflow = True
@@ -239,25 +130,41 @@ class PiflowScheduler(FlowMatchEulerDiscreteScheduler):
sigma_src = self.sigmas[step_index].to(device=sample.device)
token_shape = (sample.shape[0], *((sample.ndim - 1) * [1]))
sigma = sigma_src.expand(sample.shape[0]).reshape(token_shape)
segment = (raw_src - raw_dst).expand(sample.shape[0])
policy = DXPolicy(
model_output,
sample,
sigma,
segment,
shift=float(self.config.shift),
mode="grid",
eps=self.eps,
)
updated, _, _ = policy_rollout_fm(
sample,
sigma,
raw_src.expand(sample.shape[0]),
raw_dst.expand(sample.shape[0]),
self.num_policy_substeps,
policy,
)
return updated.to(dtype=sample.dtype)
segment = (raw_src - raw_dst).expand(sample.shape[0]).reshape(token_shape)
shift = float(self.config.shift)
policy_src = sigma / (shift + (1 - shift) * sigma)
policy_dst = (policy_src - segment).clamp(min=0)
policy_segment = (policy_src - policy_dst).clamp(min=self.eps)
x0_grid = sample.unsqueeze(1) - sigma.unsqueeze(1) * model_output
raw_t = raw_src.expand(sample.shape[0]).reshape(token_shape)
delta = raw_t - raw_dst.expand(sample.shape[0]).reshape(token_shape)
substeps = (delta * self.num_policy_substeps).round().long().clamp(min=1)
substep_size = delta / substeps
for index in range(substeps.max().item()):
# the checkpoint predicts x0 on a grid in unwarped time
policy_t = sigma / (shift + (1 - shift) * sigma)
t = ((policy_t - policy_dst) / policy_segment).clamp(0, 1) * (
self.n_grid - 1
)
t0 = t.floor().long().clamp(0, self.n_grid - 2)
t1 = t0 + 1
indices = torch.stack([t0, t1], dim=1)
values = torch.gather(
x0_grid, 1, indices.expand(-1, -1, *x0_grid.shape[2:])
)
x0 = (t1 - t) * values[:, 0] + (t - t0) * values[:, 1]
velocity = (sample - x0) / sigma.clamp(min=self.eps)
# release grid interpolation temporaries before the state update/next gather
del values, x0, indices, t, t0, t1, policy_t
next_raw = (raw_t - substep_size).clamp(min=0)
next_sigma = shift_timesteps(next_raw, shift)
updated = sample + velocity * (next_sigma - sigma)
active = substeps > index
sample = torch.where(active, updated, sample)
sigma = torch.where(active, next_sigma, sigma)
raw_t = torch.where(active, next_raw, raw_t)
return sample
def _step_index_for(self, timestep: torch.Tensor | float) -> int:
if self.step_index is None:
@@ -48,9 +48,7 @@ class RMSNorm(nn.Module):
self.gamma = nn.Parameter(torch.ones(dim, 1, 1, 1))
def forward(self, x: Tensor) -> Tensor:
# Upcast to float32 around the normalize (matches the Diffusers reference's
# Kandinsky6SRLatentUpscalerRMSNorm): without it, bf16 activations drift from the
# reference across the many ResidualBlock/X2Branch calls that route through this norm.
# preserve the reference's fp32 reduction followed by bf16 scale/gain
normalized = F.normalize(x.float(), dim=1).to(x.dtype)
return normalized * self.scale * self.gamma
@@ -96,31 +94,8 @@ class ResidualBlock(nn.Module):
return self.shortcut(x) + h
def nearest_2x(x: Tensor) -> Tensor:
"""Nearest-neighbour 2x resize of H and W of ``[B, C, T, H, W]`` (T folded into the batch)."""
b, c, t, h, w = x.shape
x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w)
x = F.interpolate(x, scale_factor=2, mode="nearest")
return x.reshape(b, t, c, 2 * h, 2 * w).permute(0, 2, 1, 3, 4)
class PXSUpsample(nn.Module):
"""2x spatial upsample ``linear(up + spatial_conv(up))`` with ``up = nearest_2x(x)``."""
def __init__(self, channels: int) -> None:
super().__init__()
self.spatial_conv = nn.Conv3d(
channels, channels, kernel_size=(1, 3, 3), padding=(0, 1, 1)
)
self.linear = nn.Conv3d(channels, channels, kernel_size=1)
def forward(self, x: Tensor) -> Tensor:
up = nearest_2x(x)
return self.linear(up + self.spatial_conv(up))
class X2Finisher(nn.Module):
"""The ``PXSUpsample`` conv pair applied on the unchanged grid (no resize)."""
"""Residual spatial convolution and channel projection, without resizing."""
def __init__(self, channels: int) -> None:
super().__init__()
@@ -133,19 +108,21 @@ class X2Finisher(nn.Module):
return self.linear(x + self.spatial_conv(x))
class PXSUpsample(X2Finisher):
"""Apply the same projection after nearest-neighbor 2x spatial resizing."""
def forward(self, x: Tensor) -> Tensor:
b, c, t, h, w = x.shape
x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w)
x = F.interpolate(x, scale_factor=2, mode="nearest")
x = x.reshape(b, t, c, 2 * h, 2 * w).permute(0, 2, 1, 3, 4)
return super().forward(x)
def _stem(in_channels: int, out_channels: int) -> nn.Sequential:
return nn.Sequential(ReplicateTimeConv3d(in_channels, out_channels, kernel_size=3))
def _mid_output_head(channels: int, out_channels: int) -> nn.Sequential:
"""Unused training head whose checkpoint layout remains positional."""
return nn.Sequential(
RMSNorm(channels),
nn.SiLU(),
ReplicateTimeConv3d(channels, out_channels, kernel_size=3),
)
class Kandinsky6SRLatentUpscalerOutputHead(nn.Module):
"""Named ``norm -> activation -> conv`` projection used by current checkpoints."""
@@ -182,18 +159,6 @@ def _run_blocks(blocks: nn.Sequential, x: Tensor, zq: Tensor) -> Tensor:
return x
def _second_stage(
upsample: nn.Module,
blocks: nn.Sequential,
head: Kandinsky6SRLatentUpscalerOutputHead,
x: Tensor,
zq: Tensor,
) -> Tensor:
"""``upsample -> blocks -> head``, with the head's first (norm) layer conditioned on ``zq``."""
x = _run_blocks(blocks, upsample(x), zq)
return head(x, zq)
class X2Branch(nn.Module):
"""Weights exclusive to the x2 path."""
@@ -213,7 +178,8 @@ class X2Branch(nn.Module):
x = _run_blocks(self.adapter, x, zq)
x = self.finisher(x)
x = _run_blocks(self.mid_blocks, x, zq)
return _second_stage(self.upsample, self.blocks, self.output_proj, x, zq)
x = _run_blocks(self.blocks, self.upsample(x), zq)
return self.output_proj(x, zq)
class Kandinsky6SRLatentUpscaler(nn.Module):
@@ -227,7 +193,9 @@ class Kandinsky6SRLatentUpscaler(nn.Module):
er = config.expand_ratio
self.input_proj = _stem(c, w1)
# The 2x deep-supervision head of training: part of the checkpoint, never run at inference.
self.mid_output_head = _mid_output_head(w2, c)
self.mid_output_head = nn.Sequential(
RMSNorm(w2), nn.SiLU(), ReplicateTimeConv3d(w2, c, kernel_size=3)
)
self.output_proj = Kandinsky6SRLatentUpscalerOutputHead(w3, c, zq_dim=c)
self.upsample_1 = PXSUpsample(w1)
self.upsample_2 = PXSUpsample(w2)
@@ -254,7 +222,8 @@ class Kandinsky6SRLatentUpscaler(nn.Module):
)
x = _run_blocks(self.pre_blocks, self.input_proj(z), zq)
x = _run_blocks(self.mid_blocks, self.upsample_1(x), zq)
return _second_stage(self.upsample_2, self.post_blocks, self.output_proj, x, zq)
x = _run_blocks(self.post_blocks, self.upsample_2(x), zq)
return self.output_proj(x, zq)
class Kandinsky6SRLatentUpscalerBank(nn.Module, LayerwiseOffloadableModuleMixin):
@@ -48,24 +48,13 @@ class Kandinsky6AudioVAE(nn.Module, LayerwiseOffloadableModuleMixin):
# audio latent stride; frame-count alignment is owned by the pipeline config
self.downsample_factor = 1024
self.mel_converter = MelConverter(
sampling_rate=44100,
n_fft=2048,
num_mels=128,
hop_size=512,
win_size=2048,
fmin=0,
fmax=22050,
)
self.mel_converter = MelConverter()
# checkpoint weights include the encoder regardless of need_vae_encoder
self.vae = MMAudioVAE(mode=arch.mode, need_encoder=True)
# normalize random initialization before loading; checkpoint weights are
# already normalized and must never be renormalized
self.vae.remove_weight_norm()
self.vocoder = BigVGANV2(dict(arch.vocoder_config))
# strip PyTorch weight_norm before loading plain .weight checkpoint keys;
# this differs from MMAudio's RMS renormalization above
self.vocoder.remove_weight_norm()
# resolution containers are not called; hooks belong on their blocks
self.layer_names = (
@@ -1,350 +1,141 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2024 NVIDIA CORPORATION.
# Adapted from NVIDIA BigVGAN-v2 (MIT license) via FastVideo's
# fastvideo/models/audio/bigvgan.py. The CUDA activation kernel is
# intentionally excluded; this module uses the portable PyTorch path for
# deterministic loading and parity across platforms.
# Adapted from NVIDIA BigVGAN-v2 (MIT) via FastVideo.
# Original license terms are retained in THIRD_PARTY_LICENSES.
#
# This top-level generator class (BigVGANV2) and its Snake/SnakeBeta/
# AMPBlock1/AMPBlock2 building blocks are freshly ported here rather than
# reused from this codebase's sibling ``minimax_h3_audio_vae/bigvgan.py`` --
# that module's ``BigVGAN``/``AMPBlock1`` only support resblock="1" and
# activation="snakebeta" (MiniMax's own DAC-vocoder config never needs more),
# while this implementation supports either resblock variant or activation. The
# alias-free resampling primitives (LowPassFilter1d/UpSample1d/DownSample1d/
# Activation1d) ARE architecture-agnostic DSP building blocks shared with
# that same sibling module, though -- both import ``Activation1d`` from
# ``runtime/models/vaes/alias_free.py`` (sibling of this package, alongside
# ``common.py``) instead of each keeping its own copy.
from __future__ import annotations
"""BigVGAN-v2 with the plain weights stored in Kandinsky audio checkpoints."""
from types import SimpleNamespace
from typing import Any
import torch
import torch.nn as nn
from torch.nn import Conv1d, ConvTranspose1d
from torch.nn.utils.parametrizations import weight_norm
from torch.nn.utils.parametrize import remove_parametrizations
from torch import nn
from sglang.multimodal_gen.runtime.models.vaes.alias_free import Activation1d
class AttrDict(dict):
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.__dict__ = self
def get_padding(kernel_size: int, dilation: int = 1) -> int:
return int((kernel_size * dilation - dilation) / 2)
def init_weights(module: nn.Module, mean: float = 0.0, std: float = 0.01) -> None:
if "Conv" in module.__class__.__name__:
module.weight.data.normal_(mean, std)
class Snake(nn.Module):
def __init__(
self,
in_features: int,
alpha: float = 1.0,
alpha_trainable: bool = True,
alpha_logscale: bool = False,
) -> None:
def __init__(self, channels: int, *, logscale: bool, separate_beta: bool):
super().__init__()
self.in_features = in_features
self.alpha_logscale = alpha_logscale
initial = (
torch.zeros(in_features) * alpha
if alpha_logscale
else torch.ones(in_features) * alpha
)
self.alpha = nn.Parameter(initial, requires_grad=alpha_trainable)
self.no_div_by_zero = 1e-9
initial = torch.zeros(channels) if logscale else torch.ones(channels)
self.alpha = nn.Parameter(initial)
self.beta = nn.Parameter(initial.clone()) if separate_beta else None
self.logscale = logscale
def forward(self, x: torch.Tensor) -> torch.Tensor:
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
if self.alpha_logscale:
alpha = torch.exp(alpha)
return x + (1.0 / (alpha + self.no_div_by_zero)) * torch.pow(
torch.sin(x * alpha), 2
alpha = self.alpha[None, :, None]
beta = alpha if self.beta is None else self.beta[None, :, None]
if self.logscale:
alpha = alpha.exp()
beta = alpha if self.beta is None else beta.exp()
return x + (1.0 / (beta + 1e-9)) * torch.sin(x * alpha).pow(2)
def _activation(config: SimpleNamespace, channels: int) -> Activation1d:
if config.activation not in ("snake", "snakebeta"):
raise ValueError(f"Unsupported BigVGAN activation: {config.activation}")
return Activation1d(
Snake(
channels,
logscale=config.snake_logscale,
separate_beta=config.activation == "snakebeta",
)
)
class SnakeBeta(nn.Module):
def __init__(
self,
in_features: int,
alpha: float = 1.0,
alpha_trainable: bool = True,
alpha_logscale: bool = False,
) -> None:
class AMPBlock(nn.Module):
def __init__(self, config, channels: int, kernel: int, dilations):
super().__init__()
self.in_features = in_features
self.alpha_logscale = alpha_logscale
initial = (
torch.zeros(in_features) * alpha
if alpha_logscale
else torch.ones(in_features) * alpha
self.paired = config.resblock == "1"
convs = nn.ModuleList(
nn.Conv1d(
channels,
channels,
kernel,
dilation=rate,
padding=(kernel - 1) * rate // 2,
)
for rate in dilations
)
self.alpha = nn.Parameter(initial.clone(), requires_grad=alpha_trainable)
self.beta = nn.Parameter(initial.clone(), requires_grad=alpha_trainable)
self.no_div_by_zero = 1e-9
def forward(self, x: torch.Tensor) -> torch.Tensor:
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
beta = self.beta.unsqueeze(0).unsqueeze(-1)
if self.alpha_logscale:
alpha = torch.exp(alpha)
beta = torch.exp(beta)
return x + (1.0 / (beta + self.no_div_by_zero)) * torch.pow(
torch.sin(x * alpha), 2
)
def _activation(name: str, channels: int, logscale: bool) -> Activation1d:
if name == "snake":
activation = Snake(channels, alpha_logscale=logscale)
elif name == "snakebeta":
activation = SnakeBeta(channels, alpha_logscale=logscale)
else:
raise ValueError(f"Unsupported BigVGAN activation: {name}")
return Activation1d(activation)
class AMPBlock1(nn.Module):
def __init__(
self,
config: AttrDict,
channels: int,
kernel_size: int = 3,
dilation: tuple[int, ...] = (1, 3, 5),
activation: str = "snake",
) -> None:
super().__init__()
self.convs1 = nn.ModuleList(
[
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
stride=1,
dilation=rate,
padding=get_padding(kernel_size, rate),
)
)
for rate in dilation
]
)
self.convs1.apply(init_weights)
self.convs2 = nn.ModuleList(
[
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
stride=1,
dilation=1,
padding=get_padding(kernel_size, 1),
)
)
for _ in dilation
]
)
self.convs2.apply(init_weights)
# retain both published state-dict layouts
if self.paired:
self.convs1 = convs
self.convs2 = nn.ModuleList(
nn.Conv1d(channels, channels, kernel, padding=(kernel - 1) // 2)
for _ in dilations
)
else:
self.convs = convs
self.activations = nn.ModuleList(
[
_activation(activation, channels, config.snake_logscale)
for _ in range(len(self.convs1) + len(self.convs2))
]
_activation(config, channels)
for _ in range(len(dilations) * (2 if self.paired else 1))
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
acts1, acts2 = self.activations[::2], self.activations[1::2]
for conv1, conv2, act1, act2 in zip(
self.convs1, self.convs2, acts1, acts2, strict=True
):
residual = conv2(act2(conv1(act1(x))))
x = residual + x
if self.paired:
for index, (conv1, conv2) in enumerate(zip(self.convs1, self.convs2)):
hidden = conv1(self.activations[2 * index](x))
x = conv2(self.activations[2 * index + 1](hidden)) + x
else:
for conv, activation in zip(self.convs, self.activations):
x = conv(activation(x)) + x
return x
def remove_weight_norm(self) -> None:
for layer in self.convs1:
remove_parametrizations(layer, "weight")
for layer in self.convs2:
remove_parametrizations(layer, "weight")
class AMPBlock2(nn.Module):
def __init__(
self,
config: AttrDict,
channels: int,
kernel_size: int = 3,
dilation: tuple[int, ...] = (1, 3, 5),
activation: str = "snake",
) -> None:
super().__init__()
self.convs = nn.ModuleList(
[
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
stride=1,
dilation=rate,
padding=get_padding(kernel_size, rate),
)
)
for rate in dilation
]
)
self.convs.apply(init_weights)
self.activations = nn.ModuleList(
[
_activation(activation, channels, config.snake_logscale)
for _ in self.convs
]
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
for conv, activation in zip(self.convs, self.activations, strict=True):
x = conv(activation(x)) + x
return x
def remove_weight_norm(self) -> None:
for layer in self.convs:
remove_parametrizations(layer, "weight")
class BigVGANV2(nn.Module):
"""BigVGAN-v2 using checkpoint vocoder_config and the portable PyTorch path."""
def __init__(self, config: dict[str, Any]) -> None:
super().__init__()
config = dict(config)
config.pop("_class_name", None)
config.pop("architectures", None)
weight_norm_removed = bool(config.pop("weight_norm_removed", False))
self.config = AttrDict(config)
if self.config.get("use_cuda_kernel", False):
if config.get("use_cuda_kernel", False):
raise ValueError(
"Kandinsky6 BigVGANV2 supports only the portable PyTorch path"
)
self.config["use_cuda_kernel"] = False
self.num_kernels = len(self.config.resblock_kernel_sizes)
self.num_upsamples = len(self.config.upsample_rates)
self.conv_pre = weight_norm(
Conv1d(
self.config.num_mels,
self.config.upsample_initial_channel,
7,
1,
padding=3,
)
)
if self.config.resblock == "1":
block_class = AMPBlock1
elif self.config.resblock == "2":
block_class = AMPBlock2
else:
raise ValueError(f"Unsupported BigVGAN resblock: {self.config.resblock}")
h = SimpleNamespace(**config)
if h.resblock not in ("1", "2"):
raise ValueError(f"Unsupported BigVGAN resblock: {h.resblock}")
self.num_kernels = len(h.resblock_kernel_sizes)
self.conv_pre = nn.Conv1d(h.num_mels, h.upsample_initial_channel, 7, padding=3)
self.ups = nn.ModuleList()
for index, (rate, kernel) in enumerate(
zip(
self.config.upsample_rates,
self.config.upsample_kernel_sizes,
strict=True,
)
):
self.resblocks = nn.ModuleList()
channels = h.upsample_initial_channel
for rate, kernel in zip(h.upsample_rates, h.upsample_kernel_sizes, strict=True):
self.ups.append(
nn.ModuleList(
[
weight_norm(
ConvTranspose1d(
self.config.upsample_initial_channel // (2**index),
self.config.upsample_initial_channel
// (2 ** (index + 1)),
kernel,
rate,
padding=(kernel - rate) // 2,
)
nn.ConvTranspose1d(
channels,
channels // 2,
kernel,
rate,
padding=(kernel - rate) // 2,
)
]
)
)
self.resblocks = nn.ModuleList()
for index in range(len(self.ups)):
channels = self.config.upsample_initial_channel // (2 ** (index + 1))
for kernel, dilation in zip(
self.config.resblock_kernel_sizes,
self.config.resblock_dilation_sizes,
strict=True,
):
self.resblocks.append(
block_class(
self.config,
channels,
kernel,
tuple(dilation),
activation=self.config.activation,
)
channels //= 2
self.resblocks.extend(
AMPBlock(h, channels, kernel, dilations)
for kernel, dilations in zip(
h.resblock_kernel_sizes, h.resblock_dilation_sizes, strict=True
)
channels = self.config.upsample_initial_channel // (2 ** len(self.ups))
self.activation_post = _activation(
self.config.activation, channels, self.config.snake_logscale
)
self.activation_post = _activation(h, channels)
self.conv_post = nn.Conv1d(
channels, 1, 7, padding=3, bias=config.get("use_bias_at_final", True)
)
self.use_bias_at_final = self.config.get("use_bias_at_final", True)
self.conv_post = weight_norm(
Conv1d(channels, 1, 7, 1, padding=3, bias=self.use_bias_at_final)
)
for upsampler in self.ups:
upsampler.apply(init_weights)
self.conv_post.apply(init_weights)
self.use_tanh_at_final = self.config.get("use_tanh_at_final", True)
if weight_norm_removed:
self.remove_weight_norm()
self.use_tanh_at_final = config.get("use_tanh_at_final", True)
def forward(self, mel: torch.Tensor) -> torch.Tensor:
hidden = self.conv_pre(mel)
for index in range(self.num_upsamples):
for upsampler in self.ups[index]:
for index, upsamplers in enumerate(self.ups):
for upsampler in upsamplers:
hidden = upsampler(hidden)
accumulated = None
for kernel in range(self.num_kernels):
block_output = self.resblocks[index * self.num_kernels + kernel](hidden)
output = self.resblocks[index * self.num_kernels + kernel](hidden)
if accumulated is None:
accumulated = block_output
accumulated = output
else:
# Preserve BigVGAN's published sequential accumulation
# order. Tree reduction drifts through later nonlinear
# upsampling stages with the full checkpoint.
accumulated += block_output
assert accumulated is not None
# preserve sequential accumulation through nonlinear upsampling
accumulated += output
hidden = accumulated / self.num_kernels
hidden = self.conv_post(self.activation_post(hidden))
if self.use_tanh_at_final:
return torch.tanh(hidden)
return torch.clamp(hidden, min=-1.0, max=1.0)
def remove_weight_norm(self) -> None:
try:
for upsamplers in self.ups:
for upsampler in upsamplers:
remove_parametrizations(upsampler, "weight")
for block in self.resblocks:
block.remove_weight_norm()
remove_parametrizations(self.conv_pre, "weight")
remove_parametrizations(self.conv_post, "weight")
except ValueError:
# Idempotent for pipeline setup and converted checkpoints.
return
return torch.tanh(hidden) if self.use_tanh_at_final else hidden.clamp(-1.0, 1.0)
@@ -1,25 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
#
# Adapted from MMAudio (https://github.com/hkchengrex/MMAudio), MIT License,
# Copyright (c) 2024 Sony Research Inc., via FastVideo's
# fastvideo/models/audio/mmaudio_vae.py. This is the mel-spectrogram
# <-> latent 1D-conv VAE codec nested at ``Kandinsky6AudioVAE.vae``
# (see kandinsky6_audio.py in the parent directory).
# Copyright (c) 2024 Sony Research Inc., via FastVideo.
# Original license terms are retained in THIRD_PARTY_LICENSES.
#
# Ported verbatim except: FastVideo's ``MMAudioVAE.load_weights()`` (and its
# ``fastvideo.models.loader.weight_utils.default_weight_loader`` import) is
# dropped -- sglang's generic VAELoader never calls a per-submodule
# ``load_weights()`` hook, it does a single plain ``nn.Module.load_state_dict``
# over the whole ``Kandinsky6AudioVAE`` tree, so that method would be dead code
# here.
"""Native 1D audio VAE used by MMAudio (mel-spectrogram <-> latent codec)."""
from __future__ import annotations
import logging
import math
from typing import Any
from collections import OrderedDict
import torch
import torch.nn as nn
@@ -29,435 +19,6 @@ from einops import rearrange
logger = logging.getLogger(__name__)
DATA_MEAN_80D = [
-1.6058,
-1.3676,
-1.2520,
-1.2453,
-1.2078,
-1.2224,
-1.2419,
-1.2439,
-1.2922,
-1.2927,
-1.3170,
-1.3543,
-1.3401,
-1.3836,
-1.3907,
-1.3912,
-1.4313,
-1.4152,
-1.4527,
-1.4728,
-1.4568,
-1.5101,
-1.5051,
-1.5172,
-1.5623,
-1.5373,
-1.5746,
-1.5687,
-1.6032,
-1.6131,
-1.6081,
-1.6331,
-1.6489,
-1.6489,
-1.6700,
-1.6738,
-1.6953,
-1.6969,
-1.7048,
-1.7280,
-1.7361,
-1.7495,
-1.7658,
-1.7814,
-1.7889,
-1.8064,
-1.8221,
-1.8377,
-1.8417,
-1.8643,
-1.8857,
-1.8929,
-1.9173,
-1.9379,
-1.9531,
-1.9673,
-1.9824,
-2.0042,
-2.0215,
-2.0436,
-2.0766,
-2.1064,
-2.1418,
-2.1855,
-2.2319,
-2.2767,
-2.3161,
-2.3572,
-2.3954,
-2.4282,
-2.4659,
-2.5072,
-2.5552,
-2.6074,
-2.6584,
-2.7107,
-2.7634,
-2.8266,
-2.8981,
-2.9673,
]
DATA_STD_80D = [
1.0291,
1.0411,
1.0043,
0.9820,
0.9677,
0.9543,
0.9450,
0.9392,
0.9343,
0.9297,
0.9276,
0.9263,
0.9242,
0.9254,
0.9232,
0.9281,
0.9263,
0.9315,
0.9274,
0.9247,
0.9277,
0.9199,
0.9188,
0.9194,
0.9160,
0.9161,
0.9146,
0.9161,
0.9100,
0.9095,
0.9145,
0.9076,
0.9066,
0.9095,
0.9032,
0.9043,
0.9038,
0.9011,
0.9019,
0.9010,
0.8984,
0.8983,
0.8986,
0.8961,
0.8962,
0.8978,
0.8962,
0.8973,
0.8993,
0.8976,
0.8995,
0.9016,
0.8982,
0.8972,
0.8974,
0.8949,
0.8940,
0.8947,
0.8936,
0.8939,
0.8951,
0.8956,
0.9017,
0.9167,
0.9436,
0.9690,
1.0003,
1.0225,
1.0381,
1.0491,
1.0545,
1.0604,
1.0761,
1.0929,
1.1089,
1.1196,
1.1176,
1.1156,
1.1117,
1.1070,
]
DATA_MEAN_128D = [
-3.3462,
-2.6723,
-2.4893,
-2.3143,
-2.2664,
-2.3317,
-2.1802,
-2.4006,
-2.2357,
-2.4597,
-2.3717,
-2.4690,
-2.5142,
-2.4919,
-2.6610,
-2.5047,
-2.7483,
-2.5926,
-2.7462,
-2.7033,
-2.7386,
-2.8112,
-2.7502,
-2.9594,
-2.7473,
-3.0035,
-2.8891,
-2.9922,
-2.9856,
-3.0157,
-3.1191,
-2.9893,
-3.1718,
-3.0745,
-3.1879,
-3.2310,
-3.1424,
-3.2296,
-3.2791,
-3.2782,
-3.2756,
-3.3134,
-3.3509,
-3.3750,
-3.3951,
-3.3698,
-3.4505,
-3.4509,
-3.5089,
-3.4647,
-3.5536,
-3.5788,
-3.5867,
-3.6036,
-3.6400,
-3.6747,
-3.7072,
-3.7279,
-3.7283,
-3.7795,
-3.8259,
-3.8447,
-3.8663,
-3.9182,
-3.9605,
-3.9861,
-4.0105,
-4.0373,
-4.0762,
-4.1121,
-4.1488,
-4.1874,
-4.2461,
-4.3170,
-4.3639,
-4.4452,
-4.5282,
-4.6297,
-4.7019,
-4.7960,
-4.8700,
-4.9507,
-5.0303,
-5.0866,
-5.1634,
-5.2342,
-5.3242,
-5.4053,
-5.4927,
-5.5712,
-5.6464,
-5.7052,
-5.7619,
-5.8410,
-5.9188,
-6.0103,
-6.0955,
-6.1673,
-6.2362,
-6.3120,
-6.3926,
-6.4797,
-6.5565,
-6.6511,
-6.8130,
-6.9961,
-7.1275,
-7.2457,
-7.3576,
-7.4663,
-7.6136,
-7.7469,
-7.8815,
-8.0132,
-8.1515,
-8.3071,
-8.4722,
-8.7418,
-9.3975,
-9.6628,
-9.7671,
-9.8863,
-9.9992,
-10.0860,
-10.1709,
-10.5418,
-11.2795,
-11.3861,
]
DATA_STD_128D = [
2.3804,
2.4368,
2.3772,
2.3145,
2.2803,
2.2510,
2.2316,
2.2083,
2.1996,
2.1835,
2.1769,
2.1659,
2.1631,
2.1618,
2.1540,
2.1606,
2.1571,
2.1567,
2.1612,
2.1579,
2.1679,
2.1683,
2.1634,
2.1557,
2.1668,
2.1518,
2.1415,
2.1449,
2.1406,
2.1350,
2.1313,
2.1415,
2.1281,
2.1352,
2.1219,
2.1182,
2.1327,
2.1195,
2.1137,
2.1080,
2.1179,
2.1036,
2.1087,
2.1036,
2.1015,
2.1068,
2.0975,
2.0991,
2.0902,
2.1015,
2.0857,
2.0920,
2.0893,
2.0897,
2.0910,
2.0881,
2.0925,
2.0873,
2.0960,
2.0900,
2.0957,
2.0958,
2.0978,
2.0936,
2.0886,
2.0905,
2.0845,
2.0855,
2.0796,
2.0840,
2.0813,
2.0817,
2.0838,
2.0840,
2.0917,
2.1061,
2.1431,
2.1976,
2.2482,
2.3055,
2.3700,
2.4088,
2.4372,
2.4609,
2.4731,
2.4847,
2.5072,
2.5451,
2.5772,
2.6147,
2.6529,
2.6596,
2.6645,
2.6726,
2.6803,
2.6812,
2.6899,
2.6916,
2.6931,
2.6998,
2.7062,
2.7262,
2.7222,
2.7158,
2.7041,
2.7485,
2.7491,
2.7451,
2.7485,
2.7233,
2.7297,
2.7233,
2.7145,
2.6958,
2.6788,
2.6439,
2.6007,
2.4786,
2.2469,
2.1877,
2.1392,
2.0717,
2.0107,
1.9676,
1.9140,
1.7102,
0.9101,
0.7164,
]
_NORMALIZATION_EPSILON = 1e-4
_SILU_DIVISOR = 0.596
_RESIDUAL_BLEND = 0.3
@@ -516,33 +77,22 @@ class ResnetBlock1D(nn.Module):
*,
in_dim: int,
out_dim: int | None = None,
conv_shortcut: bool = False,
kernel_size: int = 3,
use_norm: bool = True,
) -> None:
super().__init__()
self.in_dim = in_dim
self.out_dim = in_dim if out_dim is None else out_dim
self.use_conv_shortcut = conv_shortcut
self.use_norm = use_norm
self.conv1 = _conv1d(in_dim, self.out_dim, kernel_size)
self.conv2 = _conv1d(self.out_dim, self.out_dim, kernel_size)
if self.in_dim != self.out_dim:
if conv_shortcut:
self.conv_shortcut = _conv1d(in_dim, self.out_dim, kernel_size)
else:
self.nin_shortcut = _conv1d(in_dim, self.out_dim, 1)
self.nin_shortcut = _conv1d(in_dim, self.out_dim, 1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.use_norm:
x = _rms_normalize(x, dim=1)
x = _rms_normalize(x, dim=1)
hidden = self.conv1(F.silu(x) / _SILU_DIVISOR)
hidden = self.conv2(F.silu(hidden) / _SILU_DIVISOR)
if self.in_dim != self.out_dim:
shortcut = (
self.conv_shortcut if self.use_conv_shortcut else self.nin_shortcut
)
x = shortcut(x)
x = self.nin_shortcut(x)
return torch.lerp(x, hidden, _RESIDUAL_BLEND) / _RESIDUAL_DIVISOR
@@ -566,30 +116,60 @@ class AttnBlock1D(nn.Module):
class Upsample1D(nn.Module):
def __init__(self, in_channels: int, with_conv: bool) -> None:
def __init__(self, in_channels: int) -> None:
super().__init__()
self.with_conv = with_conv
if with_conv:
self.conv = _conv1d(in_channels, in_channels, 3)
self.conv = _conv1d(in_channels, in_channels, 3)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = F.interpolate(x, scale_factor=2.0, mode="nearest-exact")
return self.conv(x) if self.with_conv else x
return self.conv(x)
class Downsample1D(nn.Module):
def __init__(self, in_channels: int, with_conv: bool) -> None:
def __init__(self, in_channels: int) -> None:
super().__init__()
self.with_conv = with_conv
if with_conv:
self.conv1 = _conv1d(in_channels, in_channels, 1)
self.conv2 = _conv1d(in_channels, in_channels, 1)
self.conv1 = _conv1d(in_channels, in_channels, 1)
self.conv2 = _conv1d(in_channels, in_channels, 1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.with_conv:
x = self.conv1(x)
x = F.avg_pool1d(x, kernel_size=2, stride=2)
return self.conv2(x) if self.with_conv else x
return self.conv2(F.avg_pool1d(self.conv1(x), kernel_size=2, stride=2))
def _mid_block(channels: int, kernel_size: int = 3) -> nn.Sequential:
return nn.Sequential(
OrderedDict(
block_1=ResnetBlock1D(in_dim=channels, kernel_size=kernel_size),
attn_1=AttnBlock1D(channels),
block_2=ResnetBlock1D(in_dim=channels, kernel_size=kernel_size),
)
)
class ResolutionBlock1D(nn.Module):
"""Residual/attention stack with checkpoint-compatible block and attn names."""
def __init__(self, in_dim, out_dim, count, attention, kernel_size=3):
super().__init__()
self.block = nn.ModuleList()
self.attn = nn.ModuleList()
for index in range(count):
self.block.append(
ResnetBlock1D(
in_dim=in_dim if index == 0 else out_dim,
out_dim=out_dim,
kernel_size=kernel_size,
)
)
if attention:
self.attn.append(AttnBlock1D(out_dim))
def forward(self, x, clip_act):
for index, block in enumerate(self.block):
x = block(x)
if self.attn:
x = self.attn[index](x)
x = x.clamp(-clip_act, clip_act)
return x
class Encoder1D(nn.Module):
@@ -603,73 +183,38 @@ class Encoder1D(nn.Module):
down_layers: list[int],
in_dim: int,
embed_dim: int,
resamp_with_conv: bool = True,
double_z: bool = True,
kernel_size: int = 3,
clip_act: float = 256.0,
) -> None:
super().__init__()
self.dim = dim
self.num_layers = len(ch_mult)
self.num_res_blocks = num_res_blocks
self.in_channels = in_dim
self.clip_act = clip_act
self.down_layers = down_layers
self.attn_layers = attn_layers
self.conv_in = _conv1d(in_dim, dim, kernel_size)
in_ch_mult = (1,) + ch_mult
self.in_ch_mult = in_ch_mult
self.down = nn.ModuleList()
for level in range(self.num_layers):
block = nn.ModuleList()
attn = nn.ModuleList()
block_in = dim * in_ch_mult[level]
block_out = dim * ch_mult[level]
for _ in range(num_res_blocks):
block.append(
ResnetBlock1D(
in_dim=block_in,
out_dim=block_out,
kernel_size=kernel_size,
use_norm=True,
)
)
block_in = block_out
if level in attn_layers:
attn.append(AttnBlock1D(block_in))
down = nn.Module()
down.block = block
down.attn = attn
down = ResolutionBlock1D(
block_in, block_out, num_res_blocks, level in attn_layers, kernel_size
)
if level in down_layers:
down.downsample = Downsample1D(block_in, resamp_with_conv)
down.downsample = Downsample1D(block_out)
self.down.append(down)
self.mid = nn.Module()
self.mid.block_1 = ResnetBlock1D(
in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True
)
self.mid.attn_1 = AttnBlock1D(block_in)
self.mid.block_2 = ResnetBlock1D(
in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True
)
output_dim = 2 * embed_dim if double_z else embed_dim
self.conv_out = _conv1d(block_in, output_dim, kernel_size)
self.mid = _mid_block(block_out, kernel_size)
self.conv_out = _conv1d(block_out, 2 * embed_dim, kernel_size)
self.learnable_gain = nn.Parameter(torch.zeros([]))
def forward(self, x: torch.Tensor) -> torch.Tensor:
states = [self.conv_in(x)]
for level in range(self.num_layers):
for block_index in range(self.num_res_blocks):
hidden = self.down[level].block[block_index](states[-1])
if len(self.down[level].attn) > 0:
hidden = self.down[level].attn[block_index](hidden)
states.append(hidden.clamp(-self.clip_act, self.clip_act))
hidden = self.conv_in(x)
for level, down in enumerate(self.down):
hidden = down(hidden, self.clip_act)
if level in self.down_layers:
states.append(self.down[level].downsample(states[-1]))
hidden = self.mid.block_1(states[-1])
hidden = self.mid.attn_1(hidden)
hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act)
hidden = down.downsample(hidden)
hidden = self.mid(hidden).clamp(-self.clip_act, self.clip_act)
return _conv1d_with_gain(
self.conv_out, F.silu(hidden) / _SILU_DIVISOR, self.learnable_gain + 1
)
@@ -685,47 +230,27 @@ class Decoder1D(nn.Module):
num_res_blocks: int,
attn_layers: list[int],
down_layers: list[int],
in_dim: int,
embed_dim: int,
kernel_size: int = 3,
resamp_with_conv: bool = True,
clip_act: float = 256.0,
) -> None:
super().__init__()
self.ch = dim
self.num_layers = len(ch_mult)
self.num_res_blocks = num_res_blocks
self.in_channels = in_dim
self.clip_act = clip_act
self.down_layers = [level + 1 for level in down_layers]
block_in = dim * ch_mult[-1]
self.conv_in = _conv1d(embed_dim, block_in, kernel_size)
self.mid = nn.Module()
self.mid.block_1 = ResnetBlock1D(
in_dim=block_in, out_dim=block_in, use_norm=True
)
self.mid.attn_1 = AttnBlock1D(block_in)
self.mid.block_2 = ResnetBlock1D(
in_dim=block_in, out_dim=block_in, use_norm=True
)
self.mid = _mid_block(block_in)
self.up = nn.ModuleList()
for level in reversed(range(self.num_layers)):
block = nn.ModuleList()
attn = nn.ModuleList()
block_out = dim * ch_mult[level]
for _ in range(num_res_blocks + 1):
block.append(
ResnetBlock1D(in_dim=block_in, out_dim=block_out, use_norm=True)
)
block_in = block_out
if level in attn_layers:
attn.append(AttnBlock1D(block_in))
up = nn.Module()
up.block = block
up.attn = attn
up = ResolutionBlock1D(
block_in, block_out, num_res_blocks + 1, level in attn_layers
)
block_in = block_out
if level in self.down_layers:
up.upsample = Upsample1D(block_in, resamp_with_conv)
up.upsample = Upsample1D(block_in)
self.up.insert(0, up)
self.conv_out = _conv1d(block_in, out_dim, kernel_size)
@@ -733,15 +258,9 @@ class Decoder1D(nn.Module):
def forward(self, z: torch.Tensor) -> torch.Tensor:
hidden = self.conv_in(z)
hidden = self.mid.block_1(hidden)
hidden = self.mid.attn_1(hidden)
hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act)
hidden = self.mid(hidden).clamp(-self.clip_act, self.clip_act)
for level in reversed(range(self.num_layers)):
for block_index in range(self.num_res_blocks + 1):
hidden = self.up[level].block[block_index](hidden)
if len(self.up[level].attn) > 0:
hidden = self.up[level].attn[block_index](hidden)
hidden = hidden.clamp(-self.clip_act, self.clip_act)
hidden = self.up[level](hidden, self.clip_act)
if level in self.down_layers:
hidden = self.up[level].upsample(hidden)
return _conv1d_with_gain(
@@ -752,32 +271,26 @@ class Decoder1D(nn.Module):
class MMAudioVAE(nn.Module):
"""MMAudio mel-spectrogram VAE for 16 kHz or 44.1 kHz audio."""
def __init__(
self, mode: str | dict[str, Any] = "44k", need_encoder: bool = False
) -> None:
def __init__(self, mode: str = "44k", need_encoder: bool = False) -> None:
super().__init__()
if isinstance(mode, dict):
config = mode
mode = config.get("mode", "44k")
need_encoder = config.get("need_encoder", need_encoder)
if mode == "16k":
data_dim, embed_dim, hidden_dim = 80, 20, 384
data_mean, data_std = DATA_MEAN_80D, DATA_STD_80D
elif mode == "44k":
data_dim, embed_dim, hidden_dim = 128, 40, 512
data_mean, data_std = DATA_MEAN_128D, DATA_STD_128D
else:
raise ValueError(f"Unknown MMAudio VAE mode: {mode}")
self.mode = mode
self.embed_dim = embed_dim
self._weights_normalized = False
# checkpoint buffers are required by native VAELoader's strict load
self.register_buffer(
"data_mean", torch.tensor(data_mean, dtype=torch.float32).view(1, -1, 1)
"data_mean", torch.zeros(1, data_dim, 1, dtype=torch.float32)
)
self.register_buffer(
"data_std", torch.tensor(data_std, dtype=torch.float32).view(1, -1, 1)
"data_std", torch.ones(1, data_dim, 1, dtype=torch.float32)
)
self.encoder = None
if need_encoder:
self.encoder = Encoder1D(
dim=hidden_dim,
@@ -794,7 +307,6 @@ class MMAudioVAE(nn.Module):
num_res_blocks=2,
attn_layers=[3],
down_layers=[0],
in_dim=data_dim,
out_dim=data_dim,
embed_dim=embed_dim,
)
@@ -803,7 +315,7 @@ class MMAudioVAE(nn.Module):
self, mel: torch.Tensor, normalize_input: bool = True
) -> DiagonalGaussianDistribution:
self._require_normalized_weights()
if not hasattr(self, "encoder"):
if self.encoder is None:
raise RuntimeError("This MMAudio VAE was loaded decoder-only")
if normalize_input:
mel = self.normalize(mel)
@@ -831,17 +343,7 @@ class MMAudioVAE(nn.Module):
@torch.no_grad()
def remove_weight_norm(self) -> MMAudioVAE:
# Idempotent: the diffusers reference calls this exactly once, in
# ``MMAudioVAE.__init__``, BEFORE ``from_pretrained`` overwrites the
# freshly-constructed (random) Conv1d weights with a checkpoint's
# real, already-final values -- so a real checkpoint's own weights
# are never renormalized. Mirroring that timing here (see
# ``Kandinsky6AudioVAE.__init__``) means this method only ever runs
# against pre-load weights too; the guard below additionally makes a
# second, redundant call (e.g. the wrapper's own public
# ``remove_weight_norm()``) a true no-op instead of re-mutating an
# already-normalized (possibly checkpoint-loaded) weight a second
# time.
# normalize only random initialization; checkpoint weights are already final
if self._weights_normalized:
return self
for name, module in self.named_modules():
@@ -38,15 +38,19 @@ from sglang.multimodal_gen.runtime.models.vaes.common import (
)
from sglang.multimodal_gen.runtime.models.vaes.wanvae import WanRMS_norm
# Pixel frames per encode segment after the first (which has one more: the causal first frame).
# pixel frames per segment after the extra causal first frame
_SEGMENT_FRAMES = 16
# Inputs above this many elements are convolved in temporal chunks: at 4x SR decode tiles
# (~1536x832) the full-resolution 256-channel convs of the last decoder level see ~5.6e9
# elements per segment, beyond 32-bit indexing, and chunking also keeps the transient conv
# workspace to a fraction of the activation.
# temporal chunks bound workspace and avoid int32 indexing overflow at 4x decode
_MAX_CONV_NUMEL = 2 * 10**9
def _segment_sizes(frames: int, stride: int) -> list[int]:
"""The first causal segment has one extra frame; the final segment may be short."""
return [min(frames, stride + 1)] + [
min(stride, frames - start) for start in range(stride + 1, frames, stride)
]
class _SegmentCache:
"""Causal state carried from one temporal segment to the next during a single encode / decode call."""
@@ -56,17 +60,12 @@ class _SegmentCache:
def _silu(x: torch.Tensor) -> torch.Tensor:
# The output norms use this form and the resblocks use F.silu; the two are not bitwise
# equal in bf16, so this distinction (inherited from the Diffusers / FastVideo ports) is
# deliberate, not a simplification opportunity.
# output norms require this form; F.silu in resblocks rounds differently in bf16
return x * torch.sigmoid(x)
class _K6RMSNorm(WanRMS_norm):
"""``WanRMS_norm`` with an explicit float32 upcast around the normalize, matching the
Diffusers reference's ``Kandinsky6SRRMSNorm`` (which upcasts unconditionally, not only for
fp16/bf16/fp8 inputs): at bf16/fp16 activations, normalizing without upcasting measurably
drifts from the reference over many resnet blocks."""
"""Wan RMS norm with the reference's unconditional fp32 reduction."""
def forward(self, x: torch.Tensor) -> torch.Tensor:
normed = F.normalize(x.float(), dim=1).to(x.dtype)
@@ -89,10 +88,7 @@ class _ChunkedConv3d(nn.Conv3d):
f"padding {self.padding}"
)
if self.stride[0] != 1:
# A strided conv can't reuse the stride-1 path's "prefix each chunk with the
# previous chunk's tail" trick below (a strided conv's output doesn't align with
# input chunk boundaries), so slide one kernel-width window at a time instead, only
# raising if even a single window would overflow.
# strided outputs may cross chunk boundaries: use one kernel window per output
stride = self.stride[0]
window_numel = x.shape[0] * x.shape[1] * k * x.shape[3] * x.shape[4]
if window_numel >= _MAX_CONV_NUMEL:
@@ -100,8 +96,6 @@ class _ChunkedConv3d(nn.Conv3d):
f"frames are too big for Conv3d even one stride-{stride} window at a time "
f"({window_numel} elements per window)"
)
# Bare super() doesn't resolve inside a list comprehension's own implicit scope,
# hence the loop.
window_outputs = []
for i in range(0, frames - k + 1, stride):
window_outputs.append(super().forward(x[:, :, i : i + k]))
@@ -229,21 +223,14 @@ class _ResnetBlock3D(nn.Module):
if in_channels != out_channels:
self.nin_shortcut = _ChunkedConv3d(in_channels, out_channels, kernel_size=1)
def _norm(
self,
norm: nn.Module,
h: torch.Tensor,
zq: torch.Tensor | None,
cache: _SegmentCache,
) -> torch.Tensor:
return norm(h) if zq is None else norm(h, zq, cache)
def forward(
self, x: torch.Tensor, cache: _SegmentCache, zq: torch.Tensor | None = None
) -> torch.Tensor:
h = F.silu(self._norm(self.norm1, x, zq, cache), inplace=True)
h = self.norm1(x) if zq is None else self.norm1(x, zq, cache)
h = F.silu(h, inplace=True)
h = self.conv1(h, cache)
h = F.silu(self._norm(self.norm2, h, zq, cache), inplace=True)
h = self.norm2(h) if zq is None else self.norm2(h, zq, cache)
h = F.silu(h, inplace=True)
h = self.conv2(h, cache)
if hasattr(self, "nin_shortcut"):
x = self.nin_shortcut(x)
@@ -375,8 +362,6 @@ class _PXSUpsample(nn.Module):
x = x[:, :, 1:]
x = self.temporal_conv(x, cache) + x
frames, height, width = x.shape[-3:]
# Chunked along channels: see _chunked_interpolate_nearest's docstring for the overflow
# this guards against (the old vendored code did not chunk this specific call).
x = _chunked_interpolate_nearest(x, (frames, height * 2, width * 2))
x.add_(self.spatial_conv(x))
return self.linear(x)
@@ -402,7 +387,6 @@ class _Encoder3D(nn.Module):
temporal_compress_start_level: int,
) -> None:
super().__init__()
self.num_res_blocks = num_res_blocks
time_levels = range(
temporal_compress_start_level,
temporal_compress_start_level + int(math.log2(temporal_compress_times)),
@@ -494,10 +478,7 @@ class _Decoder3D(nn.Module):
return self.conv_out(_silu(self.norm_out(h, z, cache)), cache)
# The structural knobs every Encoder3D / Decoder3D constructor kwarg comes from; everything
# else in a checkpoint's encoder_config / decoder_config (``checkpoint_list``, ``resolution``,
# the placeholder ``"None"`` channel field, ...) is ignored, the same way the old vendored
# module silently dropped unknown keys via its own ``**ignore_kwargs`` / unused params.
# constructor fields; resolution and training-only metadata do not affect the architecture
_ARCH_KEYS = (
"ch",
"ch_mult",
@@ -508,12 +489,7 @@ _ARCH_KEYS = (
def _arch_kwargs(conf: dict[str, Any], extra: str, which: str) -> dict[str, Any]:
# norm_type is the one legacy knob this port still checks: unlike padding_mode /
# downsample_version / fix_pxs / double_z (which the new Encoder3D / Decoder3D no longer
# branch on at all -- they always run the "zeros" / v2-pixel-unshuffle / fixed-stride /
# double-z behavior those knobs used to select), a group_norm checkpoint would silently
# get rms_norm layers instead of a clear error, which is the one mismatch worth failing
# loudly on here.
# never silently substitute RMSNorm for an unsupported group-norm checkpoint
norm_type = conf.get("norm_type", "rms_norm")
if norm_type != "rms_norm":
raise ValueError(
@@ -596,12 +572,7 @@ class Kandinsky6SRVAE(nn.Module, LayerwiseOffloadableModuleMixin):
def encode(self, x: torch.Tensor) -> tuple[torch.Tensor, list[int]]:
"""``[B, C, T, H, W]`` normalized pixels -> ``(raw latent mean, pixel-frame segment sizes)``."""
split_list = [_SEGMENT_FRAMES + 1]
remaining = x.size(2) - split_list[0]
while remaining > 0:
split_list.append(_SEGMENT_FRAMES)
remaining -= _SEGMENT_FRAMES
split_list[-1] += remaining
split_list = _segment_sizes(x.size(2), _SEGMENT_FRAMES)
cache = _SegmentCache()
latents = []
for segment in torch.split(x, split_list, dim=2):
@@ -634,15 +605,9 @@ class Kandinsky6SRVAE(nn.Module, LayerwiseOffloadableModuleMixin):
return DecoderOutput(sample=self._decode(z))
def _decode(self, z: torch.Tensor) -> torch.Tensor:
segment = _SEGMENT_FRAMES // self.temporal_compression
num_frames = z.size(2)
if num_frames == 1:
split_list = [1]
else:
split_list = [segment] * ((num_frames - 1) // segment)
if (num_frames - 1) % segment:
split_list.append((num_frames - 1) % segment)
split_list[0] += 1
split_list = _segment_sizes(
z.size(2), _SEGMENT_FRAMES // self.temporal_compression
)
cache = _SegmentCache()
samples = []
for chunk in torch.split(z, split_list, dim=2):
@@ -15,22 +15,12 @@ from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
ComposedPipelineBase,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import (
Kandinsky6SRDecodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
Kandinsky6SRDenoisingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import (
Kandinsky6SREncodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
Kandinsky6SRInputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
Kandinsky6SRLatentPrepStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import (
Kandinsky6SROutputStage,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
@@ -29,6 +29,9 @@ from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_c
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
PiflowScheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import (
@@ -48,17 +51,8 @@ from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYP
class Kandinsky6DenoisingStage(DenoisingStage):
"""Run the Kandinsky6 joint video+audio denoising loop."""
def __init__(self, transformer, scheduler, pipeline=None) -> None:
super().__init__(
transformer=transformer, scheduler=scheduler, pipeline=pipeline
)
def _owns_compile_warmup_lifecycle(self) -> bool:
# forward() overrides DenoisingStage.forward (the joint video+audio
# loop cannot reuse the parent's single-tensor _denoise()), but it
# still wraps the loop in `_offload_for_torch_compile_warmup` itself
# below -- same opt-in as MiniMaxH3DenoisingStage, the codebase's
# other joint-modality denoising stage.
# forward wraps the joint loop in the shared compile warmup lifecycle
return True
def component_uses(
@@ -155,7 +149,6 @@ class Kandinsky6DenoisingStage(DenoisingStage):
t_expand: torch.Tensor,
visual_rope_pos: list[torch.Tensor],
scale_factor: tuple[float, ...],
sparse_params: dict[str, Any] | None,
visual_token_type_ids: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run sequential or distributed CFG branches and combine video/audio velocity."""
@@ -172,7 +165,6 @@ class Kandinsky6DenoisingStage(DenoisingStage):
timestep=t_expand,
visual_rope_pos=visual_rope_pos,
scale_factor=scale_factor,
sparse_params=sparse_params,
visual_token_type_ids=visual_token_type_ids,
# BCG clones tuple leaves so the next CFG replay cannot
# overwrite the previous branch's predictions
@@ -227,7 +219,7 @@ class Kandinsky6DenoisingStage(DenoisingStage):
scheduler = batch.scheduler
if scheduler is None:
raise ValueError("scheduler must be set for Kandinsky6 denoising.")
use_piflow = bool(getattr(scheduler, "is_piflow", False))
use_piflow = isinstance(scheduler, PiflowScheduler)
if use_piflow and (
not math.isfinite(batch.guidance_scale)
or abs(batch.guidance_scale - 1.0) > 1e-6
@@ -237,30 +229,10 @@ class Kandinsky6DenoisingStage(DenoisingStage):
pipeline_config = server_args.pipeline_config
arch = pipeline_config.dit_config.arch_config
# The runtime transformer already rejects attention_engine="nabla"
# at construction time (NABLA sparse attention is not yet ported),
# so sparse_params stays None on every currently-reachable
# configuration. A future NABLA-capable DiT would fill this in.
if arch.attention_engine == "nabla":
raise NotImplementedError(
"Kandinsky6 NABLA sparse-attention metadata construction is not implemented; "
"the runtime transformer also rejects attention_engine='nabla' at construction "
"time. Use attention_engine='auto' or 'sdpa'."
)
sparse_params = None
device = get_local_torch_device()
target_dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision]
# .clone(): batch.latents was created by Kandinsky6LatentPreparationStage
# under an active torch.inference_mode() context further up the
# pipeline, which marks it as an "inference tensor" -- such tensors
# can only be mutated in place while still inside an inference_mode
# context. This stage runs under plain @torch.no_grad() instead, and
# mutates `video` in place every denoising step (video[..., :num_channels]
# = ...), so clone once here to get an ordinary, freely-mutable tensor
# before the loop starts. audio is never mutated in place (only
# reassigned via out-of-place `+`), so it needs no clone.
# clone inference tensors before the in-place video updates under no_grad
video = batch.latents.clone()
audio = batch.audio_latents
num_channels = int(arch.in_visual_dim)
@@ -295,40 +267,25 @@ class Kandinsky6DenoisingStage(DenoisingStage):
spatial_ratio = pipeline_config.vae_config.arch_config.spatial_compression_ratio
patch_size = arch.patch_size
# Constant per-request geometry -- computed once outside the loop,
# not per-step.
# geometry is constant across steps
num_video_frames = video.shape[1] - (1 if tail_cond else 0)
t_positions = torch.arange(num_video_frames, device=device)
if tail_cond:
# The appended reference frame reuses T-position 0's rope row
# (RoPE3D is a pure position -> table lookup, so duplicating the
# first row is equivalent to the diffusers reference's
# ``torch.cat([rope, rope[:1]])``).
# the reference frame reuses time position zero
t_positions = torch.cat([t_positions, t_positions.new_zeros(1)])
visual_rope_pos = [
t_positions,
torch.arange(height // spatial_ratio // patch_size[1], device=device),
torch.arange(width // spatial_ratio // patch_size[2], device=device),
]
# Fixed per-checkpoint RoPE frequency scaling read from the DiT's
# arch config (transformer/config.json's "scale_factor", both real
# Pro checkpoints ship [1.0, 2.0, 2.0]) -- matches the diffusers
# reference, which resolves this once in
# ``Kandinsky6TI2VAPipeline.__init__`` and reuses it for every
# request regardless of the request's own height/width. NOT a
# function of the request's resolution.
# RoPE scaling comes from the checkpoint, not the request resolution
scale_factor = arch.scale_factor
image_latent = batch.image_latent
total_steps = int(batch.timesteps.shape[0])
self._maybe_enable_cache_dit_and_torch_compile(total_steps, batch)
# PiFlow's noisy video state is tracked in fp32 across steps here,
# separately from the `video` buffer's bf16 storage dtype -- see the
# module docstring. Always a Tensor (not `Tensor | None`) even on the
# flow-Euler path, where it is simply unused: the initial cast is one
# cheap op, and keeping it non-Optional avoids re-narrowing it after
# every `if use_piflow:` branch re-entry in the loop below.
# pi-Flow retains fp32 state between bf16 DiT calls
video_state = video[..., :num_channels].to(torch.float32)
with self.use_declared_component(
@@ -371,7 +328,6 @@ class Kandinsky6DenoisingStage(DenoisingStage):
t_expand=t_expand,
visual_rope_pos=visual_rope_pos,
scale_factor=scale_factor,
sparse_params=sparse_params,
visual_token_type_ids=visual_token_type_ids,
)
@@ -397,9 +353,7 @@ class Kandinsky6DenoisingStage(DenoisingStage):
if use_piflow:
video_state[:, -1:] = ref_frame.to(torch.float32)
# Manual Euler update for audio, using the SAME per-step
# sigma delta the scheduler.step() call above just
# consumed for video.
# Euler audio reuses the video's sigma delta without advancing it
if use_piflow:
audio = audio_scheduler.step(
audio_vel, timestep, audio, return_dict=False
@@ -1,8 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky 6 video super-resolution (VSR) stages and framework-independent helpers.
Import the stage classes from their modules (``input_stage``, ``encode_stage``,
``latent_prep_stage``, ``denoising_stage``, ``decode_stage``, ``output_stage``); this package
``__init__`` stays empty so that configs can import the pure helpers (``video_utils``,
``tiling``, ...) cheaply.
"""
"""SR stages and helpers; import stages explicitly to keep config imports lightweight."""
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Decode stage of Kandinsky 6 video SR: denoised latent chunks -> uint8 tiles."""
import torch
from sglang.multimodal_gen.runtime.distributed import (
get_decode_parallel_group_coordinator,
get_local_torch_device,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.models.vaes.common import (
can_install_spatial_shard_parallel_decode,
has_decode_parallel_world,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_DENOISED_KEY,
SR_TILES_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
decode_chunks,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
class Kandinsky6SRDecodeStage(PipelineStage):
"""VAE-decodes every denoised chunk into row-major uint8 tiles (``[3, T, Hb, Wb]``)."""
def __init__(self, vae) -> None:
super().__init__()
self.vae = vae
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
return [
ComponentUse(
self._component_stage_name(stage_name),
"vae",
phase="decode_tiles",
target_dtype=vae_dtype,
)
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
denoised = batch.extra.pop(SR_DENOISED_KEY)
device = get_local_torch_device()
group = None
if (
server_args.pipeline_config.vae_config.use_parallel_tiling
and has_decode_parallel_world()
and not can_install_spatial_shard_parallel_decode(
server_args.pipeline_config.vae_config
)
):
group = get_decode_parallel_group_coordinator()
with self.use_declared_component(
component_name="vae", module=self.vae, phase="decode_tiles"
) as vae:
assert vae is not None
self.vae = vae
tiles = decode_chunks(
denoised,
vae,
scaling_factor=vae.scaling_factor,
device=device,
group=group,
)
batch.extra[SR_TILES_KEY] = tiles
return batch
@@ -1,137 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Text-free SR tile denoising with shared residency, compile and cache-DiT hooks.
Each chunk resets its scheduler; the generic single-latent CFG loop and
prompt-padding BCG path do not apply."""
from __future__ import annotations
import torch
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_CHUNKS_KEY,
SR_DENOISED_KEY,
SR_DIT_SPEC_KEY,
SR_SAMPLING_SPEC_KEY,
effective_scheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
denoise_chunks,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
StageValidators as V,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
VerificationResult,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision import precision_to_dtype
class Kandinsky6SRDenoisingStage(DenoisingStage):
"""Runs the bundle's scheduler over every tile chunk prepared by the latent-prep stage."""
def __init__(self, transformer, scheduler, pipeline=None) -> None:
super().__init__(
transformer=transformer,
scheduler=scheduler,
pipeline=pipeline,
transformer_2=None,
vae=None,
)
def _owns_compile_warmup_lifecycle(self) -> bool:
# ``forward`` below does not go through the shared ``_denoise`` loop that this guard
# normally protects, so claim ownership explicitly (same as MiniMaxH3DenoisingStage):
# the offload/restore wrapper is still entered through ``_offload_for_torch_compile_warmup``.
return True
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
"""Validate tile-chunk state instead of the shared text/CFG loop inputs."""
result = VerificationResult()
result.add_check(
"extra[SR_SAMPLING_SPEC_KEY]",
batch.extra.get(SR_SAMPLING_SPEC_KEY),
V.not_none,
)
result.add_check(
"extra[SR_DIT_SPEC_KEY]", batch.extra.get(SR_DIT_SPEC_KEY), V.not_none
)
result.add_check(
"extra[SR_CHUNKS_KEY]", batch.extra.get(SR_CHUNKS_KEY), V.list_not_empty
)
return result
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
result.add_check(
"extra[SR_DENOISED_KEY]", batch.extra.get(SR_DENOISED_KEY), V.list_not_empty
)
return result
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
dit_dtype = precision_to_dtype(
server_args.pipeline_config.dit_precision, "dit_precision"
)
return [
ComponentUse(
self._component_stage_name(stage_name),
"transformer",
phase="denoise_tiles",
target_dtype=dit_dtype,
preferred_ready_after_request=True,
memory_intensive=True,
)
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
with self._offload_for_torch_compile_warmup(batch):
return self._denoise_tiles(batch, server_args)
def _denoise_tiles(self, batch: Req, server_args: ServerArgs) -> Req:
spec = batch.extra.pop(SR_SAMPLING_SPEC_KEY)
dit_spec = batch.extra.pop(SR_DIT_SPEC_KEY)
chunks = batch.extra.pop(SR_CHUNKS_KEY)
device = get_local_torch_device()
# Reuse the shared cache-dit / torch.compile wiring (both are no-ops unless the
# corresponding server arg actually requests them): ``spec.steps_per_chunk`` is this
# request's DiT-call count, the closest analogue of the global ``num_inference_steps``
# those hooks expect.
self._maybe_enable_cache_dit_and_torch_compile(spec.steps_per_chunk, batch)
scheduler = effective_scheduler(spec, self.scheduler)
with self.use_declared_component(
component_name="transformer", module=self.transformer, phase="denoise_tiles"
) as transformer:
assert transformer is not None
self.transformer = transformer
total = spec.steps_per_chunk * len(chunks)
with self.progress_bar(
total=total, batch=batch, desc="Kandinsky6 SR denoising"
) as progress:
denoised = denoise_chunks(
chunks,
transformer,
scheduler,
dit_spec=dit_spec,
spec=spec,
device=device,
step_context=lambda step: set_forward_context(
current_timestep=step, attn_metadata=None, forward_batch=batch
),
on_step=progress.update,
)
self._finish_active_component_use()
batch.extra[SR_DENOISED_KEY] = denoised
return batch
@@ -1,65 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Whole-video KVAE encode of the latent-upscaler (LU) path of Kandinsky 6 video SR."""
import torch
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_LR_LATENT_KEY,
SR_TILING_SCALE_KEY,
SR_VIDEO_KEY,
uses_latent_path,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
encode_video_to_lr_latent,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
class Kandinsky6SREncodeStage(PipelineStage):
"""Encode the whole clip only for the LU path; pixel mode encodes tiles later."""
def __init__(self, vae, latent_upscaler=None) -> None:
super().__init__()
self.vae = vae
# Only consulted to pick the path (does the bank have this scale?).
self.latent_upscaler = latent_upscaler
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
return [
ComponentUse(
self._component_stage_name(stage_name),
"vae",
phase="encode_video",
target_dtype=vae_dtype,
start_at_stage_entry=False,
)
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
if not uses_latent_path(self.latent_upscaler, tiling_scale):
return batch
video = batch.extra.pop(SR_VIDEO_KEY)
with self.use_declared_component(
component_name="vae", module=self.vae, phase="encode_video"
) as vae:
assert vae is not None
self.vae = vae
lr_latent = encode_video_to_lr_latent(
video, vae, device=get_local_torch_device()
)
batch.extra[SR_LR_LATENT_KEY] = lr_latent.cpu()
return batch
@@ -1,95 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Input stage of Kandinsky 6 video SR: source video -> aligned clip (+ source audio)."""
import os
import torch
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_REQUESTED_HW_KEY,
SR_TILING_SCALE_KEY,
SR_VIDEO_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
VAE_SPATIAL_FACTOR,
pad_to_spatial_factor,
pre_upscale_video,
resolve_scale_request,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
SOURCE_AUDIO_SAMPLE_RATE,
DecodedClip,
decode_clip,
extract_source_audio,
synthetic_clip,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
# Warmup requests carry no (usable) video: run the real path on a small synthetic clip.
WARMUP_CLIP_FRAMES = 9
WARMUP_CLIP_HW = (256, 384)
class Kandinsky6SRInputStage(PipelineStage):
"""Decode [T, 3, H, W] uint8 video, align fps/frames/size and retain source audio.
Record requested dimensions before VAE padding so output can remove it."""
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
tiling_scale, pre_upscale = resolve_scale_request(batch.sr_resolution_scale)
clip = self._load_clip(batch)
video = clip.frames
if pre_upscale != 1.0:
video = pre_upscale_video(video, pre_upscale, VAE_SPATIAL_FACTOR)
frames = video.shape[0]
# Align to the VAE spatial factor for every scale, not only 2.25 (whose pixel
# pre-upscale happens to already land on an aligned size): without this, a source
# whose dims are not already a multiple of 16 (e.g. a plain 1080p source at x2/x4,
# which runs no pixel resize at all) fails partway through the whole-video KVAE
# encode's four halvings instead of at a clear boundary. Padding (not resizing) keeps
# every source pixel; batch.height / batch.width below, and the output stage's final
# crop, restore the originally requested dimensions.
video, (requested_h, requested_w) = pad_to_spatial_factor(
video, VAE_SPATIAL_FACTOR
)
batch.extra[SR_VIDEO_KEY] = video
batch.extra[SR_TILING_SCALE_KEY] = tiling_scale
batch.extra[SR_REQUESTED_HW_KEY] = (requested_h, requested_w)
batch.fps = clip.fps
batch.num_frames = frames
batch.height = requested_h * tiling_scale
batch.width = requested_w * tiling_scale
self._attach_source_audio(batch, frames=frames, fps=clip.fps)
return batch
def _load_clip(self, batch: Req) -> DecodedClip:
if batch.is_warmup:
height, width = WARMUP_CLIP_HW
frames = synthetic_clip(
height=height, width=width, frames=WARMUP_CLIP_FRAMES
)
return DecodedClip(frames=frames, fps=24, source_fps=24.0)
path = batch.sampling_params.source_video_path()
if path is None or not os.path.isfile(path):
raise ValueError(f"Kandinsky6 SR input video not found: {path!r}")
self.log_info("Decoding %s", path)
return decode_clip(path)
@staticmethod
def _attach_source_audio(batch: Req, *, frames: int, fps: int) -> None:
if batch.is_warmup:
return
path = batch.sampling_params.source_video_path()
audio = extract_source_audio(
path, sample_rate=SOURCE_AUDIO_SAMPLE_RATE, max_seconds=frames / fps
)
if audio is None:
batch.audio = None
batch.audio_sample_rate = None
return
batch.audio = torch.from_numpy(audio)
batch.audio_sample_rate = SOURCE_AUDIO_SAMPLE_RATE
@@ -1,162 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Prepare noisy tile chunks through the latent upscaler or pixel/VAE fallback.
Validate sampler/head compatibility and step count before processing tiles."""
from functools import partial
import torch
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_CHUNKS_KEY,
SR_DIT_SPEC_KEY,
SR_LR_LATENT_KEY,
SR_PLAN_KEY,
SR_SAMPLING_SPEC_KEY,
SR_TILING_SCALE_KEY,
SR_VIDEO_KEY,
build_dit_spec,
build_sampling_spec,
check_denoising_request,
uses_latent_path,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
DitSpec,
SamplingSpec,
module_dtype,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
TilePlan,
plan_tiles,
prepare_lu_tile_latents,
prepare_pixel_tile_latents,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
VAE_SPATIAL_FACTOR,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision import (
resolve_precision,
)
class Kandinsky6SRLatentPrepStage(PipelineStage):
"""LU-or-pixel tile encode + upscale -> initial noisy latent, one chunk at a time."""
def __init__(self, vae, transformer, latent_upscaler=None, scheduler=None) -> None:
super().__init__()
self.vae = vae
self.transformer = transformer # arch-config / dtype only: never called here
self.latent_upscaler = latent_upscaler
self.scheduler = (
scheduler # resolved sampler validation only: never stepped here
)
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
stage_name = self._component_stage_name(stage_name)
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
uses: list[ComponentUse] = []
if self.latent_upscaler is not None:
uses.append(
ComponentUse(
stage_name,
"latent_upscaler",
phase="upscale_tiles",
target_dtype=torch.bfloat16,
)
)
uses.append(
ComponentUse(
stage_name, "vae", phase="encode_tiles", target_dtype=vae_dtype
)
)
return uses
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
arch = server_args.pipeline_config.dit_config.arch_config
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
seed = batch.seed if isinstance(batch.seed, int) else batch.seed[0]
spec = build_sampling_spec(
arch=arch,
tiling_scale=tiling_scale,
seed=seed,
num_steps=batch.num_inference_steps,
tiles_batch_size=batch.sr_tiles_batch_size,
tile_min_overlap=batch.sr_tile_min_overlap,
scheduler=self.scheduler,
)
check_denoising_request(spec, batch.num_inference_steps)
dit_spec = build_dit_spec(self.transformer)
device = get_local_torch_device()
plan, chunks = self._prepare_chunks(batch, spec, dit_spec, device)
batch.extra[SR_PLAN_KEY] = plan
batch.extra[SR_CHUNKS_KEY] = chunks
batch.extra[SR_SAMPLING_SPEC_KEY] = spec
batch.extra[SR_DIT_SPEC_KEY] = dit_spec
return batch
def _prepare_chunks(
self, batch: Req, spec: SamplingSpec, dit_spec: DitSpec, device: torch.device
) -> tuple[TilePlan, list[torch.Tensor]]:
common = dict(
visual_size=spec.visual_size,
tiling_scale=spec.tiling_scale,
tile_min_overlap=spec.tile_min_overlap,
)
if uses_latent_path(self.latent_upscaler, spec.tiling_scale):
lr_latent = batch.extra.pop(SR_LR_LATENT_KEY)
plan = plan_tiles(
frame_hw=(
lr_latent.shape[-2] * VAE_SPATIAL_FACTOR,
lr_latent.shape[-1] * VAE_SPATIAL_FACTOR,
),
**common,
)
with self.use_declared_component(
component_name="latent_upscaler",
module=self.latent_upscaler,
phase="upscale_tiles",
) as bank:
assert bank is not None
self.latent_upscaler = bank
chunks = prepare_lu_tile_latents(
lr_latent,
plan,
upscale_fn=partial(bank.upscale, scale=spec.tiling_scale),
lu_dtype=module_dtype(bank),
scaling_factor=self.vae.scaling_factor,
dit_spec=dit_spec,
spec=spec,
device=device,
)
return plan, chunks
video = batch.extra.pop(SR_VIDEO_KEY)
plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common)
with self.use_declared_component(
component_name="vae", module=self.vae, phase="encode_tiles"
) as vae:
assert vae is not None
self.vae = vae
chunks = prepare_pixel_tile_latents(
video,
plan,
vae=vae,
scaling_factor=vae.scaling_factor,
dit_spec=dit_spec,
spec=spec,
device=device,
)
return plan, chunks
@@ -35,23 +35,12 @@ def degrade_lq_latent(
return (1 - noise_scale) * lq_latent + noise_scale * eps
def _cast_floating(value: torch.Tensor, dtype: torch.dtype | None) -> torch.Tensor:
"""Cast floating-point ``value`` to ``dtype`` (no-op for ``None`` / non-float / same dtype)."""
if dtype is not None and value.is_floating_point() and value.dtype != dtype:
return value.to(dtype=dtype)
return value
def build_initial_latent( # noqa: PLR0913
def build_initial_latent(
*,
instruct_type: InstructType,
visual_cond: bool,
in_visual_dim: int,
lq_latent: torch.Tensor,
batch_size: int,
duration: int,
height: int,
width: int,
device: str | int | torch.device,
seed: int,
lq_noise_scale: float,
@@ -59,76 +48,33 @@ def build_initial_latent( # noqa: PLR0913
lq_channel_noise_scale: float,
dtype: torch.dtype | None = None,
) -> torch.Tensor:
"""Build [starting | conditioning | mask] for the selected instruction mode.
Inputs and output use [B*T, H, W, C]; the output has 2C+1 channels except
unconditioned noise mode. Cast LQ to the DiT dtype before drawing noise."""
lq_latent = lq_latent.to(device)
lq_latent = _cast_floating(lq_latent, dtype)
"""Build packed [B*T, H, W, C] noise and optional conditioning without reseeding global RNG."""
if instruct_type not in ("noise", "channel", "hybrid", "hybrid_anchor"):
raise ValueError(f"Unsupported SR instruct_type={instruct_type!r}")
lq_latent = lq_latent.to(device=device, dtype=dtype)
generator = torch.Generator(device=device).manual_seed(seed)
if instruct_type == "channel":
starting = torch.randn(
(*lq_latent.shape[:-1], in_visual_dim), device=device, generator=generator
)
else:
starting = degrade_lq_latent(
lq_latent, lq_noise_scale, lq_noise_type, generator=generator
)
if instruct_type == "noise":
g_noise = torch.Generator(device=device)
g_noise.manual_seed(seed)
degraded_lq = degrade_lq_latent(
lq_latent, lq_noise_scale, lq_noise_type, generator=g_noise
)
if not visual_cond:
# No conditioning channels in the model input layer (e.g. the
# t2v-initialized KVAE SR DiT) — feed the bare C-channel latent.
return degraded_lq
zero_cond = torch.zeros_like(degraded_lq)
zero_mask = torch.zeros(
[*degraded_lq.shape[:-1], 1], dtype=degraded_lq.dtype, device=device
return starting
conditioning = torch.zeros_like(starting)
mask = torch.zeros_like(starting[..., :1])
elif instruct_type == "hybrid_anchor":
# no HR anchor exists at inference; conditioning noise still uses the global RNG
conditioning = degrade_lq_latent(
torch.zeros_like(lq_latent), lq_channel_noise_scale, noise_type="linear"
)
return torch.cat([degraded_lq, zero_cond, zero_mask], dim=-1)
if instruct_type in ("channel", "hybrid"):
if instruct_type == "hybrid":
g_noise = torch.Generator(device=device)
g_noise.manual_seed(seed)
starting = degrade_lq_latent(
lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise
)
else:
g = torch.Generator(device=device)
g.manual_seed(seed)
starting = torch.randn(
batch_size * duration,
height,
width,
in_visual_dim,
device=device,
generator=g,
)
channel_lq = degrade_lq_latent(
mask = torch.zeros_like(lq_latent[..., :1])
else:
conditioning = degrade_lq_latent(
lq_latent, lq_channel_noise_scale, noise_type="linear"
)
mask = torch.ones_like(lq_latent[..., :1])
return torch.cat([starting, channel_lq, mask], dim=-1)
if instruct_type == "hybrid_anchor":
# Anchor-free inference (tiled real-world LQ has no HR ground truth):
# a zeroed anchor + zeroed mask runs the model with no anchor signal,
# equivalent to anchor_indices=[] for the packed-latent datasets.
anchor_latent = torch.zeros_like(lq_latent)
anchor_mask = torch.zeros_like(lq_latent[..., :1])
g_noise = torch.Generator(device=device)
g_noise.manual_seed(seed)
starting = degrade_lq_latent(
lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise
)
anchor = degrade_lq_latent(
anchor_latent, lq_channel_noise_scale, noise_type="linear"
)
return torch.cat([starting, anchor, anchor_mask], dim=-1)
msg = f"generate_sample_sr does not support instruct_type={instruct_type!r}"
raise ValueError(msg)
__all__ = [
"InstructType",
"LqNoiseType",
"build_initial_latent",
"degrade_lq_latent",
]
return torch.cat([starting, conditioning, mask], dim=-1)
@@ -1,72 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Stitch / crop / resize / output stage of Kandinsky 6 video SR."""
import torch
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
resolve_target_hw,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_PLAN_KEY,
SR_REQUESTED_HW_KEY,
SR_TILES_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
stitch_tiles,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
crop_to_hw,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
to_output_video,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
resize_to_target,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
class Kandinsky6SROutputStage(PipelineStage):
"""Hann-blends the decoded tiles, restores the requested output size, applies the
optional delivery resize and returns ``output = [1, 3, T, H, W]`` (fp16 in [0, 1])
plus the mono source audio.
CPU only: every model phase already ran. ``batch.height`` / ``batch.width`` are set
to the final size, ``batch.fps`` was set by the input stage (and now also carried on
the returned ``OutputBatch``, so a worker-resampled fps reaches the client-side save
path unchanged).
"""
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
tiles = batch.extra.pop(SR_TILES_KEY)
plan = batch.extra.pop(SR_PLAN_KEY)
video = stitch_tiles(tiles, plan)
del tiles
requested_hw = batch.extra.pop(SR_REQUESTED_HW_KEY, None)
if requested_hw is not None:
requested_h, requested_w = requested_hw
video = crop_to_hw(
video,
(requested_h * plan.tiling_scale, requested_w * plan.tiling_scale),
)
target_hw = resolve_target_hw(
batch.sr_target_resolution,
tuple(video.shape[-2:]),
batch.sr_target_resize_mode,
)
if target_hw is not None:
video = resize_to_target(video, target_hw)
batch.height, batch.width = video.shape[-2:]
return OutputBatch(
output=to_output_video(video),
audio=batch.audio,
audio_sample_rate=batch.audio_sample_rate,
fps=batch.fps,
metrics=batch.metrics,
usage=batch.usage,
)
@@ -16,7 +16,6 @@ from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
DitSpec,
PiflowParams,
SamplingSpec,
module_dtype,
)
@@ -31,10 +30,7 @@ SR_TILING_SCALE_KEY = "kandinsky6_sr_tiling_scale" # int tile scale (2 or 4)
SR_LR_LATENT_KEY = "kandinsky6_sr_lr_latent" # [T', C, h, w] fp32 (LU path)
SR_PLAN_KEY = "kandinsky6_sr_tile_plan" # tiled.TilePlan
SR_TILES_KEY = "kandinsky6_sr_tiles" # list of uint8 [3, T, Hb, Wb]
# Handoff between the three model-phase stages (latent prep -> denoise -> decode), split out
# of the single combined stage so the denoising stage can subclass the shared DenoisingStage
# (see denoising_stage.py): SamplingSpec / DitSpec are immutable msgspec.Struct values (plain
# data, like TilePlan above), and the chunk lists are CPU tensors throughout.
# stages exchange immutable specs and CPU chunks, not resident model objects
SR_SAMPLING_SPEC_KEY = "kandinsky6_sr_sampling_spec" # sampling.SamplingSpec
SR_DIT_SPEC_KEY = "kandinsky6_sr_dit_spec" # sampling.DitSpec
SR_CHUNKS_KEY = (
@@ -43,11 +39,7 @@ SR_CHUNKS_KEY = (
SR_DENOISED_KEY = (
"kandinsky6_sr_denoised" # list of fp32 [B, T', H', W', C] (post-denoise)
)
# (H, W) of the clip actually requested, before any VAE-spatial-factor alignment padding the
# input stage added so whole-video KVAE encoding survives every supported sr_resolution_scale
# (not only 2.25, whose pixel pre-upscale happens to already land on an aligned size). The
# output stage crops the stitched result from (padded_H * scale, padded_W * scale) down to
# (H * scale, W * scale) so the delivered video matches what was actually asked for.
# original (H, W), used to crop VAE alignment padding from the stitched output
SR_REQUESTED_HW_KEY = "kandinsky6_sr_requested_hw"
@@ -71,7 +63,56 @@ def build_dit_spec(dit: Any) -> DitSpec:
)
def _rope_scale_factor(arch: Kandinsky6SRArchConfig) -> tuple[float, float, float]:
def build_sampling_spec(
*,
arch: Kandinsky6SRArchConfig,
tiling_scale: int,
seed: int,
num_steps: int,
tiles_batch_size: int,
tile_min_overlap: float,
scheduler: FlowMatchEulerDiscreteScheduler,
) -> SamplingSpec:
"""Validate the loaded scheduler and resolve DiT calls before encoding tiles."""
if num_steps < 1:
raise ValueError(
f"Kandinsky6 SR: num_inference_steps must be >= 1, got {num_steps}"
)
if not isinstance(scheduler, FlowMatchEulerDiscreteScheduler):
raise ValueError(
"Kandinsky6 SR requires a FlowMatchEulerDiscreteScheduler or "
"PiflowScheduler from the checkpoint"
)
is_piflow = isinstance(scheduler, PiflowScheduler)
n_grid = scheduler.config.n_grid if is_piflow else 1
expected_width = arch.in_visual_dim * n_grid
if arch.out_visual_dim != expected_width:
raise ValueError(
f"Kandinsky6 SR: {type(scheduler).__name__} requires a {expected_width}-channel "
f"head, but out_visual_dim={arch.out_visual_dim}; use the scheduler "
"from the same checkpoint as the transformer"
)
if is_piflow:
nfe = scheduler.config.nfe
if nfe is None or nfe < 1:
raise ValueError(
f"Kandinsky6 SR: PiflowScheduler requires nfe >= 1, got {nfe}"
)
if arch.sr_cap_noise_timestep and arch.effective_override("instruct_type") in (
"noise",
"hybrid",
):
raise NotImplementedError(
"piflow sampler does not support cap_noise_timestep for noise/hybrid instruct"
)
if num_steps != nfe:
logger.warning(
"Kandinsky6 SR: ignoring num_inference_steps=%d; the pi-Flow checkpoint "
"requires %d DiT calls per tile",
num_steps,
nfe,
)
num_steps = nfe
table = arch.sr_scale_factor
visual_size = arch.sr_visual_size[0]
if table is None:
@@ -84,138 +125,17 @@ def _rope_scale_factor(arch: Kandinsky6SRArchConfig) -> tuple[float, float, floa
raise ValueError(
f"sr_scale_factor has no entry for visual size {visual_size}: {table}"
)
return tuple(float(value) for value in values)
def piflow_params_from_scheduler(scheduler: Any) -> PiflowParams | None:
"""pi-Flow sampler values of a ``PiflowScheduler`` component (``None``: flow-Euler)."""
if not isinstance(scheduler, PiflowScheduler):
return None
config = scheduler.config
if config["nfe"] is None:
return None
return PiflowParams(
nfe=int(config["nfe"]),
num_policy_substeps=int(config["num_policy_substeps"]),
final_step_size_scale=float(config["final_step_size_scale"]),
shift=float(config["shift"]),
n_grid=int(config["n_grid"]),
eps=float(config["eps"]),
)
def _describe_scheduler(scheduler: Any) -> str:
if scheduler is None:
return "missing"
if isinstance(scheduler, PiflowScheduler):
return "a PiflowScheduler without nfe"
return f"a {type(scheduler).__name__}"
def _check_sampler_fits_head(
*, arch: Kandinsky6SRArchConfig, piflow: PiflowParams | None, scheduler: Any
) -> None:
"""Reject incompatible scheduler and DiT head widths before encoding tiles."""
head, grid = arch.head_width, arch.base_out_visual_dim
if piflow is None:
if head != grid:
raise ValueError(
f"Kandinsky6SR: the transformer head is {head} channels wide (a DX / "
f"pi-Flow head: {grid} channels per latent), but the sampler resolved "
"to flow-Euler: the scheduler component is "
f"{_describe_scheduler(scheduler)}, and only a PiflowScheduler with "
"nfe selects pi-Flow. Use the scheduler of the same repo as the "
"transformer."
)
return
if piflow.nfe < 1:
raise ValueError(
"Kandinsky6SR: the pi-Flow sampler needs nfe >= 1 model calls per tile, "
f"got nfe={piflow.nfe}."
)
if head != piflow.n_grid * grid:
raise ValueError(
f"Kandinsky6SR: the PiflowScheduler (nfe {piflow.nfe}, n_grid "
f"{piflow.n_grid}) needs a {piflow.n_grid} x {grid} = "
f"{piflow.n_grid * grid} channel DX head, but the transformer head is "
f"{head} channels wide (out_visual_dim {arch.out_visual_dim}). The "
"scheduler and the transformer come from different repos."
)
def build_sampling_spec(
*,
arch: Kandinsky6SRArchConfig,
tiling_scale: int,
seed: int,
num_steps: int,
tiles_batch_size: int,
tile_min_overlap: float,
scheduler: Any = None,
) -> SamplingSpec:
"""Resolve sampling from the bundle scheduler, falling back to legacy DiT fields.
num_steps counts DiT calls, not Diffusers timestep grid points."""
piflow = piflow_params_from_scheduler(scheduler)
if piflow is None and arch.is_piflow:
piflow = PiflowParams(
nfe=arch.piflow_nfe,
num_policy_substeps=arch.piflow_num_policy_substeps,
final_step_size_scale=arch.piflow_final_step_size_scale,
shift=arch.piflow_shift,
n_grid=arch.n_grid,
eps=arch.piflow_eps,
)
_check_sampler_fits_head(arch=arch, piflow=piflow, scheduler=scheduler)
return SamplingSpec(
tiling_scale=tiling_scale,
tiles_batch_size=tiles_batch_size,
seed=seed,
num_steps=num_steps,
is_piflow=is_piflow,
tile_min_overlap=tile_min_overlap,
visual_size=arch.sr_visual_size[0],
scale_factor=_rope_scale_factor(arch),
scheduler_scale=arch.sr_scheduler_scale,
visual_size=visual_size,
scale_factor=tuple(float(value) for value in values),
lq_noise_scale=arch.sr_lq_noise_scale,
lq_noise_type=arch.sr_lq_noise_type,
lq_channel_noise_scale=arch.sr_lq_channel_noise_scale,
cap_noise_timestep=arch.sr_cap_noise_timestep,
piflow=piflow,
)
def effective_scheduler(spec: SamplingSpec, scheduler: Any) -> Any:
"""Use the loaded scheduler when compatible; synthesize one for legacy configs."""
if spec.piflow is not None:
if isinstance(scheduler, PiflowScheduler):
return scheduler
return PiflowScheduler(
nfe=spec.piflow.nfe,
n_grid=spec.piflow.n_grid,
shift=spec.piflow.shift,
eps=spec.piflow.eps,
final_step_size_scale=spec.piflow.final_step_size_scale,
num_policy_substeps=spec.piflow.num_policy_substeps,
)
if isinstance(scheduler, FlowMatchEulerDiscreteScheduler):
return scheduler
return FlowMatchEulerDiscreteScheduler(shift=spec.scheduler_scale)
def check_denoising_request(spec: SamplingSpec, num_inference_steps: int) -> None:
"""Validate the requested DiT-call count before denoising.
Diffusers counts timestep grid points instead: its default 5 is 4 calls here."""
if num_inference_steps < 1:
raise ValueError(
f"Kandinsky6 SR: num_inference_steps must be >= 1, got {num_inference_steps}"
)
if spec.piflow is not None and num_inference_steps != spec.piflow.nfe:
logger.warning(
"Kandinsky6 SR: num_inference_steps=%d was requested, but this is a pi-Flow "
"bundle distilled for nfe=%d; num_inference_steps is ignored for pi-Flow "
"bundles and every tile will run exactly nfe=%d DiT calls instead.",
num_inference_steps,
spec.piflow.nfe,
spec.piflow.nfe,
)
@@ -1,191 +1,133 @@
# SPDX-License-Identifier: Apache-2.0
"""Scheduler-driven denoising for Kandinsky 6 SR tile chunks."""
from collections.abc import Callable
from collections.abc import Callable, Sequence
from contextlib import AbstractContextManager, nullcontext
from typing import Any
import msgspec
import torch
# (latent [B, T, H, W, C], model time [B] (already * 1000), step index) -> DiT output
DitFn = Callable[[torch.Tensor, torch.Tensor, int], torch.Tensor]
StepCallback = Callable[[], None]
StepContext = Callable[[int], AbstractContextManager]
class PiflowParams(msgspec.Struct, frozen=True, kw_only=True):
"""pi-Flow parameters from the scheduler or legacy flat DiT config."""
nfe: int
num_policy_substeps: int
final_step_size_scale: float
shift: float
n_grid: int
eps: float
class DitSpec(msgspec.Struct, frozen=True, kw_only=True):
"""Post-override behaviour of the DiT that the sampling code needs."""
"""Post-override input layout and loaded parameter dtype."""
instruct_type: str | None
visual_cond: bool
in_visual_dim: int
use_motion_score: bool
patch_size: tuple[int, int, int]
# The DiT's loaded parameter dtype (``module_dtype(dit)``); the reference casts the
# LQ latent to this dtype before building the initial latent, so a bf16-loaded
# checkpoint's LQ-noise mix rounds in bf16, not fp32. ``None`` (parameter-less DiT,
# e.g. a mocked one in a test) keeps the LQ latent's own dtype.
dtype: torch.dtype | None = None
class SamplingSpec(msgspec.Struct, frozen=True, kw_only=True):
"""Everything besides the components that defines one tiled SR run."""
"""Serializable settings shared by tile preparation and denoising."""
tiling_scale: int
tiles_batch_size: int
seed: int
num_steps: int
is_piflow: bool
tile_min_overlap: float
visual_size: int
scale_factor: tuple[float, float, float]
scheduler_scale: float
lq_noise_scale: float
lq_noise_type: str
lq_channel_noise_scale: float
cap_noise_timestep: bool
piflow: PiflowParams | None
@property
def steps_per_chunk(self) -> int:
"""DiT calls per chunk of tiles: ``nfe`` (pi-Flow), else ``num_steps``."""
return self.piflow.nfe if self.piflow is not None else self.num_steps
def bf16_autocast(device: torch.device | str) -> AbstractContextManager:
"""CUDA bf16 autocast (a no-op on other devices), as the reference wraps its models."""
if torch.device(device).type != "cuda":
return nullcontext()
return torch.autocast(device_type="cuda", dtype=torch.bfloat16)
def module_dtype(module: torch.nn.Module) -> torch.dtype | None:
"""Dtype of the first parameter (``None`` for a parameter-less module)."""
return next((param.dtype for param in module.parameters()), None)
def cast_to_module_dtype(module: torch.nn.Module, value: torch.Tensor) -> torch.Tensor:
"""Cast a floating tensor to the module's parameter dtype (reference helper)."""
dtype = module_dtype(module)
if dtype is not None and value.is_floating_point() and value.dtype != dtype:
return value.to(dtype=dtype)
return value
def euler_start_timestep(
*, cap_noise_timestep: bool, lq_noise_scale: float, instruct_type: str | None
) -> float:
"""Euler start time: the LQ noise level when capped for noise-type conditioning."""
capped = cap_noise_timestep and instruct_type in (
"noise",
"hybrid",
"hybrid_anchor",
return (
value.to(dtype=dtype)
if dtype is not None and value.is_floating_point()
else value
)
return lq_noise_scale if capped else 1.0
def check_sampler_options(
*, piflow: PiflowParams | None, cap_noise_timestep: bool, instruct_type: str | None
) -> None:
"""Reject option combinations the reference sampler rejects."""
if (
piflow is not None
and cap_noise_timestep
and instruct_type in ("noise", "hybrid")
):
raise NotImplementedError(
"piflow sampler does not support cap_noise_timestep for noise/hybrid "
"instruct"
)
def make_dit_fn(
dit: Callable[..., torch.Tensor],
*,
latent_frames_hw: tuple[int, int, int],
patch_size: tuple[int, int, int],
scale_factor: tuple[float, float, float],
use_motion_score: bool,
step_context: StepContext | None = None,
) -> DitFn:
"""Bind per-chunk RoPE/motion inputs and an optional per-step context."""
frames, height, width = latent_frames_hw
context = step_context or (lambda step: nullcontext())
def call(x: torch.Tensor, model_time: torch.Tensor, step: int) -> torch.Tensor:
rope_pos = [
torch.arange(frames, device=x.device),
torch.arange(height // patch_size[1], device=x.device),
torch.arange(width // patch_size[2], device=x.device),
]
motion = (
torch.full((1,), 900.0, device=x.device, dtype=x.dtype)
if use_motion_score
else None
)
with context(step):
return dit(
x,
model_time,
rope_pos,
scale_factor=scale_factor,
motion_score=motion,
)
return call
@torch.no_grad()
def denoise_with_scheduler(
x: torch.Tensor,
dit_fn: DitFn,
scheduler: Any,
*,
channels: int,
is_piflow: bool,
on_step: StepCallback | None = None,
) -> torch.Tensor:
"""Denoise one chunk after set_timesteps; keep conditioning channels fixed.
Promote predictions to fp32 so flow-Euler does not downcast its running
state to the bf16 prediction dtype. pi-Flow already preserves the state dtype."""
state = x[..., :channels].float()
cond = x[..., channels:]
for step, t in enumerate(scheduler.timesteps):
model_time = t.to(device=x.device, dtype=torch.float32).expand(state.shape[0])
hidden_states = torch.cat([state, cond], dim=-1) if cond.shape[-1] else state
prediction = dit_fn(hidden_states, model_time, step)
if not is_piflow:
prediction = prediction.float()
state = scheduler.step(prediction, t, state, return_dict=False)[0]
if on_step is not None:
on_step()
def _denoise_chunk(x, dit, scheduler, dit_spec, spec, step_context, on_step):
# each chunk starts at step zero and releases its GPU temporaries before the next
scheduler._step_index = None
scheduler.set_begin_index(0)
state = x[..., : dit_spec.in_visual_dim].float()
cond = x[..., dit_spec.in_visual_dim :]
frames, height, width = x.shape[1:4]
rope_pos = [
torch.arange(frames, device=x.device),
torch.arange(height // dit_spec.patch_size[1], device=x.device),
torch.arange(width // dit_spec.patch_size[2], device=x.device),
]
motion = (
torch.full((1,), 900.0, device=x.device, dtype=state.dtype)
if dit_spec.use_motion_score and not spec.is_piflow
else None
)
with bf16_autocast(x.device):
for step, t in enumerate(scheduler.timesteps):
model_time = t.to(device=x.device, dtype=torch.float32).expand(
state.shape[0]
)
hidden_states = (
torch.cat([state, cond], dim=-1) if cond.shape[-1] else state
)
with step_context(step) if step_context is not None else nullcontext():
prediction = dit(
hidden_states,
model_time,
rope_pos,
scale_factor=spec.scale_factor,
motion_score=motion,
)
# Euler otherwise downcasts the running state to the prediction dtype
if not spec.is_piflow:
prediction = prediction.float()
state = scheduler.step(prediction, t, state, return_dict=False)[0]
if on_step is not None:
on_step()
return state.float()
__all__ = [
"DitFn",
"DitSpec",
"PiflowParams",
"SamplingSpec",
"StepCallback",
"StepContext",
"bf16_autocast",
"cast_to_module_dtype",
"check_sampler_options",
"denoise_with_scheduler",
"euler_start_timestep",
"make_dit_fn",
"module_dtype",
]
def denoise_chunks(
chunks: Sequence[torch.Tensor],
dit: Callable[..., torch.Tensor],
scheduler: Any,
*,
dit_spec: DitSpec,
spec: SamplingSpec,
device: torch.device,
step_context: Callable[[int], AbstractContextManager] | None = None,
on_step: Callable[[], None] | None = None,
) -> list[torch.Tensor]:
"""Denoise CPU chunks sequentially, preserving conditioning and fp32 state."""
if spec.is_piflow:
scheduler.set_timesteps(spec.num_steps, device=device)
else:
capped = spec.cap_noise_timestep and dit_spec.instruct_type in (
"noise",
"hybrid",
"hybrid_anchor",
)
start = spec.lq_noise_scale if capped else 1.0
sigmas = torch.linspace(start, 0.0, spec.num_steps + 1)[:-1].tolist()
results = []
for chunk in chunks:
if not spec.is_piflow:
scheduler.set_timesteps(sigmas=sigmas, device=device)
results.append(
_denoise_chunk(
chunk.to(device), dit, scheduler, dit_spec, spec, step_context, on_step
).cpu()
)
return results
@@ -0,0 +1,463 @@
# SPDX-License-Identifier: Apache-2.0
"""SR pipeline stages, grouped by component while retaining separate offload lifetimes."""
from __future__ import annotations
import os
from functools import partial
import torch
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
resolve_target_hw,
)
from sglang.multimodal_gen.runtime.distributed import (
get_decode_parallel_group_coordinator,
get_local_torch_device,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.models.vaes.common import (
can_install_spatial_shard_parallel_decode,
has_decode_parallel_world,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_CHUNKS_KEY,
SR_DENOISED_KEY,
SR_DIT_SPEC_KEY,
SR_LR_LATENT_KEY,
SR_PLAN_KEY,
SR_REQUESTED_HW_KEY,
SR_SAMPLING_SPEC_KEY,
SR_TILES_KEY,
SR_TILING_SCALE_KEY,
SR_VIDEO_KEY,
build_dit_spec,
build_sampling_spec,
uses_latent_path,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
denoise_chunks,
module_dtype,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
decode_chunks,
encode_pixel_tile,
encode_video_to_lr_latent,
plan_tiles,
prepare_tile_latents,
stitch_tiles,
upscale_lr_latent_tile,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
VAE_SPATIAL_FACTOR,
pad_to_spatial_factor,
pre_upscale_video,
resolve_scale_request,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
SOURCE_AUDIO_SAMPLE_RATE,
DecodedClip,
decode_clip,
extract_source_audio,
resize_to_target,
synthetic_clip,
to_output_video,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
StageValidators as V,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
VerificationResult,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision import (
precision_to_dtype,
resolve_precision,
)
WARMUP_CLIP_FRAMES = 9
WARMUP_CLIP_HW = (256, 384)
class Kandinsky6SRInputStage(PipelineStage):
"""Decode [T, 3, H, W] uint8 video, align fps/frames/size and retain source audio.
Record requested dimensions before VAE padding so output can remove it."""
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
tiling_scale, pre_upscale = resolve_scale_request(batch.sr_resolution_scale)
if batch.is_warmup:
height, width = WARMUP_CLIP_HW
clip = DecodedClip(
frames=synthetic_clip(
height=height, width=width, frames=WARMUP_CLIP_FRAMES
),
fps=24,
source_fps=24.0,
)
else:
path = batch.sampling_params.source_video_path()
if path is None or not os.path.isfile(path):
raise ValueError(f"Kandinsky6 SR input video not found: {path!r}")
self.log_info("Decoding %s", path)
clip = decode_clip(path)
video = clip.frames
if pre_upscale != 1.0:
video = pre_upscale_video(video, pre_upscale, VAE_SPATIAL_FACTOR)
frames = video.shape[0]
# pad every scale for the VAE; output crops padding without resizing source pixels
video, (requested_h, requested_w) = pad_to_spatial_factor(
video, VAE_SPATIAL_FACTOR
)
batch.extra[SR_VIDEO_KEY] = video
batch.extra[SR_TILING_SCALE_KEY] = tiling_scale
batch.extra[SR_REQUESTED_HW_KEY] = (requested_h, requested_w)
batch.fps = clip.fps
batch.num_frames = frames
batch.height = requested_h * tiling_scale
batch.width = requested_w * tiling_scale
if not batch.is_warmup:
audio = extract_source_audio(
path,
sample_rate=SOURCE_AUDIO_SAMPLE_RATE,
max_seconds=frames / clip.fps,
)
batch.audio = torch.from_numpy(audio) if audio is not None else None
batch.audio_sample_rate = (
SOURCE_AUDIO_SAMPLE_RATE if audio is not None else None
)
return batch
class Kandinsky6SREncodeStage(PipelineStage):
"""Encode the whole clip only for the LU path; pixel mode encodes tiles later."""
def __init__(self, vae, latent_upscaler=None) -> None:
super().__init__()
self.vae = vae
# Only consulted to pick the path (does the bank have this scale?).
self.latent_upscaler = latent_upscaler
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
return [
ComponentUse(
self._component_stage_name(stage_name),
"vae",
phase="encode_video",
target_dtype=vae_dtype,
start_at_stage_entry=False,
)
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
if not uses_latent_path(self.latent_upscaler, tiling_scale):
return batch
video = batch.extra.pop(SR_VIDEO_KEY)
with self.use_declared_component(
component_name="vae", module=self.vae, phase="encode_video"
) as vae:
assert vae is not None
self.vae = vae
lr_latent = encode_video_to_lr_latent(
video, vae, device=get_local_torch_device()
)
batch.extra[SR_LR_LATENT_KEY] = lr_latent.cpu()
return batch
class Kandinsky6SRLatentPrepStage(PipelineStage):
"""LU-or-pixel tile encode + upscale -> initial noisy latent, one chunk at a time."""
def __init__(self, vae, transformer, latent_upscaler=None, scheduler=None) -> None:
super().__init__()
self.vae = vae
self.transformer = transformer # arch-config / dtype only: never called here
self.latent_upscaler = latent_upscaler
self.scheduler = (
scheduler # resolved sampler validation only: never stepped here
)
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
stage_name = self._component_stage_name(stage_name)
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
uses: list[ComponentUse] = []
if self.latent_upscaler is not None:
uses.append(
ComponentUse(
stage_name,
"latent_upscaler",
phase="upscale_tiles",
target_dtype=torch.bfloat16,
)
)
uses.append(
ComponentUse(
stage_name, "vae", phase="encode_tiles", target_dtype=vae_dtype
)
)
return uses
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
arch = server_args.pipeline_config.dit_config.arch_config
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
seed = batch.seed if isinstance(batch.seed, int) else batch.seed[0]
spec = build_sampling_spec(
arch=arch,
tiling_scale=tiling_scale,
seed=seed,
num_steps=batch.num_inference_steps,
tiles_batch_size=batch.sr_tiles_batch_size,
tile_min_overlap=batch.sr_tile_min_overlap,
scheduler=self.scheduler,
)
dit_spec = build_dit_spec(self.transformer)
device = get_local_torch_device()
latent_path = uses_latent_path(self.latent_upscaler, tiling_scale)
source = batch.extra.pop(SR_LR_LATENT_KEY if latent_path else SR_VIDEO_KEY)
factor = VAE_SPATIAL_FACTOR if latent_path else 1
plan = plan_tiles(
frame_hw=tuple(size * factor for size in source.shape[-2:]),
visual_size=spec.visual_size,
tiling_scale=tiling_scale,
tile_min_overlap=spec.tile_min_overlap,
)
with self.use_declared_component(
component_name="latent_upscaler" if latent_path else "vae",
module=self.latent_upscaler if latent_path else self.vae,
phase="upscale_tiles" if latent_path else "encode_tiles",
) as component:
assert component is not None
if latent_path:
self.latent_upscaler = component
tile_encoder = partial(
upscale_lr_latent_tile,
upscale_fn=partial(component.upscale, scale=tiling_scale),
lu_dtype=module_dtype(component),
scaling_factor=self.vae.scaling_factor,
device=device,
)
else:
self.vae = component
tile_encoder = partial(
encode_pixel_tile,
vae=component,
scaling_factor=component.scaling_factor,
device=device,
)
chunks = prepare_tile_latents(
source,
plan,
tile_encoder=tile_encoder,
latent_path=latent_path,
dit_spec=dit_spec,
spec=spec,
device=device,
)
batch.extra[SR_PLAN_KEY] = plan
batch.extra[SR_CHUNKS_KEY] = chunks
batch.extra[SR_SAMPLING_SPEC_KEY] = spec
batch.extra[SR_DIT_SPEC_KEY] = dit_spec
return batch
class Kandinsky6SRDenoisingStage(DenoisingStage):
"""Runs the bundle's scheduler over every tile chunk prepared by the latent-prep stage."""
def _owns_compile_warmup_lifecycle(self) -> bool:
# forward wraps the tiled loop in the shared compile warmup lifecycle
return True
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
"""Validate tile-chunk state instead of the shared text/CFG loop inputs."""
result = VerificationResult()
result.add_check(
"extra[SR_SAMPLING_SPEC_KEY]",
batch.extra.get(SR_SAMPLING_SPEC_KEY),
V.not_none,
)
result.add_check(
"extra[SR_DIT_SPEC_KEY]", batch.extra.get(SR_DIT_SPEC_KEY), V.not_none
)
result.add_check(
"extra[SR_CHUNKS_KEY]", batch.extra.get(SR_CHUNKS_KEY), V.list_not_empty
)
return result
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
result.add_check(
"extra[SR_DENOISED_KEY]", batch.extra.get(SR_DENOISED_KEY), V.list_not_empty
)
return result
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
dit_dtype = precision_to_dtype(
server_args.pipeline_config.dit_precision, "dit_precision"
)
return [
ComponentUse(
self._component_stage_name(stage_name),
"transformer",
phase="denoise_tiles",
target_dtype=dit_dtype,
preferred_ready_after_request=True,
memory_intensive=True,
)
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
with self._offload_for_torch_compile_warmup(batch):
return self._denoise_tiles(batch, server_args)
def _denoise_tiles(self, batch: Req, server_args: ServerArgs) -> Req:
spec = batch.extra.pop(SR_SAMPLING_SPEC_KEY)
dit_spec = batch.extra.pop(SR_DIT_SPEC_KEY)
chunks = batch.extra.pop(SR_CHUNKS_KEY)
device = get_local_torch_device()
self._maybe_enable_cache_dit_and_torch_compile(spec.num_steps, batch)
with self.use_declared_component(
component_name="transformer", module=self.transformer, phase="denoise_tiles"
) as transformer:
assert transformer is not None
self.transformer = transformer
total = spec.num_steps * len(chunks)
with self.progress_bar(
total=total, batch=batch, desc="Kandinsky6 SR denoising"
) as progress:
denoised = denoise_chunks(
chunks,
transformer,
self.scheduler,
dit_spec=dit_spec,
spec=spec,
device=device,
step_context=lambda step: set_forward_context(
current_timestep=step, attn_metadata=None, forward_batch=batch
),
on_step=progress.update,
)
self._finish_active_component_use()
batch.extra[SR_DENOISED_KEY] = denoised
return batch
class Kandinsky6SRDecodeStage(PipelineStage):
"""VAE-decodes every denoised chunk into row-major uint8 tiles (``[3, T, Hb, Wb]``)."""
def __init__(self, vae) -> None:
super().__init__()
self.vae = vae
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
return [
ComponentUse(
self._component_stage_name(stage_name),
"vae",
phase="decode_tiles",
target_dtype=vae_dtype,
)
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
denoised = batch.extra.pop(SR_DENOISED_KEY)
device = get_local_torch_device()
group = None
if (
server_args.pipeline_config.vae_config.use_parallel_tiling
and has_decode_parallel_world()
and not can_install_spatial_shard_parallel_decode(
server_args.pipeline_config.vae_config
)
):
group = get_decode_parallel_group_coordinator()
with self.use_declared_component(
component_name="vae", module=self.vae, phase="decode_tiles"
) as vae:
assert vae is not None
self.vae = vae
tiles = decode_chunks(
denoised,
vae,
scaling_factor=vae.scaling_factor,
device=device,
group=group,
)
batch.extra[SR_TILES_KEY] = tiles
return batch
class Kandinsky6SROutputStage(PipelineStage):
"""Hann-blends the decoded tiles, restores the requested output size, applies the
optional delivery resize and returns ``output = [1, 3, T, H, W]`` (fp16 in [0, 1])
plus the mono source audio.
CPU only: every model phase already ran. ``batch.height`` / ``batch.width`` are set
to the final size, ``batch.fps`` was set by the input stage (and now also carried on
the returned ``OutputBatch``, so a worker-resampled fps reaches the client-side save
path unchanged).
"""
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
tiles = batch.extra.pop(SR_TILES_KEY)
plan = batch.extra.pop(SR_PLAN_KEY)
video = stitch_tiles(tiles, plan)
del tiles
requested_hw = batch.extra.pop(SR_REQUESTED_HW_KEY, None)
if requested_hw is not None:
requested_h, requested_w = requested_hw
video = video[
...,
: requested_h * plan.tiling_scale,
: requested_w * plan.tiling_scale,
]
target_hw = resolve_target_hw(
batch.sr_target_resolution,
tuple(video.shape[-2:]),
batch.sr_target_resize_mode,
)
if target_hw is not None:
video = resize_to_target(video, target_hw)
batch.height, batch.width = video.shape[-2:]
return OutputBatch(
output=to_output_video(video),
audio=batch.audio,
audio_sample_rate=batch.audio_sample_rate,
fps=batch.fps,
metrics=batch.metrics,
usage=batch.usage,
)
@@ -6,10 +6,10 @@ Each chunk uses seed + its first tile index, matching the reference.
"""
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Any
import msgspec
import torch
from torch.nn import functional as F
from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latents import (
@@ -18,14 +18,8 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
DitSpec,
SamplingSpec,
StepCallback,
StepContext,
bf16_autocast,
cast_to_module_dtype,
check_sampler_options,
denoise_with_scheduler,
euler_start_timestep,
make_dit_fn,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
RESOLUTIONS,
@@ -36,7 +30,6 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
extract_all_tiles,
latent_tile_grid_from_pixel_grid,
stitch_tiles_hanning,
upsample_tiles_to_base,
)
LatentUpscaleFn = Callable[[torch.Tensor], torch.Tensor]
@@ -147,188 +140,57 @@ def encode_pixel_tile(
return latent.squeeze(0).permute(1, 2, 3, 0).float() * scaling_factor
def build_chunk_latent(
lq_tiles: Sequence[torch.Tensor],
*,
seed: int,
dit_spec: DitSpec,
spec: SamplingSpec,
device: torch.device,
) -> torch.Tensor:
"""Initial noisy latent ``[B, T', H', W', C]`` of one chunk of scaled LQ tiles."""
lq = torch.cat(list(lq_tiles), dim=0)
batch = len(lq_tiles)
x = build_initial_latent(
instruct_type=dit_spec.instruct_type,
visual_cond=dit_spec.visual_cond,
in_visual_dim=dit_spec.in_visual_dim,
lq_latent=lq,
batch_size=batch,
duration=lq.shape[0] // batch,
height=lq.shape[1],
width=lq.shape[2],
device=device,
seed=seed,
lq_noise_scale=spec.lq_noise_scale,
lq_noise_type=spec.lq_noise_type,
lq_channel_noise_scale=spec.lq_channel_noise_scale,
# The DiT's own loaded dtype (reference: cast_to_module_dtype(dit, ...) before
# building the initial latent), not a hard-coded fp32 regardless of precision.
dtype=dit_spec.dtype,
)
return x.reshape(batch, -1, *x.shape[1:])
@torch.no_grad()
def prepare_lu_tile_latents(
lr_latent: torch.Tensor,
def prepare_tile_latents(
source: torch.Tensor,
plan: TilePlan,
*,
upscale_fn: LatentUpscaleFn,
lu_dtype: torch.dtype | None,
scaling_factor: float,
tile_encoder: Callable[[torch.Tensor], torch.Tensor],
latent_path: bool,
dit_spec: DitSpec,
spec: SamplingSpec,
device: torch.device,
spatial_factor: int = VAE_SPATIAL_FACTOR,
) -> list[torch.Tensor]:
"""LU path: initial latents (CPU) of every chunk from the whole-video latent."""
latent_grid = latent_tile_grid_from_pixel_grid(plan.pixel_grid, spatial_factor)
tiles = extract_all_tiles(lr_latent, latent_grid)
chunks: list[torch.Tensor] = []
for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size):
lq_tiles = [
upscale_lr_latent_tile(
tile,
upscale_fn,
lu_dtype=lu_dtype,
scaling_factor=scaling_factor,
device=device,
)
for tile in tiles[start:stop]
]
chunk = build_chunk_latent(
lq_tiles,
seed=spec.seed + start,
dit_spec=dit_spec,
spec=spec,
device=device,
)
chunks.append(chunk.cpu())
return chunks
@torch.no_grad()
def prepare_pixel_tile_latents(
video: torch.Tensor,
plan: TilePlan,
*,
vae: torch.nn.Module,
scaling_factor: float,
dit_spec: DitSpec,
spec: SamplingSpec,
device: torch.device,
) -> list[torch.Tensor]:
"""Pixel path: bilinear tile -> KVAE encode -> initial latents (CPU) per chunk."""
tiles = extract_all_tiles(video, plan.pixel_grid)
chunks: list[torch.Tensor] = []
for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size):
enlarged = upsample_tiles_to_base(tiles[start:stop], *plan.base_hw)
lq_tiles = [
encode_pixel_tile(tile, vae, scaling_factor=scaling_factor, device=device)
for tile in enlarged
]
chunk = build_chunk_latent(
lq_tiles,
seed=spec.seed + start,
dit_spec=dit_spec,
spec=spec,
device=device,
)
chunks.append(chunk.cpu())
return chunks
def reset_scheduler_for_chunk(scheduler: Any) -> None:
"""Reset step and begin indices so each tile chunk starts at the same timestep."""
scheduler._step_index = None
scheduler.set_begin_index(0)
@torch.no_grad()
def denoise_chunk(
x: torch.Tensor,
dit: Callable[..., torch.Tensor],
scheduler: Any,
*,
dit_spec: DitSpec,
spec: SamplingSpec,
step_context: StepContext | None = None,
on_step: StepCallback | None = None,
) -> torch.Tensor:
"""Denoise one [B, T, H, W, C] chunk, resetting the configured scheduler."""
is_piflow = spec.piflow is not None
check_sampler_options(
piflow=spec.piflow,
cap_noise_timestep=spec.cap_noise_timestep,
instruct_type=dit_spec.instruct_type,
"""Encode/upscale one tile batch at a time; keep prepared chunks on CPU."""
grid = (
latent_tile_grid_from_pixel_grid(plan.pixel_grid, spatial_factor)
if latent_path
else plan.pixel_grid
)
if not is_piflow:
# The Euler start timestep is only meaningful for the scheduler this chunk is about to
# run: re-derive it instead of baking a stale one into ``set_timesteps`` up front.
start = euler_start_timestep(
cap_noise_timestep=spec.cap_noise_timestep,
lq_noise_scale=spec.lq_noise_scale,
tiles = extract_all_tiles(source, grid)
chunks = []
for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size):
inputs = tiles[start:stop]
if not latent_path:
inputs = [
F.interpolate(
tile.float(),
size=plan.base_hw,
mode="bilinear",
align_corners=False,
).permute(0, 2, 3, 1)
for tile in inputs
]
lq_tiles = [tile_encoder(tile) for tile in inputs]
lq = torch.cat(lq_tiles, dim=0)
x = build_initial_latent(
instruct_type=dit_spec.instruct_type,
visual_cond=dit_spec.visual_cond,
in_visual_dim=dit_spec.in_visual_dim,
lq_latent=lq,
device=device,
seed=spec.seed + start,
lq_noise_scale=spec.lq_noise_scale,
lq_noise_type=spec.lq_noise_type,
lq_channel_noise_scale=spec.lq_channel_noise_scale,
# conditioning noise must round in the loaded DiT dtype
dtype=dit_spec.dtype,
)
sigmas = torch.linspace(start, 0.0, spec.num_steps + 1)[:-1].tolist()
scheduler.set_timesteps(sigmas=sigmas, device=x.device)
reset_scheduler_for_chunk(scheduler)
dit_fn = make_dit_fn(
dit,
latent_frames_hw=tuple(x.shape[1:4]),
patch_size=dit_spec.patch_size,
scale_factor=spec.scale_factor,
use_motion_score=dit_spec.use_motion_score and not is_piflow,
step_context=step_context,
)
with bf16_autocast(x.device):
return denoise_with_scheduler(
x,
dit_fn,
scheduler,
channels=dit_spec.in_visual_dim,
is_piflow=is_piflow,
on_step=on_step,
)
def denoise_chunks(
chunks: Sequence[torch.Tensor],
dit: Callable[..., torch.Tensor],
scheduler: Any,
*,
dit_spec: DitSpec,
spec: SamplingSpec,
device: torch.device,
step_context: StepContext | None = None,
on_step: StepCallback | None = None,
) -> list[torch.Tensor]:
"""Denoise chunks sequentially on device and return CPU latents."""
if spec.piflow is not None:
scheduler.set_timesteps(spec.piflow.nfe, device=device)
return [
denoise_chunk(
chunk.to(device),
dit,
scheduler,
dit_spec=dit_spec,
spec=spec,
step_context=step_context,
on_step=on_step,
).cpu()
for chunk in chunks
]
chunk = x.reshape(len(lq_tiles), -1, *x.shape[1:])
del lq, x
chunks.append(chunk.cpu())
return chunks
@torch.no_grad()
@@ -58,23 +58,6 @@ def extract_all_tiles(
return tiles
def _normalize_blend(acc: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
"""Normalize Hann-weighted tiles without flooring positive weights.
Corner weights are tiny but valid; clamping them darkens tile boundaries.
Pixels outside all tiles have zero weight and remain zero."""
covered = weight > 0
safe_weight = torch.where(covered, weight, torch.ones_like(weight))
return torch.where(covered, acc / safe_weight, torch.zeros_like(acc))
def hanning_window_2d(h: int, w: int, device: torch.device) -> torch.Tensor:
"""Create a [h, w] Hann window with nonzero endpoints."""
wy = torch.hann_window(h + 2, device=device)[1:-1]
wx = torch.hann_window(w + 2, device=device)[1:-1]
return wy[:, None] * wx[None, :]
def stitch_tiles_hanning(
tiles: list[torch.Tensor],
grid: TileGrid,
@@ -88,8 +71,11 @@ def stitch_tiles_hanning(
hr_tile_h, hr_tile_w = first.shape[2], first.shape[3]
device = first.device
window = hanning_window_2d(hr_tile_h, hr_tile_w, device)
window = window.unsqueeze(0).unsqueeze(0) # [1, 1, hr_tile_h, hr_tile_w]
# nonzero endpoints keep the outermost tile pixels covered
wy = torch.hann_window(hr_tile_h + 2, device=device)[1:-1]
wx = torch.hann_window(hr_tile_w + 2, device=device)[1:-1]
window = (wy[:, None] * wx[None, :])[None, None]
del wy, wx
out_h = original_h * scale
out_w = original_w * scale
@@ -106,7 +92,10 @@ def stitch_tiles_hanning(
weight_acc[:, :, y : y + hr_tile_h, x : x + hr_tile_w] += window
tile_idx += 1
return _normalize_blend(pred_acc, weight_acc)
# preserve tiny positive corner weights; only uncovered pixels stay zero
covered = weight_acc > 0
safe_weight = torch.where(covered, weight_acc, torch.ones_like(weight_acc))
return torch.where(covered, pred_acc / safe_weight, torch.zeros_like(pred_acc))
def axis_positions_even(
@@ -179,18 +168,6 @@ def closest_base_resolution(
return min(resolutions[visual_size], key=lambda hw: abs(hw[1] / hw[0] - ratio))
def upsample_tiles_to_base(
raw_tiles: list[torch.Tensor], base_h: int, base_w: int
) -> list[torch.Tensor]:
"""Bilinearly resize ``[T, C, h, w]`` tiles to ``base_h x base_w`` and return ``[T, H, W, C]``."""
return [
functional.interpolate(
tile.float(), size=(base_h, base_w), mode="bilinear", align_corners=False
).permute(0, 2, 3, 1)
for tile in raw_tiles
]
def resolve_scale_request(scale: float) -> tuple[int, float]:
"""Decompose total scale into (integer tiling scale, pixel pre-upscale)."""
requested = float(scale)
@@ -240,9 +217,3 @@ def pad_to_spatial_factor(
return video, (height, width)
padded = functional.pad(video, (0, pad_w, 0, pad_h), mode="replicate")
return padded, (height, width)
def crop_to_hw(video: torch.Tensor, hw: tuple[int, int]) -> torch.Tensor:
"""Crop [..., H, W] to hw from the top-left, undoing bottom/right padding."""
height, width = hw
return video[..., :height, :width]
@@ -1,24 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
"""Streaming video / audio input and output helpers of Kandinsky 6 video SR.
# Frame selection and resize adapted from the Kandinsky 6 SR inference reference
# (k6_video / sr_core, Apache-2.0).
"""Bounded-memory SR video/audio I/O, preserving reference frame selection and resize."""
Decoding follows the reference CLI (``read_video_tchw_uint8`` ->
``resample_to_target_fps`` -> ``clip_to_aligned_frames``) frame for frame, but streams:
only the frames that survive fps resampling and the 121-frame cap are converted to RGB
and kept, and decoding stops as soon as the outcome is known. PyAV is imported lazily
(it is part of the ``diffusion`` extra).
"""
import logging
import msgspec
import numpy as np
import torch
from torch.nn import functional as F
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
MAX_NUM_FRAMES,
RESAMPLE_FPS_TOLERANCE,
TARGET_FPS,
clip_to_aligned_frames,
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
ASPECT_MISMATCH_TOLERANCE,
)
logger = logging.getLogger(__name__)
# trained on 5 s at 24 fps; select at most 121 frames, aligned to 1+8k
TARGET_FPS = 24
MAX_NUM_FRAMES = 121
RESAMPLE_FPS_TOLERANCE = 1.5
SOURCE_AUDIO_SAMPLE_RATE = 44100
OUTPUT_FRAME_CHUNK = 8
@@ -32,16 +32,10 @@ class DecodedClip(msgspec.Struct, frozen=True):
class FrameSelector:
"""Streaming form of ``resample_to_target_fps`` + the first-121-frames cap.
"""Stream reference indices round(i*step), capped at int(total/step) and max_frames.
Feed source frames in order: :meth:`keep` says whether frame ``index`` is one of the
selected ones (call :meth:`mark_kept` for each), :meth:`enough` says when no further
frame can change the result, and :meth:`limit` trims the selection once the source
length is known. For a source faster than the target the selected indices are
``round(i * step)``, ``step = source_fps / target_fps``, of which there are
``int(total / step)`` (those below ``total``), exactly as in the reference;
otherwise every frame is selected.
"""
In-tolerance and slower sources keep every frame. Stop only once the source
length guarantees that no retained frame will be trimmed by limit()."""
def __init__(
self,
@@ -116,9 +110,9 @@ def decode_clip(
kept = selector.limit(seen, len(frames))
if kept == 0:
raise ValueError(f"Video {path} has no readable frames.")
video = torch.stack(frames[:kept]).contiguous()
kept = 1 + 8 * ((kept - 1) // 8)
return DecodedClip(
frames=clip_to_aligned_frames(video, max_frames),
frames=torch.stack(frames[:kept]).contiguous(),
fps=selector.effective_fps,
source_fps=source_fps,
)
@@ -188,3 +182,48 @@ def to_output_video(video: torch.Tensor) -> torch.Tensor:
(chunk.float() + 0.5) / 255.0
).to(torch.float16)
return out
def resize_to_target(video: torch.Tensor, target_hw: tuple[int, int]) -> torch.Tensor:
"""Downscale [C, T, H, W] uint8 video in bounded-memory chunks, without cropping."""
channels, frames, height, width = video.shape
target_h, target_w = target_hw
if (height, width) == (target_h, target_w):
return video
if target_h > height or target_w > width:
raise ValueError(
f"target {target_w}x{target_h} exceeds the SR result {width}x{height}; "
"pick a lower tier or a higher --resolution-scale for this source."
)
source_ratio, target_ratio = width / height, target_w / target_h
if abs(source_ratio - target_ratio) / target_ratio > ASPECT_MISMATCH_TOLERANCE:
logger.warning(
"Aspect mismatch: SR %sx%s is %.3f, target %sx%s is %.3f; resizing "
"anisotropically (no crop), the picture will be squeezed by %.1f%%.",
width,
height,
source_ratio,
target_w,
target_h,
target_ratio,
abs(source_ratio / target_ratio - 1) * 100,
)
out = torch.empty((channels, frames, target_h, target_w), dtype=torch.uint8)
for start in range(0, frames, OUTPUT_FRAME_CHUNK):
chunk = video[:, start : start + OUTPUT_FRAME_CHUNK].permute(1, 0, 2, 3).float()
resized = F.interpolate(
chunk, size=target_hw, mode="bilinear", antialias=True, align_corners=False
)
out[:, start : start + OUTPUT_FRAME_CHUNK] = (
resized.clamp(0, 255).round().to(torch.uint8).permute(1, 0, 2, 3)
)
logger.info(
"Final output: %sx%s -> %sx%s (downscale x%.2f / x%.2f)",
width,
height,
target_w,
target_h,
width / target_w,
height / target_h,
)
return out
@@ -1,136 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# Vendored from sr_core (parity-tested against the k6_video reference); adapted from
# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0).
"""Frame selection and bounded-memory output resize for Kandinsky 6 SR."""
from __future__ import annotations
import logging
import torch
from torch.nn import functional as f # noqa: N812
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
ASPECT_MISMATCH_TOLERANCE,
)
logger = logging.getLogger(__name__)
# --- Model / pipeline contract (from the reference ``constants.py``; NOT configurable) ---
# The SR model was trained on 5 s clips at 24 fps -> 121 pixel frames
# (= 1 + 8*15); feeding more frames or another rate is out of contract.
TARGET_FPS: int = 24
MAX_NUM_FRAMES: int = 121
# Treat ``|src_fps - TARGET_FPS| < tolerance`` as already-at-target (no
# resample): covers 23.98 (NTSC) and 25.00 sources.
RESAMPLE_FPS_TOLERANCE: float = 1.5
# Frames per interpolate call when downscaling the output (a 2K clip is
# ~2.4 GB as uint8, so the resize walks the clip in chunks).
RESIZE_FRAME_CHUNK: int = 8
def select_frame_indices(
total_frames: int, src_fps: float, target_fps: float
) -> list[int]:
"""Select rounded frame indices to downsample src_fps to target_fps."""
step = src_fps / target_fps
indices = [round(i * step) for i in range(int(total_frames / step))]
return [i for i in indices if i < total_frames]
def resample_to_target_fps(
video: torch.Tensor,
src_fps: float,
target_fps: int = TARGET_FPS,
) -> tuple[torch.Tensor, int]:
"""Downsample high-fps inputs; retain near-target and lower-fps inputs.
Return [T, C, H, W] video and its effective playback rate."""
if abs(src_fps - target_fps) < RESAMPLE_FPS_TOLERANCE:
return video, round(src_fps)
if src_fps > target_fps:
indices = select_frame_indices(video.shape[0], src_fps, target_fps)
logger.warning(
"Source fps %.2f > target %sfps: downsampling %s frames -> %s (fixed-stride); "
"source temporal detail beyond %sfps is discarded.",
src_fps,
target_fps,
video.shape[0],
len(indices),
target_fps,
)
return video[indices], target_fps
logger.warning(
"Source fps %.2f < target %sfps: keeping native frames (no minterpolate upsample); "
"output is mildly out of distribution.",
src_fps,
target_fps,
)
return video, round(src_fps)
def clip_to_aligned_frames(
video: torch.Tensor, max_num_frames: int = MAX_NUM_FRAMES
) -> torch.Tensor:
"""Cap the frame count, then floor-align to 1+8k; reject empty video."""
if max_num_frames > 0:
video = video[:max_num_frames]
aligned = 1 + 8 * ((video.shape[0] - 1) // 8) if video.shape[0] > 0 else 0
if aligned == 0:
msg = "Video has no readable frames."
raise ValueError(msg)
return video[:aligned]
def resize_to_target(video: torch.Tensor, target_hw: tuple[int, int]) -> torch.Tensor:
"""Downscale [C, T, H, W] uint8 video in bounded-memory chunks, without cropping."""
_c, _t, height, width = video.shape
target_h, target_w = target_hw
if (height, width) == (target_h, target_w):
return video
if target_h > height or target_w > width:
msg = (
f"target {target_w}x{target_h} exceeds the SR result {width}x{height} — that would "
f"upscale. Pick a lower tier or a higher --resolution-scale for this source."
)
raise ValueError(msg)
source_ratio, target_ratio = width / height, target_w / target_h
if abs(source_ratio - target_ratio) / target_ratio > ASPECT_MISMATCH_TOLERANCE:
logger.warning(
"Aspect mismatch: SR %sx%s is %.3f, target %sx%s is %.3f — resizing anisotropically "
"(no crop), the picture will be squeezed by %.1f%%.",
width,
height,
source_ratio,
target_w,
target_h,
target_ratio,
abs(source_ratio / target_ratio - 1) * 100,
)
channels, frames = video.shape[0], video.shape[1]
out = torch.empty((channels, frames, target_h, target_w), dtype=torch.uint8)
for start in range(0, frames, RESIZE_FRAME_CHUNK):
chunk = video[:, start : start + RESIZE_FRAME_CHUNK].permute(1, 0, 2, 3).float()
resized = f.interpolate(
chunk,
size=(target_h, target_w),
mode="bilinear",
antialias=True,
align_corners=False,
)
out[:, start : start + RESIZE_FRAME_CHUNK] = (
resized.clamp(0, 255).round().to(torch.uint8).permute(1, 0, 2, 3)
)
logger.info(
"Final output: %sx%s -> %sx%s (downscale x%.2f / x%.2f)",
width,
height,
target_w,
target_h,
width / target_w,
height / target_h,
)
return out
@@ -5,7 +5,28 @@ from types import SimpleNamespace
import pytest
from sglang.multimodal_gen.runtime import server_args as server_args_module
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.server_args import set_global_server_args
from sglang.srt.utils.network import get_free_port_below_ephemeral
@pytest.fixture(scope="module")
def single_process_model_parallel():
with pytest.MonkeyPatch.context() as patch:
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT=str(get_free_port_below_ephemeral()),
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
patch.setenv(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
yield
def _make_unit_server_args():
@@ -3,6 +3,7 @@
import copy
from collections.abc import Callable, Mapping, Sequence
from functools import partial
from types import SimpleNamespace
from typing import Any
@@ -21,6 +22,7 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
Kandinsky6SRTransformer3DModel,
)
@@ -37,29 +39,30 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
DitSpec,
SamplingSpec,
denoise_chunks,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
LatentUpscaleFn,
decode_chunks,
denoise_chunks,
encode_pixel_tile,
encode_video_to_lr_latent,
plan_tiles,
prepare_lu_tile_latents,
prepare_pixel_tile_latents,
prepare_tile_latents,
stitch_tiles,
upscale_lr_latent_tile,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
RESOLUTIONS,
VAE_SPATIAL_FACTOR,
)
# Flat (legacy) transformer config with a pi-Flow DX head: 3 grids of 4 channels.
# official transformer layout: total output width, scheduler configured separately
TINY_DIT = dict(
in_visual_dim=4,
in_text_dim=8,
in_text_dim2=8,
time_dim=16,
out_visual_dim=4,
out_visual_dim=12,
patch_size=[1, 2, 2],
model_dim=32,
ff_dim=64,
@@ -69,15 +72,16 @@ TINY_DIT = dict(
visual_cond=False,
instruct_type="noise",
use_text=False,
attribute_overrides=None,
sr_params=dict(visual_size=[512], scale_factor={"512": [1.0, 1.0, 1.0]}),
)
TINY_PIFLOW = dict(
n_grid=3,
piflow_nfe=2,
piflow_num_policy_substeps=8,
piflow_final_step_size_scale=0.5,
piflow_shift=5.0,
piflow_eps=1e-6,
attribute_overrides={"instruct_type": "noise"},
sr_visual_size=[512],
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
nfe=2,
num_policy_substeps=8,
final_step_size_scale=0.5,
shift=5.0,
eps=1e-6,
)
TINY_KVAE = dict(
ch=8,
@@ -211,9 +215,7 @@ def super_resolve(
) -> torch.Tensor:
"""Tiled SR of a ``[T, C, H, W]`` uint8 video -> uint8 ``[3, T, H*s, W*s]``.
``upscale_fn`` selects the LU path; without it the pixel path runs. ``scheduler`` is the
already-resolved effective scheduler (``run_spec.effective_scheduler``) -- a real loaded
component, or one synthesized from ``spec.piflow`` / ``spec.scheduler_scale`` when none fits.
``upscale_fn`` selects the LU path; without it the pixel path runs.
"""
device = torch.device(device)
common = dict(
@@ -230,30 +232,43 @@ def super_resolve(
lr_latent.shape[-1] * spatial_factor,
)
plan = plan_tiles(frame_hw=frame_hw, **common)
chunks = prepare_lu_tile_latents(
lr_latent,
plan,
source = lr_latent
tile_encoder = partial(
upscale_lr_latent_tile,
upscale_fn=upscale_fn,
lu_dtype=lu_dtype,
scaling_factor=scaling_factor,
dit_spec=dit_spec,
spec=spec,
device=device,
spatial_factor=spatial_factor,
)
else:
plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common)
chunks = prepare_pixel_tile_latents(
video,
plan,
source = video
tile_encoder = partial(
encode_pixel_tile,
vae=vae,
scaling_factor=scaling_factor,
dit_spec=dit_spec,
spec=spec,
device=device,
)
chunks = prepare_tile_latents(
source,
plan,
tile_encoder=tile_encoder,
latent_path=upscale_fn is not None,
dit_spec=dit_spec,
spec=spec,
device=device,
spatial_factor=spatial_factor,
)
denoised = denoise_chunks(
chunks, dit, scheduler, dit_spec=dit_spec, spec=spec, device=device
chunks,
dit,
scheduler,
dit_spec=dit_spec,
spec=spec,
device=device,
step_context=lambda step: set_forward_context(
current_timestep=step, attn_metadata=None
),
)
tiles = decode_chunks(denoised, vae, scaling_factor=scaling_factor, device=device)
return stitch_tiles(tiles, plan)
@@ -15,6 +15,7 @@ reference).
from __future__ import annotations
import pytest
import torch
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
@@ -78,12 +79,16 @@ def test_audio_vae_weights_are_not_renormalized_after_checkpoint_load():
vae = Kandinsky6AudioVAE(config)
assert vae.vae._weights_normalized is True
# `load_state_dict(vae.state_dict())` mirrors VAELoader's real call
# (`vae.load_state_dict(loaded, strict=strict_load)`) with the module's
# own current weights standing in for "a checkpoint's own weights" --
# exactly what happens when a real checkpoint's stored tensors are
# loaded: no code path scales or otherwise disturbs them afterward.
vae.load_state_dict(vae.state_dict())
checkpoint = dict(vae.state_dict())
checkpoint["vae.data_mean"] = torch.full((1, 80, 1), -2.5)
checkpoint["vae.data_std"] = torch.full((1, 80, 1), 1.25)
vae.load_state_dict(checkpoint, strict=True)
torch.testing.assert_close(vae.vae.data_mean, checkpoint["vae.data_mean"])
torch.testing.assert_close(vae.vae.data_std, checkpoint["vae.data_std"])
for key in ("vae.data_mean", "vae.data_std"):
incomplete = {name: value for name, value in checkpoint.items() if name != key}
with pytest.raises(RuntimeError, match=key):
vae.load_state_dict(incomplete, strict=True)
latents = torch.randn(1, vae.vae.embed_dim, 4)
mel_a = vae.decode(latents.clone())
@@ -4,7 +4,6 @@
from __future__ import annotations
import importlib.util
import os
from types import SimpleNamespace
import pytest
@@ -23,10 +22,6 @@ from sglang.multimodal_gen.runtime.cache.cache_dit_integration import (
refresh_context_on_transformer,
)
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGBranch, CFGPolicy
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.layers.attention.selector import (
global_force_attn_backend_context_manager,
)
@@ -45,22 +40,7 @@ from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.server_args import get_global_server_args
from sglang.multimodal_gen.runtime.utils import precision
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
# Kandinsky6's feed-forward uses ColumnParallelLinear/RowParallelLinear, which
# need a (size-1) TP group to construct -- matches test_kandinsky6_sr_stages.py's
# fixture of the same name.
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29513",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
pytestmark = pytest.mark.usefixtures("single_process_model_parallel")
def _randomize(module: torch.nn.Module, seed: int) -> None:
@@ -70,29 +50,18 @@ def _randomize(module: torch.nn.Module, seed: int) -> None:
param.copy_(torch.randn(param.shape, generator=generator) * 0.05)
def test_time_embeddings_forward_under_bf16_linear_weights():
# Minimal, pinpoint reproduction of the review comment: a bf16-weight
# Kandinsky6TimeEmbeddings fed the (always-fp32) sinusoidal embedding must not
# raise, and must return bf16 (the Linear weights' dtype).
module = Kandinsky6TimeEmbeddings(model_dim=16, time_dim=8)
@pytest.mark.parametrize(
"modulation", [False, True], ids=["time-embedding", "modulation"]
)
def test_bf16_projection_accepts_fp32_conditioning(modulation):
module = (
Kandinsky6Modulation(time_dim=8, model_dim=16, num_params=9)
if modulation
else Kandinsky6TimeEmbeddings(model_dim=16, time_dim=8)
)
_randomize(module, seed=0)
module = module.to(torch.bfloat16)
out = module(torch.tensor([0.0, 500.0, 999.0]))
assert out.dtype == torch.bfloat16
assert torch.isfinite(out.float()).all()
def test_modulation_forward_under_bf16_linear_weights():
# Kandinsky6Modulation (the AdaLN projection every block/out-layer uses) has the
# identical autocast(dtype=float32)-doesn't-cast-weights bug; fed an fp32 time
# embedding with bf16 weights, it must not raise either.
module = Kandinsky6Modulation(time_dim=8, model_dim=16, num_params=9)
module = module.to(torch.bfloat16)
out = module(torch.randn(2, 8)) # fp32 input, as Kandinsky6TimeEmbeddings used to
inputs = torch.randn(2, 8) if modulation else torch.tensor([0.0, 500.0, 999.0])
out = module.to(torch.bfloat16)(inputs)
assert out.dtype == torch.bfloat16
assert torch.isfinite(out.float()).all()
@@ -297,7 +266,6 @@ def test_bcg_joint_cfg_output_lifetime_and_text_shape_fallback(monkeypatch, use_
t_expand=torch.tensor([500.0], device="cuda"),
visual_rope_pos=[torch.arange(2, device="cuda") for _ in range(3)],
scale_factor=(1.0, 1.0, 1.0),
sparse_params=None,
visual_token_type_ids=None,
)
@@ -360,71 +328,10 @@ def test_bcg_joint_cfg_output_lifetime_and_text_shape_fallback(monkeypatch, use_
runner.reset()
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason=(
"the DiT's fused LayerNormScaleShift/RMSNormScaleShift requires a real "
"CUDA/XPU tensor on any CUDA-capable host (see module docstring); bf16 "
"numerics are only meaningful on real hardware anyway"
),
)
def test_kandinsky6_dit_real_bf16_forward_does_not_crash():
"""Real (non-mocked) forward pass of a tiny Kandinsky6Transformer3DModel whose
weights are bf16 -- the exact configuration the SGLang PR review comment flagged
as broken (a fp32 sinusoidal embedding feeding a bf16 Linear)."""
device = torch.device("cuda")
config = Kandinsky6VideoAudioConfig()
arch_kwargs = _tiny_multimodal_arch_kwargs()
config.update_model_arch(dict(arch_kwargs))
dit = Kandinsky6Transformer3DModel(config, dict(arch_kwargs)).eval()
_randomize(dit, seed=1)
dit = dit.to(device=device, dtype=torch.bfloat16)
batch, duration, height, width = 1, 2, 2, 2
text_len, audio_len = 3, 5
hidden_states = torch.randn(
batch, duration, height, width, 4, dtype=torch.bfloat16, device=device
)
hidden_states_audio = torch.randn(
batch, audio_len, 4, dtype=torch.bfloat16, device=device
)
encoder_hidden_states = torch.randn(
batch, text_len, 8, dtype=torch.bfloat16, device=device
)
pooled_projections = torch.randn(batch, 8, dtype=torch.bfloat16, device=device)
# Scheduler timesteps are plain fp32 (not cast to the model's weight dtype by the
# caller), same as every real sampling loop -- exactly the dtype that used to
# mismatch the bf16 Linear.
timestep = torch.tensor([500.0], device=device)
visual_rope_pos = (
torch.arange(duration, device=device),
torch.arange(height, device=device),
torch.arange(width, device=device),
)
text_rope_pos = torch.arange(text_len, device=device)
video_out, audio_out = dit(
hidden_states=hidden_states,
hidden_states_audio=hidden_states_audio,
encoder_hidden_states=encoder_hidden_states,
timestep=timestep,
pooled_projections=pooled_projections,
visual_rope_pos=visual_rope_pos,
text_rope_pos=text_rope_pos,
)
assert video_out.shape == (batch, duration, height, width, 4)
assert audio_out.shape == (batch, audio_len, 4)
assert video_out.dtype == torch.bfloat16
assert audio_out.dtype == torch.bfloat16
assert torch.isfinite(video_out.float()).all()
assert torch.isfinite(audio_out.float()).all()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("separate_cfg", [False, True])
@torch.no_grad()
def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg):
def test_bf16_joint_forward_and_cache_dit_lifetime(separate_cfg):
config = Kandinsky6VideoAudioConfig()
arch = _tiny_multimodal_arch_kwargs() | {"num_visual_blocks": 4}
config.update_model_arch(arch)
@@ -445,18 +352,19 @@ def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg):
outputs = []
for step in range(3):
for branch in range(2 if separate_cfg else 1):
outputs.append(
dit(
**(
inputs
| {
"pooled_projections": inputs["pooled_projections"]
+ branch
}
),
timestep=torch.tensor([500.0 - step * 100], device="cuda"),
with set_forward_context(current_timestep=step, attn_metadata=None):
outputs.append(
dit(
**(
inputs
| {
"pooled_projections": inputs["pooled_projections"]
+ branch
}
),
timestep=torch.tensor([500.0 - step * 100], device="cuda"),
)
)
)
return outputs
def assert_exact(actual, expected):
@@ -465,6 +373,12 @@ def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg):
torch.testing.assert_close(tensor, target, rtol=0, atol=0)
reference = run_steps()
for video, audio in reference:
assert video.shape == inputs["hidden_states"].shape
assert audio.shape == inputs["hidden_states_audio"].shape
for tensor in (video, audio):
assert tensor.dtype == torch.bfloat16
assert torch.isfinite(tensor).all()
enable_cache_on_transformer(
dit,
CacheDitConfig(
@@ -1,127 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""GAP 6 regression: the Kandinsky6 IT2VA conditioning-image preprocessor
must aspect-preserving-resize-then-centre-crop, matching the diffusers
reference's ``encode_i2va_first_frame``
(``pipeline_kandinsky6_ti2va.py``, ``scale = min(src_h/height,
src_w/width)``; resize to ``(src_h/scale, src_w/scale)``; centre-crop to
``(height, width)``), not stretch-resize the whole source image to the
target box (which distorts its aspect ratio whenever the source doesn't
already match the target aspect ratio).
"""
from __future__ import annotations
import numpy as np
import PIL.Image
import pytest
import torch
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import (
Kandinsky6ImageEncodingStage,
)
def _gradient_image(width: int, height: int) -> PIL.Image.Image:
"""A deterministic, per-pixel-distinguishable RGB image (not a flat
color), so resize/crop geometry is actually verifiable pixel-for-pixel.
"""
xs = np.linspace(0, 255, width, dtype=np.uint8)
ys = np.linspace(0, 255, height, dtype=np.uint8)
grid = np.zeros((height, width, 3), dtype=np.uint8)
grid[..., 0] = xs[None, :]
grid[..., 1] = ys[:, None]
grid[..., 2] = 128
return PIL.Image.fromarray(grid, mode="RGB")
@pytest.mark.parametrize(
"src_h,src_w,height,width",
[
(1080, 1920, 512, 768), # 16:9 source, 3:2 target (audit's own case)
(1920, 1080, 512, 768), # 9:16 source, 3:2 target (audit's own case)
(1024, 1536, 512, 768), # already 3:2 -- matching aspect
],
)
def test_cover_resize_dims_matches_diffusers_formula(src_h, src_w, height, width):
# encode_i2va_first_frame: scale = min(src_h/height, src_w/width);
# resize to (int(src_h/scale), int(src_w/scale)), then centre-crop.
scale = min(src_h / height, src_w / width)
expected = (int(src_h / scale), int(src_w / scale))
assert (
Kandinsky6ImageEncodingStage._cover_resize_dims(src_h, src_w, height, width)
== expected
)
# The resize always covers the target box on both axes (never smaller).
assert expected[0] >= height
assert expected[1] >= width
@pytest.mark.parametrize(
"src_w,src_h,min_mean_abs_diff",
[
(1920, 1080, 0.015), # 16:9 landscape source (audit's own case)
(1080, 1920, 0.05), # 9:16 portrait source (audit's own case)
],
)
def test_mismatched_aspect_pil_image_is_cropped_not_stretched(
src_w, src_h, min_mean_abs_diff
):
# The model's documented default target resolution (512x768, 2:3)
# against realistic phone/webcam source aspect ratios.
height, width = 512, 768
image = _gradient_image(src_w, src_h)
produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width)
assert produced.shape == (1, 3, height, width)
# Independent oracle: the diffusers reference's own aspect-preserving
# resize + centre-crop, computed directly via PIL, without calling any
# of the code under test.
scale = min(src_h / height, src_w / width)
new_h, new_w = int(src_h / scale), int(src_w / scale)
resized = image.resize((new_w, new_h), resample=PIL.Image.BILINEAR)
top, left = (new_h - height) // 2, (new_w - width) // 2
cropped = resized.crop((left, top, left + width, top + height))
expected_np = np.array(cropped).astype(np.float32) / 255.0
expected = torch.from_numpy(expected_np).permute(2, 0, 1) * 2.0 - 1.0
assert torch.allclose(produced[0], expected, atol=2.0 / 255.0)
# The deleted stretch-resize procedure (plain `.resize((width, height))`,
# no crop) produces a materially different image whenever the source
# aspect ratio doesn't already match the target -- proving this is a
# real behavior change, not a no-op refactor.
stretched = image.resize((width, height), resample=PIL.Image.BILINEAR)
stretched_np = np.array(stretched).astype(np.float32) / 255.0
stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0
mean_abs_diff = (produced[0] - stretched_t).abs().mean().item()
assert mean_abs_diff > min_mean_abs_diff
def test_matching_aspect_pil_image_resize_only_agrees_with_stretch():
# A sanity check on the oracle itself: when the source already matches
# the target aspect ratio, cover-resize-then-crop and plain stretch
# resize agree closely (no crop actually removes anything) -- so the
# large the divergence asserted above is specifically an aspect-ratio
# effect, not a general property of the two code paths.
height, width = 64, 96
src_w, src_h = 288, 192 # exactly 3:2, matching height:width == 64:96 == 2:3...
# (use width:height == 96:64 == 3:2 to match src_w:src_h == 288:192 == 3:2)
image = _gradient_image(src_w, src_h)
produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width)
stretched = image.resize((width, height), resample=PIL.Image.BILINEAR)
stretched_np = np.array(stretched).astype(np.float32) / 255.0
stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0
mean_abs_diff = (produced[0] - stretched_t).abs().mean().item()
assert mean_abs_diff < 0.02
def test_tensor_input_uses_cover_crop_geometry_too():
height, width = 64, 96
src_h, src_w = 108, 192
tensor = torch.rand(1, 3, src_h, src_w) * 2.0 - 1.0 # already in [-1, 1]
produced = Kandinsky6ImageEncodingStage._preprocess(tensor, height, width)
assert produced.shape[-2:] == (height, width)
@@ -19,10 +19,6 @@ from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
Kandinsky6TI2VAPipelineConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.layers.attention.selector import (
global_force_attn_backend_context_manager,
)
@@ -47,24 +43,12 @@ from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import (
TINY_KVAE,
TINY_LU_MODEL,
)
from sglang.srt.utils.network import get_free_port_below_ephemeral
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.fixture(autouse=True)
def single_gpu(monkeypatch, default_global_server_args):
if not model_parallel_is_initialized():
port = get_free_port_below_ephemeral()
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT=str(port),
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
monkeypatch.setenv(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
def single_gpu(monkeypatch, default_global_server_args, single_process_model_parallel):
monkeypatch.setattr(
precision._mixed_precision_state,
"state",
@@ -84,15 +84,6 @@ def _make_pipeline() -> Kandinsky6TI2VAPipeline:
return pipeline
def test_pipeline_identity_and_required_modules():
assert Kandinsky6TI2VAPipeline.pipeline_name == "Kandinsky6TI2VAPipeline"
assert Kandinsky6TI2VAPipeline.is_video_pipeline is True
assert Kandinsky6TI2VAPipeline._required_config_modules == EXPECTED_REQUIRED_MODULES
# No separate "vocoder" module -- audio_vae bundles the mel-VAE decoder
# and the BigVGAN vocoder in one checkpoint component.
assert "vocoder" not in Kandinsky6TI2VAPipeline._required_config_modules
def test_disagg_role_rejects_non_monolithic_deployment():
pipeline = object.__new__(Kandinsky6TI2VAPipeline)
@@ -107,17 +98,16 @@ def test_disagg_role_rejects_non_monolithic_deployment():
def test_create_pipeline_stages_produces_documented_order():
pipeline = _make_pipeline()
server_args = MagicMock()
Kandinsky6TI2VAPipeline.create_pipeline_stages(pipeline, server_args)
actual_types = [type(stage) for stage in pipeline.stages]
expected_types = [stage_cls for _, stage_cls in EXPECTED_STAGE_ORDER]
assert actual_types == expected_types
actual_names = list(pipeline._stage_name_mapping.keys())
expected_names = [name for name, _ in EXPECTED_STAGE_ORDER]
assert actual_names == expected_names
assert pipeline.pipeline_name == "Kandinsky6TI2VAPipeline"
assert pipeline.is_video_pipeline is True
assert pipeline._required_config_modules == EXPECTED_REQUIRED_MODULES
pipeline.create_pipeline_stages(MagicMock())
assert [type(stage) for stage in pipeline.stages] == [
stage_cls for _, stage_cls in EXPECTED_STAGE_ORDER
]
assert [
(name, type(stage)) for name, stage in pipeline._stage_name_mapping.items()
] == EXPECTED_STAGE_ORDER
# get_module wiring: each stage received the mocked module it asked for.
denoising_stage = pipeline._stage_name_mapping["denoising_stage"]
@@ -224,7 +214,6 @@ def test_parallel_cfg_uses_serial_arithmetic_for_video_and_audio(monkeypatch):
t_expand=torch.zeros(1),
visual_rope_pos=[],
scale_factor=(1.0, 1.0, 1.0),
sparse_params=None,
visual_token_type_ids=None,
)
for result, reference in zip(actual, expected, strict=True):
@@ -237,10 +226,7 @@ def test_parallel_cfg_uses_serial_arithmetic_for_video_and_audio(monkeypatch):
def test_denoising_rejects_guidance_other_than_one_with_a_piflow_scheduler(
guidance_scale,
):
"""The distilled checkpoint's ``PiflowScheduler`` has no classifier-free guidance,
so the run must not silently ignore a guidance the caller asked for; the
Diffusers pipeline raises the same way ("PiflowScheduler requires
guidance_weight=1.0")."""
"""The distilled checkpoint must reject unsupported CFG, not silently ignore it."""
scheduler = _pro_distill_scheduler()
batch = SimpleNamespace(
timesteps=scheduler.timesteps,
@@ -272,11 +258,6 @@ def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance(
text_lens = []
class _FakeDiT(torch.nn.Module):
"""A real ``nn.Module`` (not a bare function): ``Kandinsky6DenoisingStage``
now subclasses the shared ``DenoisingStage``, whose ``__init__`` walks
``self.transformer.modules()`` to infer the attention backend -- a plain
callable has no ``.modules()``."""
def forward(self, **kwargs):
assert tuple(kwargs["scale_factor"]) == (1.0, 2.0, 2.0)
text_lens.append(kwargs["encoder_hidden_states"].shape[1])
@@ -309,28 +290,13 @@ def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance(
is_warmup=True,
sampling_params=Kandinsky6TI2VASamplingParams(quality="lossless"),
)
arch = SimpleNamespace(
attention_engine="auto",
in_visual_dim=video_channels,
patch_size=(1, 2, 2),
scale_factor=(1.0, 2.0, 2.0),
)
pipeline_config = SimpleNamespace(
dit_config=SimpleNamespace(arch_config=arch),
dit_precision="fp32",
vae_config=SimpleNamespace(
arch_config=SimpleNamespace(spatial_compression_ratio=8)
),
get_pos_prompt_embeds=lambda batch: batch.prompt_embeds[0],
# CFG is disabled in this test (guidance_scale=1.0,
# do_classifier_free_guidance=False), so the policy never needs a
# negative branch, but Kandinsky6DenoisingStage still reads
# `cfg_policy` / `get_classifier_free_guidance_scale` off the config
# unconditionally to build the (single-branch) CFGPolicy.
cfg_policy=CFGPolicy(),
get_classifier_free_guidance_scale=lambda _batch, guidance_scale: (
guidance_scale
),
pipeline_config = Kandinsky6TI2VAPipelineConfig(dit_precision="fp32")
pipeline_config.dit_config.update_model_arch(
dict(
in_visual_dim=video_channels,
patch_size=(1, 2, 2),
scale_factor=(1.0, 2.0, 2.0),
)
)
server_args = SimpleNamespace(
pipeline_config=pipeline_config, enable_cfg_parallel=False
@@ -120,4 +120,4 @@ def test_gate_sum_preserves_fp32_intermediates():
gate = torch.randn(2, 1, 1152, dtype=x.dtype, device=x.device)[..., 256:384]
expected = (x.float() + gate.float() * update.float()).to(x.dtype)
assert not torch.equal(expected, x + gate * update)
assert torch.equal(kandinsky6._apply_gate_sum(x, update, gate), expected)
assert torch.equal(kandinsky6.residual_gate_fp32(x, update, gate), expected)
@@ -5,6 +5,7 @@ import argparse
import re
import pytest
from kandinsky6_sr_tiny_components import TINY_DIT
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRArchConfig,
@@ -25,85 +26,10 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
TINY_DIT = dict(
in_visual_dim=4,
in_text_dim=8,
in_text_dim2=8,
time_dim=16,
out_visual_dim=4,
patch_size=[1, 2, 2],
model_dim=32,
ff_dim=64,
num_text_blocks=0,
num_visual_blocks=2,
axes_dims=[8, 4, 4],
visual_cond=False,
instruct_type="noise",
attention_params={"512": {"type": "flash"}},
use_text=False,
)
TINY_PIFLOW = dict(
nfe=2,
num_policy_substeps=8,
final_step_size_scale=0.5,
shift=5.0,
n_grid=3,
eps=1e-6,
)
SR_PARAMS = dict(
scale_factor={"512": [1.0, 1.0, 1.0]},
visual_size=[512],
scheduler_scale=5.0,
lq_noise_scale=0.7,
lq_noise_type="ddpm",
lq_channel_noise_scale=0.0,
cap_noise_timestep=False,
fps=24,
)
def flat_config(**changes):
"""A converted ``transformer/config.json`` (pi-Flow DX head) with ``changes`` applied."""
flat = dict(TINY_DIT, _class_name="Kandinsky6SRTransformer3DModel", n_grid=3)
flat.update(
piflow_nfe=2,
piflow_num_policy_substeps=8,
piflow_final_step_size_scale=0.5,
piflow_shift=5.0,
piflow_eps=1e-6,
attribute_overrides={"instruct_type": "noise"},
sr_visual_size=[512],
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
sr_scheduler_scale=5.0,
sr_lq_noise_scale=0.7,
sr_lq_noise_type="ddpm",
sr_lq_channel_noise_scale=0.0,
sr_cap_noise_timestep=False,
sr_fps=24,
)
flat.update(changes)
return flat
def official_config(**changes):
"""An official ``transformer/config.json``: constructor kwargs, ``sr_params``, the TOTAL DX head
width in ``out_visual_dim`` (``base * n_grid``) and no ``n_grid`` / ``piflow_*`` keys.
"""
official = {
key: value for key, value in TINY_DIT.items() if key not in ("out_visual_dim",)
}
official.update(
out_visual_dim=TINY_DIT["out_visual_dim"] * TINY_PIFLOW["n_grid"],
attribute_overrides=None,
sr_params=dict(SR_PARAMS),
)
official.update(changes)
return official
def parse(flat):
def parse(**changes):
config = Kandinsky6SRDitConfig()
config.update_model_arch(flat)
config.update_model_arch(dict(TINY_DIT, **changes))
return config.arch_config
@@ -111,8 +37,7 @@ def parse(flat):
# DiT arch config
# --------------------------------------------------------------------------- #
def test_arch_config_parses_an_official_config_and_maps_checkpoint_names():
arch = parse(official_config())
assert arch.n_grid == 1 and not arch.is_piflow # pi-Flow lives in the scheduler
arch = parse()
assert arch.out_visual_dim == 12 and arch.attribute_overrides == {}
assert arch.patch_size == (1, 2, 2) and arch.axes_dims == (8, 4, 4)
assert (arch.hidden_size, arch.num_attention_heads) == (32, 2)
@@ -146,7 +71,7 @@ def test_arch_config_parses_an_official_config_and_maps_checkpoint_names():
def test_unknown_sr_params_keys_are_rejected():
with pytest.raises(ValueError, match="sr_params has unsupported keys"):
parse(official_config(sr_params={"fps": 24, "new_knob": 1}))
parse(sr_params={"fps": 24, "new_knob": 1})
@pytest.mark.parametrize(
@@ -159,6 +84,7 @@ def test_unknown_sr_params_keys_are_rejected():
({"attribute_overrides": {"n_grid": 2}}, ValueError, "n_grid"),
({"attribute_overrides": {"instruct_type": "bogus"}}, ValueError, "bogus"),
({"piflow_eps": None}, ValueError, "piflow_eps"),
({"n_grid": 3}, ValueError, "n_grid"),
({"patch_size": [2, 2, 2]}, ValueError, "patch_size"),
({"sr_lq_noise_type": "gauss"}, ValueError, "gauss"),
],
@@ -171,25 +97,28 @@ def test_unsupported_or_unknown_config_fails_loudly_naming_the_key(
keys raise ValueError (``update_model_arch`` alone would park unknown keys in
``extra_attrs``)."""
with pytest.raises(error, match=flag):
parse(flat_config(**changes))
parse(**changes)
def test_sparse_attention_request_is_detected_from_attention_params():
assert parse(flat_config()).requested_sparse_attention() is None
nabla = flat_config(attention_params={"512": {"type": "nabla", "P": 0.9}})
assert parse(nabla).requested_sparse_attention() == "nabla"
# an override can switch it off again
overridden = flat_config(
assert parse().requested_sparse_attention() is None
assert (
parse(
attention_params={"512": {"type": "nabla", "P": 0.9}}
).requested_sparse_attention()
== "nabla"
)
overridden = parse(
attention_params={"512": {"type": "nabla"}},
attribute_overrides={"attention_params": {"512": {"type": "flash"}}},
)
assert parse(overridden).requested_sparse_attention() is None
assert overridden.requested_sparse_attention() is None
def test_default_arch_config_is_valid_and_text_free():
"""Registry / ``PipelineConfig.from_kwargs`` instantiate the default config."""
arch = Kandinsky6SRArchConfig()
assert arch.use_text is False and not arch.is_piflow and arch.n_grid == 1
assert arch.use_text is False
# --------------------------------------------------------------------------- #
@@ -37,13 +37,13 @@ from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import (
_SpatialChunkedConv3d,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
Kandinsky6SRDecodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_DENOISED_KEY,
SR_TILES_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import (
Kandinsky6SRDecodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
decode_chunks,
)
@@ -1,12 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
"""Check SR prompt admission and request fields without running a GPU scheduler.
"""Prompt admission and SR fields through the real JSON and multipart endpoints."""
JSON uses TestClient; multipart tests call the parsed endpoint directly. The
full-checkpoint test_server_kandinsky6_sr.py also exercises actual file uploads.
"""
import asyncio
import os
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
@@ -22,269 +16,80 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
)
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.runtime.entrypoints.openai import video_api
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import VideoResponse
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
resolve_sampling_params_cls,
)
from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import (
_build_video_sampling_params,
_multipart_video_extras,
)
_SR_EXTRA_FIELDS = frozenset(
{
"sr_resolution_scale",
"sr_tiles_batch_size",
"sr_tile_min_overlap",
"sr_target_resolution",
"sr_target_resize_mode",
}
)
def _fake_server_args(**overrides) -> SimpleNamespace:
base = dict(
@pytest.mark.parametrize("multipart", [False, True], ids=["json", "multipart"])
@pytest.mark.parametrize("prompt_required", [False, True], ids=["sr", "generic"])
def test_prompt_admission_and_sr_fields(tmp_path, multipart, prompt_required):
args = SimpleNamespace(
pipeline_config=Kandinsky6SRPipelineConfig(),
pipeline_class_name="Kandinsky6SRPipeline",
backend="auto",
model_id=None,
model_path="kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers",
model_path="kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers",
served_model_name="kandinsky6-sr-test",
input_save_path=os.path.expanduser(
"~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/uploads"
),
output_path=os.path.expanduser(
"~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/output"
),
input_save_path=str(tmp_path / "uploads"),
output_path=str(tmp_path / "outputs"),
attention_backend_config={},
enable_trace=False,
comfyui_mode=False,
num_gpus=1,
)
base.update(overrides)
return SimpleNamespace(**base)
def _patched_global_server_args(server_args):
return patch(
"sglang.multimodal_gen.runtime.server_args.server_args._global_server_args",
server_args,
assert resolve_sampling_params_cls(args) is Kandinsky6SRSamplingParams
params_cls = SamplingParams if prompt_required else Kandinsky6SRSamplingParams
fields = dict(
sr_resolution_scale=4,
sr_tiles_batch_size=3,
sr_tile_min_overlap=0.3,
sr_target_resolution="hd",
sr_target_resize_mode="fit",
)
assert Kandinsky6SRSamplingParams.video_request_extra_fields() == fields.keys()
assert SamplingParams.video_request_extra_fields() == frozenset()
payload = dict(video_path="unused.mp4", **fields)
request = (
{"files": {key: (None, str(value)) for key, value in payload.items()}}
if multipart
else {"json": payload}
)
# --------------------------------------------------------------------------- #
# The two declared pieces (SamplingParams-level)
# --------------------------------------------------------------------------- #
def test_prompt_optional_defaults_false_and_sr_overrides_it():
"""A generic pipeline keeps requiring ``prompt``; only the SR params opt out."""
assert SamplingParams.prompt_optional is False
assert Kandinsky6SRSamplingParams.prompt_optional is True
@pytest.mark.parametrize(
"sampling_params_cls, status",
[(Kandinsky6SRSamplingParams, 200), (SamplingParams, 400)],
)
def test_json_prompt_requirement_matches_pipeline(
tmp_path, sampling_params_cls, status
):
app = FastAPI()
app.include_router(video_api.router)
server_args = _fake_server_args(
input_save_path=str(tmp_path / "uploads"), output_path=str(tmp_path / "outputs")
)
dispatch = AsyncMock()
with (
_patched_global_server_args(server_args),
patch.object(
video_api, "resolve_sampling_params_cls", return_value=sampling_params_cls
patch(
"sglang.multimodal_gen.runtime.server_args.server_args._global_server_args",
args,
),
patch.object(video_api, "resolve_sampling_params_cls", return_value=params_cls),
patch.object(video_api, "_dispatch_job_async", dispatch),
TestClient(app) as client,
):
response = client.post(
"/v1/videos",
json={"video_path": "unused.mp4", "sr_resolution_scale": 4},
)
assert response.status_code == status, response.text
if status == 200:
response = client.post("/v1/videos", **request)
assert response.status_code == (400 if prompt_required else 200), response.text
if prompt_required:
dispatch.assert_not_called()
else:
assert response.json()["status"] == "queued"
dispatch.assert_awaited_once()
_, batch = dispatch.call_args.args
assert batch.prompt == ""
assert batch.sampling_params.sr_resolution_scale == 4
else:
dispatch.assert_not_called()
assert isinstance(batch.sampling_params, Kandinsky6SRSamplingParams)
assert batch.prompt == "" and batch.video_path == "unused.mp4"
assert {key: vars(batch.sampling_params)[key] for key in fields} == fields
def test_sr_declares_its_cli_flags_as_multipart_extra_fields():
assert Kandinsky6SRSamplingParams.video_request_extra_fields() == _SR_EXTRA_FIELDS
# The base default (what every other model still gets) stays empty.
assert SamplingParams.video_request_extra_fields() == frozenset()
# --------------------------------------------------------------------------- #
# The real multipart-field allowlist (video_api._multipart_video_extras)
# --------------------------------------------------------------------------- #
def test_multipart_extras_keep_sr_fields_only_for_the_sr_params_class():
raw_form = {
"video_path": "unused.mp4",
"sr_resolution_scale": "4",
"sr_tiles_batch_size": "3",
"not_a_declared_field": "dropped",
}
sr_extras = _multipart_video_extras(
raw_form,
@pytest.mark.parametrize("params_cls", [Kandinsky6SRSamplingParams, SamplingParams])
def test_multipart_allowlist_drops_undeclared_fields(params_cls):
extras = video_api._multipart_video_extras(
{"sr_resolution_scale": "4", "sr_tiles_batch_size": "3", "undeclared": "1"},
extra_body=None,
extra_params=None,
sampling_params_cls=Kandinsky6SRSamplingParams,
sampling_params_cls=params_cls,
)
assert sr_extras["sr_resolution_scale"] == 4 # JSON-decoded, not the raw "4"
assert sr_extras["sr_tiles_batch_size"] == 3
assert "not_a_declared_field" not in sr_extras
# A model without the override (the pre-fix behaviour) drops the sr_* fields too.
base_extras = _multipart_video_extras(
raw_form,
extra_body=None,
extra_params=None,
sampling_params_cls=SamplingParams,
assert extras == (
{"sr_resolution_scale": 4, "sr_tiles_batch_size": 3}
if params_cls is Kandinsky6SRSamplingParams
else {}
)
assert "sr_resolution_scale" not in base_extras
assert "sr_tiles_batch_size" not in base_extras
# --------------------------------------------------------------------------- #
# The registry-backed class resolution the admission gate reads
# --------------------------------------------------------------------------- #
def test_video_sampling_params_cls_resolves_to_sr_for_the_sr_pipeline():
server_args = _fake_server_args()
assert resolve_sampling_params_cls(server_args) is Kandinsky6SRSamplingParams
# --------------------------------------------------------------------------- #
# End-to-end request parsing: a prompt-less request field set -> sampling params
# --------------------------------------------------------------------------- #
def test_build_video_sampling_params_accepts_no_prompt_and_carries_sr_fields():
"""The real kwargs-building code (``_build_video_sampling_params``), given a
request built the way the multipart branch builds one for a prompt-optional
pipeline (``prompt=prompt or ""`` + the surviving sr_* extras), must construct a
real ``Kandinsky6SRSamplingParams`` with those fields set and no prompt error."""
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
VideoGenerationsRequest,
)
server_args = _fake_server_args()
raw_form = {
"video_path": "unused.mp4",
"sr_resolution_scale": "4",
"sr_tiles_batch_size": "3",
}
extras = _multipart_video_extras(
raw_form,
extra_body=None,
extra_params=None,
sampling_params_cls=Kandinsky6SRSamplingParams,
)
req = VideoGenerationsRequest(
prompt="", # what video_api.py substitutes for a missing prompt
video_path=extras.pop("video_path"),
**extras,
)
with _patched_global_server_args(server_args):
sampling_params = _build_video_sampling_params("gap1-request-id", req)
assert isinstance(sampling_params, Kandinsky6SRSamplingParams)
assert sampling_params.prompt == ""
assert sampling_params.sr_resolution_scale == 4
assert sampling_params.sr_tiles_batch_size == 3
# --------------------------------------------------------------------------- #
# The full endpoint, called directly (see module docstring for why not TestClient)
# --------------------------------------------------------------------------- #
class _FakeMultipartRequest:
"""Duck-types the slice of ``fastapi.Request`` ``create_video`` reads directly."""
def __init__(self, form: dict):
self.headers = {"content-type": "multipart/form-data; boundary=x"}
self._form = form
async def form(self):
return self._form
def test_multipart_sr_request_without_prompt_is_accepted():
"""A multipart SR request with no ``prompt`` field must be admitted (no 400), the
``sr_*`` fields must reach the queued request, and it must fail on the pre-fix
code: the old unconditional ``if not prompt: raise HTTPException(...)`` gate would
reject this before any of the rest of ``create_video`` ran."""
server_args = _fake_server_args()
request = _FakeMultipartRequest(
{"sr_resolution_scale": "4", "sr_tiles_batch_size": "3"}
)
dispatched: list[tuple] = []
async def _fake_dispatch(job_id, batch, **kwargs):
dispatched.append((job_id, batch, kwargs))
async def run():
with (
_patched_global_server_args(server_args),
patch.object(video_api, "_dispatch_job_async", side_effect=_fake_dispatch),
):
response = await video_api.create_video(
request,
prompt=None, # <-- the field under test: no prompt supplied
enhance_prompt=None,
task_type=None,
x264_preset=None,
perf_dump_path=None,
input_reference=None,
reference_url=None,
video_reference=None,
video_url=None,
video_path="unused.mp4",
model=None,
n=1,
num_outputs_per_prompt=None,
seconds=None,
size=None,
fps=None,
num_frames=None,
seed=None,
generator_device="cuda",
negative_prompt=None,
guidance_scale=None,
guidance_scale_2=None,
true_cfg_scale=None,
num_inference_steps=None,
max_sequence_length=None,
flow_shift=None,
enable_teacache=None,
enable_frame_interpolation=None,
frame_interpolation_exp=None,
frame_interpolation_scale=None,
frame_interpolation_model_path=None,
enable_upscaling=None,
upscaling_model_path=None,
upscaling_scale=None,
output_quality=None,
output_compression=None,
output_path=None,
extra_params=None,
extra_body=None,
)
await asyncio.sleep(0)
return response
response = asyncio.run(run())
assert isinstance(response, VideoResponse)
assert response.status == "queued"
assert len(dispatched) == 1
_, batch, _ = dispatched[0]
assert batch.prompt == ""
assert batch.sampling_params.sr_resolution_scale == 4
assert batch.sampling_params.sr_tiles_batch_size == 3
@@ -3,11 +3,10 @@
import copy
import json
import os
import pytest
import torch
from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL
from kandinsky6_sr_tiny_components import TINY_DIT, TINY_KVAE, TINY_LU_MODEL
from safetensors.torch import load_file, save_file
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
@@ -19,10 +18,6 @@ from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.loader.component_loaders import (
component_loader as component_loader_module,
)
@@ -34,6 +29,7 @@ from sglang.multimodal_gen.runtime.loader.fsdp_load import (
load_model_from_full_model_state_dict,
)
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
Kandinsky6SRTransformer3DModel,
)
@@ -42,39 +38,7 @@ from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscale
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
TINY_DIT = dict(
in_visual_dim=4,
in_text_dim=8,
in_text_dim2=8,
time_dim=16,
out_visual_dim=4,
patch_size=[1, 2, 2],
model_dim=32,
ff_dim=64,
num_text_blocks=0,
num_visual_blocks=2,
axes_dims=[8, 4, 4],
visual_cond=False,
instruct_type="noise",
use_text=False,
n_grid=3,
attribute_overrides={"instruct_type": "noise"},
)
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29508",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
pytestmark = pytest.mark.usefixtures("single_process_model_parallel")
@pytest.fixture(autouse=True)
@@ -138,100 +102,48 @@ def _load_through_the_real_loader(checkpoint, **changes):
return model
# --------------------------------------------------------------------------- #
# DiT
# --------------------------------------------------------------------------- #
def test_complete_checkpoint_loads_and_meta_buffers_are_rebuilt():
"""Positive control for the failure tests below, and the meta-device path: RoPE
tables and time-embedding frequencies are not in the checkpoint and must be rebuilt
by ``post_load_weights`` so that a loaded model actually runs."""
"""Meta-init must rebuild non-checkpoint RoPE and time-embedding buffers."""
reference = _dit()
checkpoint = _official_state_dict(reference)
model = _load_through_the_real_loader(checkpoint)
model.post_load_weights()
assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()])
assert not model.time_embeddings.freqs.is_meta
torch.testing.assert_close(model.pooled_bias, reference.pooled_bias)
assert model.out_layer.out_layer.weight.shape[0] == 4 * TINY_DIT["out_visual_dim"]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
x = torch.randn(1, 2, 4, 4, 4, device=device)
rope_pos = [torch.arange(2, device=device) for _ in range(3)]
out = model(x, torch.tensor([500.0], device=device), rope_pos)
with set_forward_context(current_timestep=0, attn_metadata=None):
out = model(x, torch.tensor([500.0], device=device), rope_pos)
assert out.shape == (1, 2, 4, 4, 12) and torch.isfinite(out).all()
@pytest.mark.parametrize(
"missing",
"key,unexpected,message",
[
"pooled_bias",
"visual_transformer_blocks.0.visual_modulation.out_layer.bias",
("pooled_bias", False, "was not loaded"),
(
"visual_transformer_blocks.0.visual_modulation.out_layer.bias",
False,
"was not loaded",
),
("text_embeddings.in_layer.weight", True, "unexpected keys"),
],
)
def test_missing_checkpoint_tensor_fails_the_load(missing):
"""The generic loader zero-fills missing parameters whose names contain ``bias`` (and
would only warn about others). For this model every parameter is required, so a
partial checkpoint must raise instead of yielding a silently different network."""
def test_incomplete_or_unexpected_checkpoint_fails_the_load(key, unexpected, message):
# even bias parameters must fail instead of the generic loader's zero-fill fallback
checkpoint = _official_state_dict(_dit())
del checkpoint[missing]
with pytest.raises(ValueError, match="was not loaded"):
if unexpected:
checkpoint[key] = torch.zeros(2, 2)
else:
del checkpoint[key]
with pytest.raises(ValueError, match=message):
_load_through_the_real_loader(checkpoint)
def test_unexpected_checkpoint_tensor_fails_the_load():
checkpoint = _official_state_dict(_dit())
checkpoint["text_embeddings.in_layer.weight"] = torch.zeros(2, 2)
with pytest.raises(ValueError, match="unexpected keys"):
_load_through_the_real_loader(checkpoint)
def test_official_config_builds_the_head_from_the_total_width_and_loads_official_keys():
"""An official config stores the TOTAL DX head width in ``out_visual_dim`` (n_grid lives
in the scheduler config) plus nested ``sr_params`` and ``attribute_overrides: null``.
"""
official = {
key: value
for key, value in TINY_DIT.items()
if key not in ("n_grid", "attribute_overrides")
}
official.update(
out_visual_dim=12,
attribute_overrides=None,
sr_params=dict(
scale_factor={"512": [1.0, 1.0, 1.0]},
visual_size=[512],
scheduler_scale=5.0,
lq_noise_scale=0.7,
lq_noise_type="ddpm",
lq_channel_noise_scale=0.0,
cap_noise_timestep=False,
fps=24,
),
)
config = Kandinsky6SRDitConfig()
config.update_model_arch(official)
with torch.device("meta"):
model = Kandinsky6SRTransformer3DModel(config, official)
# head width = prod(patch_size) * the TOTAL out_visual_dim
assert model.state_dict()["out_layer.out_layer.weight"].shape[0] == 4 * 12
assert model.base_out_visual_dim == 4 and model.n_grid == 1
load_model_from_full_model_state_dict(
model,
model.preprocess_loaded_state_dict(
iter(
_official_state_dict(_dit()).items()
) # same shapes as the legacy DX head
),
torch.device("cpu"),
torch.float32,
strict=False,
param_names_mapping=get_param_names_mapping(model.param_names_mapping),
)
model.post_load_weights()
assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()])
def test_wide_input_checkpoint_needs_visual_cond_to_run_under_noise():
"""A hybrid_anchor checkpoint has a ``2 * in + 1`` input layer; the default ``noise``
override would feed ``in`` channels. That mismatch must be reported, and the
@@ -260,9 +172,6 @@ def test_rope_angle_tables_stay_fp32_when_the_module_is_cast():
assert buffer.dtype == torch.float32, name
# --------------------------------------------------------------------------- #
# Fake server args for the component loaders
# --------------------------------------------------------------------------- #
class FakeServerArgs:
def __init__(self):
self.pipeline_config = Kandinsky6SRPipelineConfig()
@@ -365,12 +274,21 @@ def _tiny_lu_bundle(tmp_path, models=None, scales=(2, 4), include_scales=True):
return directory
def test_latent_upscaler_loader_loads_bank_in_bf16_eval(tmp_path):
directory = _tiny_lu_bundle(tmp_path)
bank, _ = LatentUpscalerLoader().load(
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
@pytest.mark.parametrize("legacy,library", [(False, "diffusers"), (True, "kandinsky6")])
def test_latent_upscaler_loading_and_default_scale_order(tmp_path, legacy, library):
models = [
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
for scale in (("4x", "2x") if legacy else ("2x", "4x"))
]
directory = _tiny_lu_bundle(tmp_path, models=models, include_scales=not legacy)
loader = component_loader_module.ComponentLoader.for_component_type(
"latent_upscaler", library
)
assert isinstance(loader, LatentUpscalerLoader)
bank, _ = loader.load(str(directory), FakeServerArgs(), "latent_upscaler", library)
assert bank.scales == (2, 4)
assert bank._models[0] is bank.for_scale(2)
assert bank._models[1] is bank.for_scale(4)
assert not bank.training
assert all(
p.dtype == torch.bfloat16 and not p.requires_grad for p in bank.parameters()
@@ -382,75 +300,35 @@ def test_latent_upscaler_loader_loads_bank_in_bf16_eval(tmp_path):
bank.upscale(z, scale=8)
def test_latent_upscaler_legacy_config_uses_current_default_scale_order(tmp_path):
models = [
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
for scale in ("4x", "2x")
]
directory = _tiny_lu_bundle(
tmp_path,
models=models,
include_scales=False,
)
bank, _ = LatentUpscalerLoader().load(
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
)
assert bank.scales == (2, 4)
assert bank._models[0] is bank.for_scale(2)
assert bank._models[1] is bank.for_scale(4)
def test_latent_upscaler_may_be_declared_with_the_kandinsky6_library():
"""The official ``save_pretrained`` writes ["kandinsky6", ...], the Hub repo ["diffusers", ...]."""
for library in ("diffusers", "kandinsky6"):
loader = component_loader_module.ComponentLoader.for_component_type(
"latent_upscaler", library
)
assert isinstance(loader, LatentUpscalerLoader)
def test_latent_upscaler_rejects_the_wrong_library():
with pytest.raises(AssertionError, match="latent_upscaler must be loaded from"):
component_loader_module.ComponentLoader.for_component_type(
"latent_upscaler", "transformers"
)
def test_latent_upscaler_load_errors_are_raised_not_swallowed(tmp_path):
"""``ComponentLoader.load`` falls back to a diffusers ``AutoModel`` when the
customized loader raises, which hides real errors. This component must re-raise,
with the offending config key or tensor named."""
@pytest.mark.parametrize("error_kind", ["Missing key", "bogus_field"])
def test_latent_upscaler_load_errors_are_raised_not_swallowed(tmp_path, error_kind):
directory = _tiny_lu_bundle(tmp_path)
weights_path = directory / "diffusion_pytorch_model.safetensors"
weights = load_file(str(weights_path))
weights.pop(next(iter(weights)))
save_file(weights, str(weights_path))
if error_kind == "Missing key":
path = directory / "diffusion_pytorch_model.safetensors"
weights = load_file(str(path))
weights.pop(next(iter(weights)))
save_file(weights, str(path))
else:
path = directory / "config.json"
config = json.loads(path.read_text())
config["models"][0]["model"]["bogus_field"] = 1
path.write_text(json.dumps(config))
with pytest.raises(RuntimeError, match="native fallback is disabled") as error:
LatentUpscalerLoader().load(
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
)
assert "Missing key" in str(error.value.__cause__)
bad = copy.deepcopy(TINY_LU_MODEL)
bad["bogus_field"] = 1
directory = _tiny_lu_bundle(
tmp_path / "bad",
models=[{"target_scale": "2x", "model": TINY_LU_MODEL}],
scales=(2,),
)
config_path = directory / "config.json"
config = json.loads(config_path.read_text())
config["models"][0]["model"] = bad
config_path.write_text(json.dumps(config))
with pytest.raises(RuntimeError, match="native fallback is disabled") as error:
LatentUpscalerLoader().load(
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
)
assert "bogus_field" in str(error.value.__cause__)
assert error_kind in str(error.value.__cause__)
def test_motion_attention_is_not_a_supported_entry_key():
"""The consolidated architecture (mirroring FastVideo's ``kandinsky6_sr.py`` /
commit 950a5edb) only implements the released checkpoint, which never trains
natten/shifted-window motion attention: a config that asks for it must fail loudly
at construction instead of silently building an unsupported backend."""
"""Unsupported motion attention must fail instead of silently using dense attention."""
with_motion = {
**copy.deepcopy(TINY_LU_MODEL),
"enable_x2_entry": False,
@@ -3,7 +3,6 @@
import copy
import json
import os
from collections import Counter
from types import SimpleNamespace
from typing import NamedTuple
@@ -32,10 +31,6 @@ from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
PipelineComponentLoader,
)
@@ -61,9 +56,6 @@ from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import (
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
build_sampling_spec,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
PiflowParams,
)
class Bundle(NamedTuple):
@@ -77,7 +69,6 @@ class Bundle(NamedTuple):
head_width: int # ``out_visual_dim`` of the transformer: the TOTAL head width
n_grid: int # grids of the head: 1 (flow-matching) or the scheduler's ``n_grid``
scheduler_scale: float # ``sr_params.scheduler_scale``
calls_per_tile: int # DiT calls per tile at the default ``num_inference_steps`` 5
FLOW = Bundle(
@@ -89,7 +80,6 @@ FLOW = Bundle(
head_width=64,
n_grid=1,
scheduler_scale=5.0,
calls_per_tile=5,
)
DISTILLED = Bundle(
model_index=DISTILLED_MODEL_INDEX,
@@ -100,26 +90,13 @@ DISTILLED = Bundle(
head_width=640,
n_grid=10,
scheduler_scale=3.5,
calls_per_tile=2,
)
both_repos = pytest.mark.parametrize(
"bundle", [FLOW, DISTILLED], ids=["flow_matching", "distilled_pi_flow"]
)
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29509",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
pytestmark = pytest.mark.usefixtures("single_process_model_parallel")
def _prefixes(keys, depth=2):
@@ -162,17 +139,18 @@ def test_real_transformer_config_builds_the_release_dit(bundle):
# ``out_visual_dim`` is the TOTAL head width (64 * n_grid); a pi-Flow repo keeps its
# n_grid in the scheduler config
assert state["out_layer.out_layer.weight"].shape == (bundle.head_width, 1792)
assert model.base_out_visual_dim == 64 and model.n_grid == 1
assert bundle.n_grid * model.base_out_visual_dim == bundle.head_width
assert bundle.n_grid * model.in_visual_dim == bundle.head_width
assert model.instruct_type == "hybrid_anchor" and model.visual_cond
assert model.visual_embed_dim == 2 * 64 + 1 # wide [x | cond | mask] input
arch = config.arch_config
assert arch.sr_scale_factor == {"512": [1.0, 2.0, 2.0]}
assert arch.sr_scheduler_scale == bundle.scheduler_scale and not arch.is_piflow
assert arch.sr_scheduler_scale == bundle.scheduler_scale
@both_repos
def test_real_scheduler_config_loads_through_the_scheduler_loader(bundle, tmp_path):
def test_loaded_scheduler_controls_sampling_and_rejects_a_mismatched_head(
bundle, tmp_path
):
scheduler = _load_scheduler(bundle, tmp_path)
# PiflowScheduler subclasses the flow-matching scheduler: compare exact types
assert type(scheduler) is bundle.scheduler_cls
@@ -180,43 +158,48 @@ def test_real_scheduler_config_loads_through_the_scheduler_loader(bundle, tmp_pa
if bundle is DISTILLED:
assert (scheduler.config["nfe"], scheduler.config["n_grid"]) == (2, 10)
@both_repos
def test_real_scheduler_selects_the_sampler_of_its_repo(bundle, tmp_path):
"""The distilled repo resolves to pi-Flow (``nfe`` = 2 calls per tile, whatever
``num_inference_steps`` is), the flow-matching repo to flow-Euler
(``num_inference_steps`` calls per tile directly -- this repo's own convention, not
the upstream Diffusers pipeline's timestep-grid-point count).
"""
scheduler = _load_scheduler(bundle, tmp_path)
arch = _dit_config(bundle).arch_config
for num_steps in (3, 5, 9):
spec = _sampling_spec(arch, scheduler, num_steps)
assert spec.is_piflow == (bundle is DISTILLED)
assert spec.num_steps == (2 if bundle is DISTILLED else num_steps)
mismatched = _dit_config(FLOW if bundle is DISTILLED else DISTILLED).arch_config
with pytest.raises(ValueError, match="head.*same checkpoint"):
_sampling_spec(mismatched, scheduler)
def spec(num_steps):
return build_sampling_spec(
arch=arch,
tiling_scale=2,
seed=42,
num_steps=num_steps,
tiles_batch_size=1,
tile_min_overlap=0.2,
scheduler=scheduler,
def _sampling_spec(arch, scheduler, num_steps=5):
return build_sampling_spec(
arch=arch,
scheduler=scheduler,
tiling_scale=2,
seed=42,
num_steps=num_steps,
tiles_batch_size=1,
tile_min_overlap=0.2,
)
@pytest.mark.parametrize("nfe", [None, 0, -1])
def test_piflow_requires_a_positive_checkpoint_step_count(nfe):
with pytest.raises(ValueError, match="nfe >= 1"):
_sampling_spec(
_dit_config(DISTILLED).arch_config, PiflowScheduler(nfe=nfe, n_grid=10)
)
assert spec(5).steps_per_chunk == bundle.calls_per_tile
assert spec(5).scheduler_scale == bundle.scheduler_scale
if bundle is DISTILLED:
assert spec(5).piflow == PiflowParams(
nfe=2,
num_policy_substeps=128,
final_step_size_scale=0.5,
shift=3.5,
n_grid=10,
eps=1e-6,
)
assert spec(3).steps_per_chunk == spec(9).steps_per_chunk == 2
else:
assert spec(5).piflow is None
assert (spec(3).steps_per_chunk, spec(9).steps_per_chunk) == (3, 9)
def test_sampling_rejects_missing_scheduler_steps_and_unsupported_cap():
arch = _dit_config(DISTILLED).arch_config
for scheduler in (None, object()):
with pytest.raises(ValueError, match="scheduler|Scheduler"):
_sampling_spec(arch, scheduler)
scheduler = PiflowScheduler(nfe=2, n_grid=10)
with pytest.raises(ValueError, match="num_inference_steps"):
_sampling_spec(arch, scheduler, num_steps=0)
arch.sr_cap_noise_timestep = True
arch.attribute_overrides = {"instruct_type": "noise"}
with pytest.raises(NotImplementedError, match="cap_noise_timestep"):
_sampling_spec(arch, scheduler)
def test_real_vae_config_builds_the_release_kvae_and_maps_official_keys():
@@ -23,7 +23,12 @@ from types import SimpleNamespace
import pytest
import torch
from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL, super_resolve
from kandinsky6_sr_tiny_components import (
TINY_KVAE,
TINY_LU_MODEL,
TINY_PIFLOW,
super_resolve,
)
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRDitConfig,
@@ -31,14 +36,11 @@ from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
Kandinsky6SRVAEConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.loader.utils import (
get_param_names_mapping,
hf_to_custom_state_dict,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
Kandinsky6SRTransformer3DModel,
)
@@ -58,12 +60,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
build_dit_spec,
build_sampling_spec,
effective_scheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
denoise_with_scheduler,
euler_start_timestep,
make_dit_fn,
SamplingSpec,
denoise_chunks,
module_dtype,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
@@ -105,14 +105,6 @@ TINY_DIT_CFG = dict(
attention_params={"512": {"type": "flash"}},
use_text=False,
)
TINY_PIFLOW = dict(
nfe=2,
num_policy_substeps=8,
final_step_size_scale=0.5,
shift=5.0,
n_grid=3,
eps=1e-6,
)
TINY_SR_PARAMS = dict(
scale_factor={512: [1.0, 1.0, 1.0]},
visual_size=[512],
@@ -203,20 +195,7 @@ from kandinsky_sr.core.components.video_kvae.cached_model import ( # noqa: E402
)
from omegaconf import OmegaConf # noqa: E402
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29507",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
pytestmark = pytest.mark.usefixtures("single_process_model_parallel")
def randomize(module: torch.nn.Module, seed: int, std: float = 0.05) -> None:
@@ -240,40 +219,16 @@ def build_reference_dit(*, piflow: dict | None, seed: int = 2, cfg: dict | None
return dit
def flat_transformer_config(*, cfg: dict, piflow: dict | None, overrides: dict) -> dict:
"""The flat legacy dict form of a transformer config (n_grid / piflow_* / sr_* fields)."""
flat = dict(cfg)
flat["patch_size"] = list(flat["patch_size"])
flat["axes_dims"] = list(flat["axes_dims"])
flat["n_grid"] = piflow["n_grid"] if piflow is not None else 1
if piflow is not None:
flat.update(
piflow_nfe=piflow["nfe"],
piflow_num_policy_substeps=piflow["num_policy_substeps"],
piflow_final_step_size_scale=piflow["final_step_size_scale"],
piflow_shift=piflow["shift"],
piflow_eps=piflow["eps"],
)
flat["attribute_overrides"] = dict(overrides)
flat.update(
sr_visual_size=[512],
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
sr_scheduler_scale=TINY_SR_PARAMS["scheduler_scale"],
sr_lq_noise_scale=TINY_SR_PARAMS["lq_noise_scale"],
sr_lq_noise_type=TINY_SR_PARAMS["lq_noise_type"],
sr_lq_channel_noise_scale=TINY_SR_PARAMS["lq_channel_noise_scale"],
sr_cap_noise_timestep=TINY_SR_PARAMS["cap_noise_timestep"],
sr_fps=TINY_SR_PARAMS["fps"],
)
return flat
def build_port_dit(
reference_dit, *, cfg: dict | None = None, piflow: dict | None, overrides: dict
) -> Kandinsky6SRTransformer3DModel:
"""Port DiT loaded with the reference weights (``model.`` prefix), strictly."""
flat = flat_transformer_config(
cfg=dict(cfg or TINY_DIT_CFG), piflow=piflow, overrides=overrides
cfg = dict(cfg or TINY_DIT_CFG)
flat = dict(
cfg,
out_visual_dim=cfg["out_visual_dim"] * (piflow["n_grid"] if piflow else 1),
attribute_overrides=dict(overrides),
sr_params=dict(TINY_SR_PARAMS),
)
dit_config = Kandinsky6SRDitConfig()
dit_config.update_model_arch(flat)
@@ -324,13 +279,14 @@ def _port_forward(model, x, time, *, scale_factor, motion_score=None):
torch.arange(height // model.patch_size[1]),
torch.arange(width // model.patch_size[2]),
]
return model(
x,
time,
rope_pos,
scale_factor=scale_factor,
motion_score=motion_score,
)
with set_forward_context(current_timestep=0, attn_metadata=None):
return model(
x,
time,
rope_pos,
scale_factor=scale_factor,
motion_score=motion_score,
)
def _random_latent(
@@ -427,28 +383,36 @@ def _reference_loop_inputs(x):
return visual_cu, text, text_cu, rope_pos, torch.zeros(0, dtype=torch.long)
def _port_dit_fn(model, x, scale_factor, use_motion_score=False):
return make_dit_fn(
model,
latent_frames_hw=tuple(x.shape[1:4]),
patch_size=model.patch_size,
scale_factor=scale_factor,
use_motion_score=use_motion_score,
def _port_sample(model, x, scale, scheduler, steps, capped=False):
spec = SamplingSpec(
tiling_scale=2,
tiles_batch_size=1,
seed=42,
num_steps=steps,
is_piflow=isinstance(scheduler, PiflowScheduler),
tile_min_overlap=0.2,
visual_size=512,
scale_factor=scale,
lq_noise_scale=0.7,
lq_noise_type="ddpm",
lq_channel_noise_scale=0.0,
cap_noise_timestep=capped,
)
return denoise_chunks(
[x.clone()],
model,
scheduler,
dit_spec=build_dit_spec(model),
spec=spec,
device=x.device,
step_context=lambda step: set_forward_context(
current_timestep=step, attn_metadata=None
),
)[0]
def test_piflow_loop_matches_reference_piflow_generate():
"""Guards the segment schedule, DXPolicy grid layout and rollout on a batch of tiles.
The reference packs the tiles along time with one per-frame sigma; the port batches
``[B, T, H, W, C]``. Same weights and same start latent, fp32 on CPU.
The port no longer re-derives the pi-Flow segment schedule itself (``latents.piflow_schedule``
/ ``sampling.denoise_piflow`` are gone): it drives a real ``PiflowScheduler`` object through
``denoise_with_scheduler`` instead, which the analysis in commit d8d0e79f's port (and the
``PiflowScheduler._policy_step`` implementation itself) shows reproduces the exact same
per-segment ``(raw_src, seg, raw_dst, sigma_src)`` schedule as the removed pure function did.
"""
"""Compare the packed reference loop with the batched native PiflowScheduler."""
from kandinsky_sr.core.algo.piflow_sampler import piflow_generate
reference = build_reference_dit(piflow=TINY_PIFLOW)
@@ -460,7 +424,6 @@ def test_piflow_loop_matches_reference_piflow_generate():
visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x)
packed = x.reshape(-1, *x.shape[2:]).clone()
scheduler = PiflowScheduler(**TINY_PIFLOW)
scheduler.set_timesteps(TINY_PIFLOW["nfe"], device="cpu")
with torch.no_grad():
expected = piflow_generate(
packed,
@@ -476,13 +439,7 @@ def test_piflow_loop_matches_reference_piflow_generate():
device="cpu",
**TINY_PIFLOW,
)
actual = denoise_with_scheduler(
x.clone(),
_port_dit_fn(model, x, scale),
scheduler,
channels=4,
is_piflow=True,
)
actual = _port_sample(model, x, scale, scheduler, TINY_PIFLOW["nfe"])
torch.testing.assert_close(
actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5
)
@@ -492,18 +449,8 @@ def test_piflow_loop_matches_reference_piflow_generate():
"capped", [False, True], ids=["full_range", "capped_noise_start"]
)
def test_euler_loop_matches_reference_generate(capped):
"""Guards the warped timesteps, per-step deltas and (for a wide input) the in-place
update of the first channels only, incl. the ``cap_noise_timestep`` start time.
The reference's ``generate(..., 5, ...)`` counts timestep *grid points* (5 points -> 4
Euler steps); the port's own convention counts DiT *calls* directly, so the equivalent
port call is ``num_inference_steps=4``. Both grids are the same 5-point warped-linspace
array (``FlowMatchEulerDiscreteScheduler.set_timesteps`` applies the configured ``shift``
warp to the 4 unwarped sigmas this test passes in, then appends the terminal 0 itself --
algebraically the same 5th point the reference's own ``warp(0) == 0`` produces), so the 4
step-to-step deltas ``denoise_with_scheduler`` consumes via ``scheduler.step`` are bit-for-
bit the 4 deltas the old, removed ``denoise_euler`` consumed from that same array.
"""
"""Compare noise caps and channel updates with the reference's five-point grid
(four DiT calls); both schedulers append the same terminal zero."""
from kandinsky_sr.core.algo.utils import generate
cfg = dict(TINY_DIT_CFG, use_motion_score=True, instruct_type="hybrid_anchor")
@@ -517,16 +464,11 @@ def test_euler_loop_matches_reference_generate(capped):
reference.instruct_type, reference.visual_cond = "noise", True
x = _random_latent(2 * 4 + 1, seed=9)
scale = (1.0, 1.5, 1.5)
start = euler_start_timestep(
cap_noise_timestep=capped, lq_noise_scale=0.7, instruct_type="noise"
)
assert start == (0.7 if capped else 1.0)
start = 0.7 if capped else 1.0
visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x)
packed = x.reshape(-1, *x.shape[2:]).clone()
num_inference_steps = 4 # the reference's 5 grid points == 4 actual Euler steps
scheduler = FlowMatchEulerDiscreteScheduler(shift=5.0)
sigmas = torch.linspace(start, 0.0, num_inference_steps + 1)[:-1].tolist()
scheduler.set_timesteps(sigmas=sigmas, device="cpu")
with torch.no_grad():
expected = generate(
packed,
@@ -546,13 +488,7 @@ def test_euler_loop_matches_reference_generate(capped):
5.0,
start_timestep=start,
)
actual = denoise_with_scheduler(
x.clone(),
_port_dit_fn(model, x, scale, use_motion_score=True),
scheduler,
channels=4,
is_piflow=False,
)
actual = _port_sample(model, x, scale, scheduler, num_inference_steps, capped)
torch.testing.assert_close(
actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5
)
@@ -663,6 +599,9 @@ def build_stacks(*, piflow, use_lu):
vae.load_state_dict(dict(ref_vae.state_dict()), strict=True)
port = {
"vae": vae.eval(),
"scheduler": PiflowScheduler(**piflow)
if piflow
else FlowMatchEulerDiscreteScheduler(shift=TINY_SR_PARAMS["scheduler_scale"]),
"dit": build_port_dit(
ref_dit, piflow=piflow, overrides={"instruct_type": "noise"}
),
@@ -728,7 +667,10 @@ def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps):
if pre != 1.0:
video = tiling.pre_upscale_video(video, pre, 16)
arch = port["dit"].config
port_num_steps = num_steps if arch.is_piflow else num_steps - 1
scheduler = port["scheduler"]
port_num_steps = (
num_steps if isinstance(scheduler, PiflowScheduler) else num_steps - 1
)
spec = build_sampling_spec(
arch=arch,
tiling_scale=tiling_scale,
@@ -736,6 +678,7 @@ def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps):
num_steps=port_num_steps,
tiles_batch_size=tiles_batch_size,
tile_min_overlap=0.2,
scheduler=scheduler,
)
bank = port["bank"]
use_lu = bank is not None and bank.for_scale(tiling_scale) is not None
@@ -746,7 +689,7 @@ def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps):
dit=port["dit"],
dit_spec=build_dit_spec(port["dit"]),
spec=spec,
scheduler=effective_scheduler(spec, None),
scheduler=scheduler,
device="cpu",
upscale_fn=partial(bank.upscale, scale=tiling_scale) if use_lu else None,
lu_dtype=module_dtype(bank) if use_lu else None,
@@ -782,19 +725,10 @@ def _assert_uint8_close(actual, expected, *, max_diff: int = 1):
def test_tiled_piflow_super_resolution_matches_reference(
use_lu, scale, hw, tiles_batch_size
):
"""Guards the whole orchestration: scale request + pre-upscale, tile geometry, LU or
tile encode, chunk seeding (``seed + first tile index``), pi-Flow loop, decode with
truncation to uint8 and Hann stitching, against ``run_tiled_sr`` /
``run_tiled_sr_from_pixels`` with identical random weights.
"""Compare tiled pixel/LU paths with identical random weights and seeds.
``max_diff=2`` (not the default 1): the reference's ``piflow_generate`` is the
training-only inline schedule, which clamps the *final* segment's ``raw_dst`` to
0.0 (dead code on every real checkpoint - see GAP 2 / ``latents.piflow_schedule``).
The port now deliberately matches the real ``PiflowScheduler._policy_step``
convention (``raw_dst=eps``) instead, which real checkpoints always run through.
That is a ~1e-6 raw-timestep difference in the last segment only, so it shows up
here as at most a 1-LSB-wider uint8 rounding tie on top of the pre-existing
rounding-boundary noise (still >99.99% exact pixels), not as a structural bug.
The reference ends at raw_dst=0, whereas PiflowScheduler ends at eps (1e-6).
Allow two uint8 levels of error while requiring more than 99.9% exact pixels.
"""
reference, port = build_stacks(piflow=TINY_PIFLOW, use_lu=use_lu)
video = _random_video(9, *hw)
@@ -1,253 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Validate scheduler selection, DiT call counts and incompatible head rejection."""
import os
from typing import NamedTuple
import pytest
from kandinsky6_sr_release_configs import (
DISTILLED_SCHEDULER_CONFIG,
DISTILLED_TRANSFORMER_CONFIG,
FLOW_SCHEDULER_CONFIG,
FLOW_TRANSFORMER_CONFIG,
)
from kandinsky6_sr_tiny_components import (
TINY_DIT,
build_components,
make_request,
make_stage,
random_video,
)
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
PiflowScheduler,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
Kandinsky6SRDenoisingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
Kandinsky6SRLatentPrepStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_VIDEO_KEY,
build_sampling_spec,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
RESOLUTIONS,
)
LATENT_WIDTH = TINY_DIT["in_visual_dim"]
N_GRID = (
3 # the distilled bundle's own grid width, independent of TINY_DIT's legacy fields
)
NUM_TILES = 9 # the 64x96 clip below, cut into 32x48 tiles
def official_dit(head_width):
"""Tiny transformer config in the official layout: ``out_visual_dim`` is the TOTAL
head width, and ``n_grid`` / ``piflow_*`` live in the scheduler config."""
arch = {
key: value
for key, value in TINY_DIT.items()
if key != "out_visual_dim" and not key.startswith(("n_grid", "piflow_", "sr_"))
}
return dict(
arch,
out_visual_dim=head_width,
sr_params=dict(scale_factor={"512": [1.0, 1.0, 1.0]}, visual_size=[512]),
)
FLOW_DIT = official_dit(LATENT_WIDTH) # one latent per token: flow-matching head
DX_DIT = official_dit(N_GRID * LATENT_WIDTH) # N_GRID latents per token: pi-Flow head
def flow_scheduler():
return FlowMatchEulerDiscreteScheduler(shift=5.0)
def pi_flow_scheduler(*, nfe=2, n_grid=N_GRID):
return PiflowScheduler(
nfe=nfe,
n_grid=n_grid,
shift=5.0,
eps=1e-6,
final_step_size_scale=0.5,
num_policy_substeps=8,
)
def release_scheduler(scheduler_cls, config):
return scheduler_cls(**{k: v for k, v in config.items() if not k.startswith("_")})
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29510",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
@pytest.fixture(autouse=True)
def tiny_resolutions(monkeypatch):
monkeypatch.setitem(RESOLUTIONS, 512, [(64, 64), (64, 96), (96, 64)])
class SamplingRun(NamedTuple):
tiles: int
dit_calls: int
progress_totals: list
def run_denoising(dit_arch, scheduler, *, num_steps=5):
"""Run latent-prep + denoising (the LU-less pixel path) on the 9-tile clip and count
the DiT calls next to the totals the denoising progress bar was opened with."""
vae, dit, _, server_args = build_components((), dit_arch)
dit_calls = []
dit.register_forward_hook(lambda module, args, output: dit_calls.append(1))
latent_prep = make_stage(
Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, scheduler
)
denoise = make_stage(Kandinsky6SRDenoisingStage, server_args, dit, scheduler)
progress_totals = []
progress_bar = denoise.progress_bar
def spy_on_progress_bar(**kwargs):
progress_totals.append(kwargs["total"])
return progress_bar(**kwargs)
denoise.progress_bar = spy_on_progress_bar
batch = make_request(random_video(9, 64, 96), num_steps=num_steps)
batch = latent_prep.forward(batch, server_args)
tiles = len(batch.extra["kandinsky6_sr_chunks"])
denoise.forward(batch, server_args)
return SamplingRun(tiles, len(dit_calls), progress_totals)
def sampling_spec(dit_arch, scheduler):
config = Kandinsky6SRDitConfig()
config.update_model_arch(dict(dit_arch))
return build_sampling_spec(
arch=config.arch_config,
tiling_scale=2,
seed=42,
num_steps=5,
tiles_batch_size=1,
tile_min_overlap=0.2,
scheduler=scheduler,
)
# --------------------------------------------------------------------------- #
# DiT calls per tile
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("nfe,num_steps", [(2, 3), (2, 5), (2, 9), (3, 5)])
def test_pi_flow_scheduler_makes_nfe_calls_per_tile_whatever_num_inference_steps(
nfe, num_steps
):
run = run_denoising(DX_DIT, pi_flow_scheduler(nfe=nfe), num_steps=num_steps)
assert run.tiles == NUM_TILES
assert run.dit_calls == NUM_TILES * nfe
assert run.progress_totals == [NUM_TILES * nfe]
@pytest.mark.parametrize("num_steps,calls_per_tile", [(3, 3), (5, 5), (9, 9)])
def test_flow_euler_scheduler_makes_num_inference_steps_calls_per_tile(
num_steps, calls_per_tile
):
run = run_denoising(FLOW_DIT, flow_scheduler(), num_steps=num_steps)
assert run.tiles == NUM_TILES
assert run.dit_calls == NUM_TILES * calls_per_tile
assert run.progress_totals == [NUM_TILES * calls_per_tile]
@pytest.mark.parametrize("num_steps", [3, 9])
def test_legacy_config_with_its_own_pi_flow_fields_keeps_its_sampler(num_steps):
"""A flat DiT config carries n_grid / piflow_* and needs no scheduler component: the
denoising stage synthesizes a matching PiflowScheduler from those arch fields
(run_spec.effective_scheduler) instead of leaving nothing to step."""
run = run_denoising(TINY_DIT, None, num_steps=num_steps)
assert run.dit_calls == NUM_TILES * TINY_DIT["piflow_nfe"]
assert run.progress_totals == [run.dit_calls]
# --------------------------------------------------------------------------- #
# Samplers that do not fit the DiT head
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
"scheduler",
[flow_scheduler(), PiflowScheduler(n_grid=N_GRID, nfe=None), None],
ids=["flow_euler_scheduler", "pi_flow_scheduler_without_nfe", "no_scheduler"],
)
def test_dx_head_never_runs_flow_euler(scheduler):
with pytest.raises(ValueError, match="flow-Euler"):
sampling_spec(DX_DIT, scheduler)
@pytest.mark.parametrize(
"dit_arch,n_grid,needed",
[
(FLOW_DIT, N_GRID, "3 x 4 = 12"),
(DX_DIT, 2, "2 x 4 = 8"),
(DX_DIT, 4, "4 x 4 = 16"),
],
ids=["single_head", "too_few_grids", "too_many_grids"],
)
def test_pi_flow_needs_a_head_of_n_grid_latents(dit_arch, n_grid, needed):
with pytest.raises(ValueError, match=f"needs a {needed} channel DX head"):
sampling_spec(dit_arch, pi_flow_scheduler(n_grid=n_grid))
@pytest.mark.parametrize("nfe", [0, -1])
def test_pi_flow_needs_at_least_one_call_per_tile(nfe):
with pytest.raises(ValueError, match="nfe >= 1"):
sampling_spec(DX_DIT, pi_flow_scheduler(nfe=nfe))
with pytest.raises(ValueError, match="nfe >= 1"):
sampling_spec(dict(TINY_DIT, piflow_nfe=nfe), None)
def test_release_transformer_and_scheduler_of_different_repos_are_rejected():
"""The real configs of the two official repos: their transformers differ only in
the head width, so one repo's transformer with the other's scheduler must fail."""
flow = release_scheduler(FlowMatchEulerDiscreteScheduler, FLOW_SCHEDULER_CONFIG)
distilled = release_scheduler(PiflowScheduler, DISTILLED_SCHEDULER_CONFIG)
assert sampling_spec(FLOW_TRANSFORMER_CONFIG, flow).steps_per_chunk == 5
assert sampling_spec(DISTILLED_TRANSFORMER_CONFIG, distilled).steps_per_chunk == 2
with pytest.raises(ValueError, match="640 channels wide.*flow-Euler"):
sampling_spec(DISTILLED_TRANSFORMER_CONFIG, flow)
with pytest.raises(ValueError, match="needs a 10 x 64 = 640 channel DX head"):
sampling_spec(FLOW_TRANSFORMER_CONFIG, distilled)
def test_a_mismatched_bundle_fails_before_any_tile_is_processed():
vae, dit, _, server_args = build_components((), DX_DIT)
dit_calls = []
dit.register_forward_hook(lambda module, args, output: dit_calls.append(1))
latent_prep = make_stage(
Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, flow_scheduler()
)
batch = make_request(random_video(9, 64, 96))
with pytest.raises(ValueError, match="flow-Euler"):
latent_prep.forward(batch, server_args)
assert dit_calls == []
assert SR_VIDEO_KEY in batch.extra # the clip was never cut into tiles
@@ -2,14 +2,15 @@
"""Native SR stage-chain parity, component residency and video/audio hand-offs."""
import copy
import os
from functools import partial
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
import pytest
import torch
from kandinsky6_sr_tiny_components import (
TINY_DIT,
TINY_LU_MODEL,
TINY_PIFLOW,
build_components,
make_request,
make_stage,
@@ -21,9 +22,11 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
PiflowScheduler,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
)
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
Kandinsky6SRLatentUpscalerBank,
@@ -32,26 +35,6 @@ from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import (
Kandinsky6SRPipeline,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
Kandinsky6SRDecodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
Kandinsky6SRDenoisingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import (
Kandinsky6SREncodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
WARMUP_CLIP_FRAMES,
WARMUP_CLIP_HW,
Kandinsky6SRInputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
Kandinsky6SRLatentPrepStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import (
Kandinsky6SROutputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_LR_LATENT_KEY,
SR_PLAN_KEY,
@@ -60,11 +43,20 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
SR_VIDEO_KEY,
build_dit_spec,
build_sampling_spec,
effective_scheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
module_dtype,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import (
WARMUP_CLIP_FRAMES,
WARMUP_CLIP_HW,
Kandinsky6SRDecodeStage,
Kandinsky6SRDenoisingStage,
Kandinsky6SREncodeStage,
Kandinsky6SRInputStage,
Kandinsky6SRLatentPrepStage,
Kandinsky6SROutputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
TilePlan,
)
@@ -72,19 +64,7 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
RESOLUTIONS,
)
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29509",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
pytestmark = pytest.mark.usefixtures("single_process_model_parallel")
@pytest.fixture(autouse=True)
@@ -93,22 +73,25 @@ def tiny_resolutions(monkeypatch):
@pytest.mark.parametrize(
"bank_scales",
[("2x", "4x"), (), ("4x",)],
"bank_scales,nfe,num_steps",
[(("2x", "4x"), 2, 5), ((), None, 3), (("4x",), 3, 9)],
ids=["lu_path", "pixel_path", "pixel_fallback"],
)
def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
"""encode -> latent-prep -> denoise -> decode -> output must give what
``super_resolve`` gives on the same clip, for the LU path, the pixel path, and a bank
that has no entry for the requested scale (which must fall back to the pixel path, not
fail). The 64x128 clip holds 12 overlapping 32x48 tiles, denoised in chunks of 5, 5 and
2."""
vae, dit, bank, server_args = build_components(bank_scales)
def test_stage_chain_reproduces_the_pure_orchestration(bank_scales, nfe, num_steps):
"""Cover LU/pixel fallback and scheduler reuse across chunks and requests."""
scheduler = (
PiflowScheduler(**dict(TINY_PIFLOW, nfe=nfe))
if nfe is not None
else FlowMatchEulerDiscreteScheduler(shift=5.0)
)
vae, dit, bank, server_args = build_components(
bank_scales, dict(TINY_DIT, out_visual_dim=12 if nfe is not None else 4)
)
video = random_video(9, 64, 128)
batch = make_request(video, tiles_batch_size=5)
batch = make_request(video, tiles_batch_size=5, num_steps=num_steps)
pipeline = object.__new__(Kandinsky6SRPipeline)
pipeline.modules = dict(vae=vae, transformer=dit, scheduler=None)
pipeline.modules = dict(vae=vae, transformer=dit, scheduler=scheduler)
if bank is not None:
pipeline.modules["latent_upscaler"] = bank
pipeline._stages, pipeline._stage_name_mapping = [], {}
@@ -148,7 +131,14 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
assert SR_LR_LATENT_KEY not in batch.extra and SR_VIDEO_KEY not in batch.extra
assert len(batch.extra["kandinsky6_sr_chunks"]) == 3 # 12 tiles / 5 per chunk
assert isinstance(batch.extra[SR_PLAN_KEY], TilePlan)
batch = denoise.forward(batch, server_args)
dit_calls = []
hook = dit.register_forward_hook(lambda *args: dit_calls.append(1))
with patch.object(denoise, "progress_bar", wraps=denoise.progress_bar) as progress:
batch = denoise.forward(batch, server_args)
hook.remove()
assert len(dit_calls) == 3 * (nfe or num_steps)
assert progress.call_args.kwargs["total"] == len(dit_calls)
assert denoise.scheduler is scheduler
batch = decode.forward(batch, server_args)
assert len(batch.extra[SR_TILES_KEY]) == 12
result = output.forward(batch, server_args)
@@ -159,9 +149,10 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
arch=arch,
tiling_scale=2,
seed=42,
num_steps=5,
num_steps=num_steps,
tiles_batch_size=5,
tile_min_overlap=0.2,
scheduler=scheduler,
)
expected = super_resolve(
video,
@@ -170,7 +161,7 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
dit=dit,
dit_spec=build_dit_spec(dit),
spec=spec,
scheduler=effective_scheduler(spec, None),
scheduler=scheduler,
device=next(dit.parameters()).device,
upscale_fn=partial(bank.upscale, scale=2) if use_lu else None,
lu_dtype=module_dtype(bank) if use_lu else None,
@@ -184,6 +175,24 @@ def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
assert SR_TILES_KEY not in batch.extra and SR_PLAN_KEY not in batch.extra
def test_mismatched_sampler_fails_before_encoding_tiles():
vae, dit, _, server_args = build_components(())
stage = make_stage(
Kandinsky6SRLatentPrepStage,
server_args,
vae,
dit,
None,
FlowMatchEulerDiscreteScheduler(shift=5.0),
)
batch = make_request(random_video(9, 64, 96))
with patch.object(vae, "encode", wraps=vae.encode) as encode:
with pytest.raises(ValueError, match="head.*same checkpoint"):
stage.forward(batch, server_args)
encode.assert_not_called()
assert SR_VIDEO_KEY in batch.extra
@pytest.mark.parametrize(
"role", [RoleType.ENCODER, RoleType.DENOISER, RoleType.DECODER]
)
@@ -9,7 +9,8 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.k
SamplingSpec,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
build_chunk_latent,
plan_tiles,
prepare_tile_latents,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
compute_tile_grid_even,
@@ -63,22 +64,30 @@ def test_initial_latent_uses_dit_dtype_or_preserves_input(dit_dtype):
tiles_batch_size=1,
seed=1,
num_steps=5,
is_piflow=False,
tile_min_overlap=0.2,
visual_size=512,
scale_factor=(1.0, 1.0, 1.0),
scheduler_scale=5.0,
lq_noise_scale=0.7,
lq_noise_type="linear",
lq_channel_noise_scale=0.0,
cap_noise_timestep=False,
piflow=None,
)
tile = torch.randn(4, 4, 4, 4, dtype=torch.float64)
result = build_chunk_latent(
[tile],
seed=1,
plan = plan_tiles(
frame_hw=(64, 64),
visual_size=512,
tiling_scale=2,
tile_min_overlap=0.2,
resolutions={512: [(128, 128)]},
)
result = prepare_tile_latents(
tile,
plan,
tile_encoder=lambda tile: tile.permute(0, 2, 3, 1),
latent_path=True,
dit_spec=dit_spec,
spec=spec,
device=torch.device("cpu"),
)
)[0]
assert result.dtype == (dit_dtype or tile.dtype)
@@ -15,28 +15,39 @@ from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
)
from sglang.multimodal_gen.runtime.entrypoints.utils import _sample_to_uint8_frames
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
Kandinsky6SRInputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_TILING_SCALE_KEY,
SR_VIDEO_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.stages import (
Kandinsky6SRInputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
FrameSelector,
decode_clip,
extract_source_audio,
to_output_video,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
clip_to_aligned_frames,
resample_to_target_fps,
)
FRAME_COUNTS = [1, 2, 9, 24, 50, 121, 125, 200, 301, 302, 400, 1000]
SOURCE_FPS = [12, 23.976, 24, 25, 29.97, 30, 48, 50, 59.94, 60, 120]
def clip_to_aligned_frames(video):
video = video[:121]
if not len(video):
raise ValueError("empty video")
return video[: 1 + 8 * ((len(video) - 1) // 8)]
def _reference_resample(video, fps):
if abs(fps - 24) < 1.5 or fps < 24:
return video, round(fps)
step = fps / 24
indices = [round(i * step) for i in range(int(len(video) / step))]
return video[[i for i in indices if i < len(video)]], 24
def _stream_select(total: int, fps: float) -> tuple[list[int], int]:
"""Run the streaming selector over frames ``0 .. total - 1`` (early stop included)."""
selector = FrameSelector(fps)
@@ -54,7 +65,7 @@ def _stream_select(total: int, fps: float) -> tuple[list[int], int]:
def _reference_select(total: int, fps: float) -> tuple[list[int], int] | None:
ids = torch.arange(total).view(total, 1, 1, 1)
resampled, effective_fps = resample_to_target_fps(ids, fps)
resampled, effective_fps = _reference_resample(ids, fps)
try:
clipped = clip_to_aligned_frames(resampled)
except ValueError:
@@ -147,7 +158,7 @@ def test_decode_clip_equals_reference_cli_pipeline(tmp_path, fps, frames):
path = tmp_path / "clip.mp4"
write_test_video(path, frames=frames, fps=fps)
full, source_fps = _decode_everything(path)
expected, expected_fps = resample_to_target_fps(full, source_fps)
expected, expected_fps = _reference_resample(full, source_fps)
expected = clip_to_aligned_frames(expected)
clip = decode_clip(str(path))
@@ -4,6 +4,7 @@
import math
from types import SimpleNamespace
import numpy as np
import PIL.Image
import pytest
import torch
@@ -81,6 +82,35 @@ class ImageVAE:
return SimpleNamespace(sample=lambda generator=None: self.latent)
@pytest.mark.parametrize("source_size", [(1920, 1080), (1080, 1920), (288, 192)])
@pytest.mark.parametrize("tensor_input", [False, True], ids=["pil", "tensor"])
def test_image_preprocessing_preserves_aspect_and_center_crops(
source_size, tensor_input
):
src_w, src_h = source_size
height, width = 64, 96
pixels = np.full((src_h, src_w, 3), 128, dtype=np.uint8)
pixels[..., 0] = np.linspace(0, 255, src_w, dtype=np.uint8)[None, :]
pixels[..., 1] = np.linspace(0, 255, src_h, dtype=np.uint8)[:, None]
image = PIL.Image.fromarray(pixels)
# independent PIL oracle for the reference resize + center-crop geometry
scale = min(src_h / height, src_w / width)
new_h, new_w = int(src_h / scale), int(src_w / scale)
top, left = (new_h - height) // 2, (new_w - width) // 2
expected = image.resize((new_w, new_h), PIL.Image.Resampling.BILINEAR).crop(
(left, top, left + width, top + height)
)
expected = torch.from_numpy(np.array(expected)).permute(2, 0, 1).float() / 127.5 - 1
source = (
torch.from_numpy(pixels).permute(2, 0, 1).float() / 127.5 - 1
if tensor_input
else image
)
actual = Kandinsky6ImageEncodingStage._preprocess(source, height, width)
torch.testing.assert_close(actual, expected.unsqueeze(0), atol=2 / 255, rtol=0)
@pytest.mark.parametrize("batch_size", [1, 2])
def test_image_request_broadcasts_one_reference_to_each_output(server_args, batch_size):
request = VideoGenerationsRequest(