[diffusion] model: support Kandinsky 6 (#42743)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Lev Novitskiy <57654885+leffff@users.noreply.github.com>
Co-authored-by: Kirill <kykozlov@gmail.com>
Co-authored-by: Artem Borisov <divotionchannel@gmail.com>
This commit is contained in:
Mick
2026-10-06 22:13:29 +08:00
committed by GitHub
co-authored by Mick Qian Lev Novitskiy Kirill Artem Borisov
parent 730f1f3e5b
commit 16a23a672b
115 changed files with 18816 additions and 140 deletions
@@ -151,6 +151,10 @@ Several norms look interchangeable and are not. Start here.
| Entry point | Backend | Contract | Applies to |
|---|---|---|---|
| `residual_gate_add` | JIT CUDA (contiguous), Triton (transposed) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) |
| `residual_gate_fp32` | Triton | separate FP32 multiply/add, then cast to input dtype | contiguous `[B,S,D]`, strided `[B or 1,1,D]` gate; Kandinsky6 |
These contracts differ: `residual_gate_fp32` does not round the product to
BF16/FP16, and disables FMA to preserve the eager FP32 rounding boundary.
The transposed-dense path tiles along the physical stride-1 token dimension
for coalesced residual reads and output writes. The contiguous layouts use
@@ -49,6 +49,20 @@ _HIP = frozenset({CapabilityRequirement.HIP})
# inventory: callers name the one they want via ``select_kernel``.
# ---------------------------------------------------------------------------
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
(
"diffusion.residual_gate_fp32",
KernelBackend.TRITON,
"modulate.residual_gate_fp32:residual_gate_fp32",
_CUDA,
"Residual gating with separate FP32 product and sum rounding.",
),
(
"diffusion.matrix_rope",
KernelBackend.TRITON,
"rope.matrix_rope:apply_matrix_rope",
_CUDA,
"Interleaved FP32 matrix RoPE with separate product and sum rounding.",
),
(
"diffusion.fp8_rowwise",
KernelBackend.TRITON,
@@ -590,6 +604,8 @@ for _op, _backend, _target, _caps, _description in _SPECS:
# then symbol; a new public kernel belongs here and nowhere else.
# ---------------------------------------------------------------------------
_EXPORTS: dict[str, str] = {
"apply_matrix_rope": "rope.matrix_rope",
"residual_gate_fp32": "modulate.residual_gate_fp32",
"can_use_fp8_rowwise": "quantization.fp8_rowwise_triton",
"fp8_rowwise": "quantization.fp8_rowwise_triton",
"fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
"""Residual gating with FP32 intermediates and a single final storage cast."""
import torch
import triton
import triton.language as tl
from sglang.srt.utils.custom_op import register_custom_op
@triton.jit
def _residual_gate_fp32_kernel(
x,
update,
gate,
out,
count,
TOKENS: tl.constexpr,
HIDDEN: tl.constexpr,
GATE_BATCH_STRIDE: tl.constexpr,
GATE_CHANNEL_STRIDE: tl.constexpr,
BLOCK: tl.constexpr,
):
index = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
valid = index < count
gate_index = (
index // (TOKENS * HIDDEN) * GATE_BATCH_STRIDE
+ index % HIDDEN * GATE_CHANNEL_STRIDE
)
residual = tl.load(x + index, valid, other=0).to(tl.float32)
value = tl.load(update + index, valid, other=0).to(tl.float32)
weight = tl.load(gate + gate_index, valid, other=0).to(tl.float32)
# eager rounds the multiply to FP32, not to the input dtype or an FMA
result = residual + weight * value
tl.store(out + index, result, valid)
def _fake_residual_gate_fp32(
x: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
) -> torch.Tensor:
return torch.empty_like(x)
@register_custom_op(
op_name="diffusion_residual_gate_fp32",
mutates_args=[],
fake_impl=_fake_residual_gate_fp32,
)
def _residual_gate_fp32(
x: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
) -> torch.Tensor:
output = torch.empty_like(x)
if x.numel():
with torch.cuda.device(x.device):
_residual_gate_fp32_kernel[(triton.cdiv(x.numel(), 1024),)](
x,
update,
gate,
output,
x.numel(),
TOKENS=x.shape[1],
HIDDEN=x.shape[2],
GATE_BATCH_STRIDE=gate.stride(0) if gate.shape[0] != 1 else 0,
GATE_CHANNEL_STRIDE=gate.stride(2),
BLOCK=1024,
enable_fp_fusion=False,
)
return output
def residual_gate_fp32(
x: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
) -> torch.Tensor:
"""Compute ``(x.float() + gate.float() * update.float()).to(x.dtype)``.
Unlike ``residual_gate_add``, the product is not rounded to BF16/FP16.
The CUDA path accepts contiguous [B,S,D] and a [B or 1,1,D] gate.
"""
if (
x.is_cuda
and torch.version.hip is None
and not torch.is_grad_enabled()
and x.ndim == 3
and x.shape == update.shape
and x.is_contiguous()
and update.is_contiguous()
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
and update.dtype == x.dtype
and gate.dtype in (x.dtype, torch.float32)
and gate.device == update.device == x.device
and gate.ndim == 3
and gate.shape[0] in (1, x.shape[0])
and gate.shape[1:] == (1, x.shape[2])
):
return _residual_gate_fp32(x, update, gate)
return (x.float() + gate.float() * update.float()).to(x.dtype)
@@ -0,0 +1,111 @@
# SPDX-License-Identifier: Apache-2.0
"""Interleaved RoPE with the eager FP32 matrix-product rounding boundaries."""
import torch
import triton
import triton.language as tl
from sglang.srt.utils.custom_op import register_custom_op
@triton.jit
def _matrix_rope_kernel(
x,
rope,
out,
pairs,
SEQ: tl.constexpr,
HEADS: tl.constexpr,
HALF_DIM: tl.constexpr,
BATCH_STRIDE: tl.constexpr,
SEQ_STRIDE: tl.constexpr,
PAIR_STRIDE: tl.constexpr,
ROW_STRIDE: tl.constexpr,
COL_STRIDE: tl.constexpr,
BLOCK: tl.constexpr,
):
idx = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
valid = idx < pairs
token = idx // (HEADS * HALF_DIM)
pos = (
token // SEQ * BATCH_STRIDE
+ token % SEQ * SEQ_STRIDE
+ idx % HALF_DIM * PAIR_STRIDE
)
# preserve the input cast before FP32 products when fusing a dtype conversion
x0 = tl.load(x + 2 * idx, valid, other=0).to(out.dtype.element_ty).to(tl.float32)
x1 = (
tl.load(x + 2 * idx + 1, valid, other=0).to(out.dtype.element_ty).to(tl.float32)
)
r00 = tl.load(rope + pos, valid, other=0)
r01 = tl.load(rope + pos + COL_STRIDE, valid, other=0)
r10 = tl.load(rope + pos + ROW_STRIDE, valid, other=0)
r11 = tl.load(rope + pos + ROW_STRIDE + COL_STRIDE, valid, other=0)
# do not contract the products: eager materializes them before the pair sum
y0 = x0 * r00 + x1 * r01
y1 = x0 * r10 + x1 * r11
tl.store(out + 2 * idx, y0, valid)
tl.store(out + 2 * idx + 1, y1, valid)
def _fake_apply_matrix_rope(
x: torch.Tensor, rope: torch.Tensor, dtype: torch.dtype | None = None
) -> torch.Tensor:
return torch.empty_like(x, dtype=dtype)
@register_custom_op(
op_name="diffusion_apply_matrix_rope",
mutates_args=[],
fake_impl=_fake_apply_matrix_rope,
)
def apply_matrix_rope(
x: torch.Tensor, rope: torch.Tensor, dtype: torch.dtype | None = None
) -> torch.Tensor:
"""Apply FP32 matrices to [B,S,H,D], optionally casting x to dtype first.
The result has the cast dtype; the input rounding precedes FP32 products.
"""
dtype = x.dtype if dtype is None else dtype
if not (
x.is_cuda
and torch.version.hip is None
and x.ndim == 4
and x.is_contiguous()
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
and dtype in (torch.float16, torch.bfloat16, torch.float32)
):
raise ValueError(
"matrix RoPE requires contiguous FP16/BF16/FP32 CUDA [B,S,H,D]"
)
batch, seq, heads, dim = x.shape
if not (
dim % 2 == 0
and rope.device == x.device
and rope.dtype == torch.float32
and rope.ndim == 6
and rope.shape[0] in (1, batch)
and rope.shape[1:] == (seq, 1, dim // 2, 2, 2)
):
raise ValueError("matrix RoPE requires matching FP32 [B or 1,S,1,D/2,2,2]")
output = torch.empty_like(x, dtype=dtype)
pairs = x.numel() // 2
if pairs:
with torch.cuda.device(x.device):
_matrix_rope_kernel[(triton.cdiv(pairs, 512),)](
x,
rope,
output,
pairs,
SEQ=seq,
HEADS=heads,
HALF_DIM=dim // 2,
BATCH_STRIDE=rope.stride(0) if rope.shape[0] != 1 else 0,
SEQ_STRIDE=rope.stride(1),
PAIR_STRIDE=rope.stride(3),
ROW_STRIDE=rope.stride(4),
COL_STRIDE=rope.stride(5),
BLOCK=512,
enable_fp_fusion=False,
)
return output
@@ -0,0 +1,147 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky 6 joint video/audio checkpoint architecture.
Distilled checkpoints have n_grid velocity outputs per latent channel."""
from dataclasses import dataclass, field
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
@dataclass
class Kandinsky6ArchConfig(DiTArchConfig):
"""Checkpoint fields and derived dimensions for Kandinsky6Transformer3DModel."""
# Chained layer renames apply to checkpoint tensors and LoRA A/B/alpha keys.
param_names_mapping: dict = field(
default_factory=lambda: {
r"^(visual_transformer_blocks\.\d+)\.video_dec_block\.(.*)$": r"\1.videoT.\2",
r"^(visual_transformer_blocks\.\d+)\.audio_dec_block\.(.*)$": r"\1.audioT.\2",
r"^(.*feed_forward)\.net\.0\.proj\.(.*)$": r"\1.mlp.fc_in.\2",
r"^(.*feed_forward)\.net\.2\.(.*)$": r"\1.mlp.fc_out.\2",
r"^(video_time_embeddings|audio_time_embeddings)\.timestep_embedder\.linear_1\.(.*)$": r"\1.in_layer.\2",
r"^(video_time_embeddings|audio_time_embeddings)\.timestep_embedder\.linear_2\.(.*)$": r"\1.out_layer.\2",
r"^((?:video|audio)_text_transformer_blocks\.\d+)\.attn_norm\.(.*)$": r"\1.self_attention_norm.\2",
r"^((?:video|audio)_text_transformer_blocks\.\d+)\.attn\.(.*)$": r"\1.self_attention.\2",
}
)
lora_param_names_mapping: dict = field(
default_factory=lambda: {r"^transformer\.(.*)$": r"\1"}
)
# Diffusers Kandinsky6Transformer3DModel config fields (mirror
# transformer/config.json 1:1).
in_visual_dim: int = 16
out_visual_dim: int = 16
in_text_dim: int = 3584 # Qwen2.5-7B hidden size (Reason1 text encoder)
in_text_dim2: int = 768 # CLIP pooled dim
time_dim: int = 1024
patch_size: tuple[int, int, int] = (1, 2, 2)
# checkpoint RoPE scaling is fixed across request resolutions
scale_factor: tuple[float, float, float] = (1.0, 2.0, 2.0)
model_dim: int = 4096
ff_dim: int = 16384
num_text_blocks: int = 4
num_visual_blocks: int = 60
axes_dims: tuple[int, int, int] = (32, 48, 48) # 3D RoPE T/H/W split
visual_cond: bool = True
# False selects the video-only architecture; TI2VA pipelines use True
is_multimodal: bool = True
out_audio_dim: int | None = None
in_audio_dim: int = 20
model_dim_a: int | None = None
time_dim_a: int | None = None
ff_dim_a: int | None = None
axes_dims_a: tuple[int, int, int] | None = None
audio_freqs_scaling: float = 1.0
# checkpoint metadata, not a runtime backend override; NABLA is rejected
attention_engine: str = "auto"
attention_causal: bool | None = None
attention_local: bool | None = None
attention_glob: bool | None = None
attention_window: int | None = None
attention_P: float | None = None
attention_wT: int | None = None
attention_wW: int | None = None
attention_wH: int | None = None
attention_add_sta: bool | None = None
attention_method: str | None = None
# checkpoint metadata; the pipeline passes unpadded text rather than masks
text_token_padding: bool = False
# Video<->audio fused block knobs (Kandinsky6FusedTransformerDecoderBlock).
ca_rope: bool = False
cross_gates: bool = False
fix_modulation: bool = False
# generated/reference token embeddings for tail-frame image conditioning
visual_token_type_num_embeddings: int = 2
# Derived, DiTArchConfig-contract fields (set in __post_init__ below;
# not part of the diffusers config.json).
in_channels: int = 0
out_channels: int = 0
def __post_init__(self) -> None:
super().__post_init__()
if isinstance(self.patch_size, list):
self.patch_size = tuple(self.patch_size)
if len(self.patch_size) != 3:
raise ValueError(f"patch_size must have 3 values, got {self.patch_size}.")
if isinstance(self.scale_factor, list):
self.scale_factor = tuple(self.scale_factor)
if len(self.scale_factor) != 3:
raise ValueError(
f"scale_factor must have 3 values, got {self.scale_factor}."
)
self.scale_factor = tuple(float(value) for value in self.scale_factor)
head_dim = sum(self.axes_dims)
if self.model_dim % head_dim != 0:
raise ValueError(
f"model_dim ({self.model_dim}) must be divisible by head_dim "
f"({head_dim})"
)
self.hidden_size = self.model_dim
self.num_attention_heads = self.model_dim // head_dim
self.in_channels = self.in_visual_dim
self.out_channels = self.out_visual_dim
self.num_channels_latents = self.in_visual_dim
# resolve omitted audio dimensions from the video tower
self.model_dim_a = self.model_dim_a or self.model_dim
self.time_dim_a = self.time_dim_a or self.time_dim
self.ff_dim_a = self.ff_dim_a or self.ff_dim
self.axes_dims_a = self.axes_dims_a or self.axes_dims
head_dim_a = sum(self.axes_dims_a)
if self.model_dim_a % head_dim_a != 0:
raise ValueError(
f"model_dim_a ({self.model_dim_a}) must be divisible by "
f"head_dim_a ({head_dim_a})"
)
# without fix_modulation, cross-modal modulation uses the other tower's time
# embedding, so video and audio time widths must match
if (
self.is_multimodal
and not self.fix_modulation
and self.time_dim_a != self.time_dim
):
raise ValueError(
f"time_dim_a ({self.time_dim_a}) must equal time_dim "
f"({self.time_dim}) unless fix_modulation=True "
"(Kandinsky6FusedTransformerDecoderBlock's cross-modal "
"modulation is driven by the other modality's time "
"embedding by default)."
)
@dataclass
class Kandinsky6VideoAudioConfig(DiTConfig):
arch_config: DiTArchConfig = field(default_factory=Kandinsky6ArchConfig)
prefix: str = "Kandinsky6"
__all__ = ["Kandinsky6ArchConfig", "Kandinsky6VideoAudioConfig"]
@@ -0,0 +1,269 @@
# SPDX-License-Identifier: Apache-2.0
"""SR checkpoint architecture and sampling metadata.
Official out_visual_dim is the total DX head width; n_grid lives in the scheduler.
Unsupported architecture options and unknown fields are rejected."""
from dataclasses import dataclass, field
from typing import Any
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
WIDE_INPUT_INSTRUCT_TYPES = ("channel", "hybrid", "hybrid_anchor")
SUPPORTED_INSTRUCT_TYPES = ("noise", *WIDE_INPUT_INSTRUCT_TYPES)
# Post-load behaviour overrides the port accepts (no blind ``setattr``).
ATTRIBUTE_OVERRIDE_WHITELIST = ("instruct_type", "visual_cond", "attention_params")
# Bookkeeping keys the loader leaves in the component config.
_TOLERATED_EXTRA_KEYS = frozenset({"_class_name", "_diffusers_version"})
_SPARSE_ATTENTION_TYPES = ("nabla", "nabla_framewise_causal")
_PIFLOW_FIELDS = (
"piflow_nfe",
"piflow_num_policy_substeps",
"piflow_final_step_size_scale",
"piflow_shift",
"piflow_eps",
)
_LQ_NOISE_TYPES = ("ddpm", "linear")
@dataclass
class Kandinsky6SRArchConfig(DiTArchConfig):
"""Static architecture metadata of ``Kandinsky6SRTransformer3DModel``."""
# Map the current Diffusers checkpoint names to the native SGLang port.
param_names_mapping: dict = field(
default_factory=lambda: {
r"^(.*feed_forward)\.net\.0\.proj\.(weight|bias)$": r"\1.mlp.fc_in.\2",
r"^(.*feed_forward)\.net\.2\.(weight|bias)$": r"\1.mlp.fc_out.\2",
r"^time_embeddings\.timestep_embedder\.linear_1\.(weight|bias)$": r"time_embeddings.in_layer.\1",
r"^time_embeddings\.timestep_embedder\.linear_2\.(weight|bias)$": r"time_embeddings.out_layer.\1",
}
)
# --- DiffusionTransformer3D constructor keys (trained architecture) ---
in_visual_dim: int = 4
in_text_dim: int = 3584
in_text_dim2: int = 768
time_dim: int = 512
out_visual_dim: int = 4
patch_size: tuple[int, int, int] = (1, 2, 2)
model_dim: int = 2048
ff_dim: int = 5120
num_text_blocks: int = 2
num_visual_blocks: int = 32
axes_dims: tuple[int, int, int] = (16, 24, 24)
visual_cond: bool = False
instruct_type: str | None = None
attention_params: dict[str, Any] | None = None
use_motion_score: bool = False
use_lq_modulation: bool = False
lq_modulation_sublayers: dict[str, bool] | None = None
zero_lq_in_main_path: bool = False
use_adapter: bool = False
adapter_gamma: float = 0.0
# The reference default is True; the SR release is text-free and its config says
# so explicitly.
use_text: bool = False
use_lq_noise_cond: bool = False
# --- optional legacy / test-only DX head / pi-Flow fields ---
# An official checkpoint stores the TOTAL head width in ``out_visual_dim`` and the
# pi-Flow values in its scheduler config; with ``n_grid > 1`` here the head is
# ``out_visual_dim * n_grid`` wide instead (``out_visual_dim`` being the base dim).
n_grid: int = 1
piflow_nfe: int | None = None
piflow_num_policy_substeps: int | None = None
piflow_final_step_size_scale: float | None = None
piflow_shift: float | None = None
piflow_eps: float | None = None
# Post-load attribute overrides (whitelist: instruct_type, visual_cond,
# attention_params); the architecture above stays as trained. The official
# config writes ``null`` when there are none.
attribute_overrides: dict[str, Any] | None = field(default_factory=dict)
# --- SR sampling parameters of the training config ---
# As stored in ``transformer/config.json`` (``sr_params``); expanded into the
# flat ``sr_*`` fields below in ``__post_init__``.
sr_params: dict[str, Any] | None = None
sr_visual_size: list[int] = field(default_factory=lambda: [512])
sr_scale_factor: dict[str, list[float]] | None = None
sr_scheduler_scale: float = 5.0
sr_lq_noise_scale: float = 0.7
sr_lq_noise_type: str = "ddpm"
sr_lq_channel_noise_scale: float = 0.0
sr_cap_noise_timestep: bool = False
sr_fps: int = 24
# DiTArchConfig contract fields, derived in __post_init__.
in_channels: int = 0
out_channels: int = 0
def __post_init__(self) -> None:
super().__post_init__()
if self.attribute_overrides is None:
self.attribute_overrides = {}
if self.sr_params is not None:
self._expand_sr_params(self.sr_params)
self._reject_unknown_keys()
self._reject_unsupported_flags()
self._normalize_and_validate_shapes()
self._validate_overrides()
self._validate_sampler_fields()
self.hidden_size = self.model_dim
self.num_attention_heads = self.model_dim // sum(self.axes_dims)
self.in_channels = self.in_visual_dim
self.out_channels = self.out_visual_dim
self.num_channels_latents = self.in_visual_dim
@property
def is_piflow(self) -> bool:
return self.piflow_nfe is not None
@property
def head_width(self) -> int:
"""Channels the DiT head emits per latent position (all grids of a DX head)."""
return self.out_visual_dim * self.n_grid
@property
def base_out_visual_dim(self) -> int:
"""Channels per latent grid; legacy n_grid configs store the base output width."""
return self.out_visual_dim if self.n_grid > 1 else self.in_visual_dim
@property
def trained_wide_input(self) -> bool:
"""Whether the input layer was trained on ``[x | cond | mask]`` channels."""
return bool(self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES)
def effective_override(self, name: str) -> Any:
"""Post-load value of ``name``: the override if present, else as trained."""
if name in self.attribute_overrides:
return self.attribute_overrides[name]
return getattr(self, name)
def requested_sparse_attention(self) -> str | None:
"""Sparse attention type asked for by ``attention_params``, if any."""
params = self.effective_override("attention_params")
if not isinstance(params, dict):
return None
for value in params.values():
attention_type = value.get("type") if isinstance(value, dict) else None
if attention_type in _SPARSE_ATTENTION_TYPES:
return attention_type
return None
_SR_PARAM_FIELDS = {
"visual_size": "sr_visual_size",
"scale_factor": "sr_scale_factor",
"scheduler_scale": "sr_scheduler_scale",
"lq_noise_scale": "sr_lq_noise_scale",
"lq_noise_type": "sr_lq_noise_type",
"lq_channel_noise_scale": "sr_lq_channel_noise_scale",
"cap_noise_timestep": "sr_cap_noise_timestep",
"fps": "sr_fps",
}
def _expand_sr_params(self, sr_params: dict[str, Any]) -> None:
"""Nested ``sr_params`` of ``transformer/config.json`` -> the flat ``sr_*`` fields."""
unknown = sorted(set(sr_params) - set(self._SR_PARAM_FIELDS))
if unknown:
raise ValueError(
f"Kandinsky6SR sr_params has unsupported keys {unknown}; accepted "
f"keys: {sorted(self._SR_PARAM_FIELDS)}"
)
for key, value in sr_params.items():
setattr(self, self._SR_PARAM_FIELDS[key], value)
def _reject_unknown_keys(self) -> None:
unknown = sorted(set(self.extra_attrs) - _TOLERATED_EXTRA_KEYS)
if unknown:
raise ValueError(
"Kandinsky6SR transformer config has unknown keys "
f"{unknown}; declare them on Kandinsky6SRArchConfig or fix the checkpoint config."
)
def _reject_unsupported_flags(self) -> None:
if self.use_text:
raise NotImplementedError(
"Kandinsky6SR: use_text=True (text-conditioned SR DiT) is not "
"supported; only the text-free SR DiT (use_text=False) is implemented."
)
if self.use_adapter:
raise NotImplementedError(
"Kandinsky6SR: use_adapter=True (VideoAdapter) is not supported."
)
if self.use_lq_modulation:
raise NotImplementedError(
"Kandinsky6SR: use_lq_modulation=True (per-block LQ modulation) is "
"not supported."
)
def _normalize_and_validate_shapes(self) -> None:
self.patch_size = tuple(self.patch_size)
self.axes_dims = tuple(self.axes_dims)
if len(self.patch_size) != 3 or self.patch_size[0] != 1:
raise ValueError(
"Kandinsky6SR patch_size must be (1, ph, pw) with a unit temporal "
f"patch, got {self.patch_size}"
)
if len(self.axes_dims) != 3 or any(dim % 2 for dim in self.axes_dims):
raise ValueError(
f"axes_dims must be three even sizes, got {self.axes_dims}"
)
head_dim = sum(self.axes_dims)
if self.model_dim % head_dim != 0:
raise ValueError(
f"model_dim ({self.model_dim}) must be divisible by head_dim "
f"({head_dim}) = sum(axes_dims)"
)
if self.model_dim % 2 != 0:
raise ValueError(f"model_dim must be even, got {self.model_dim}")
if self.n_grid < 1:
raise ValueError(f"n_grid must be >= 1, got {self.n_grid}")
def _validate_overrides(self) -> None:
unknown = sorted(
set(self.attribute_overrides) - set(ATTRIBUTE_OVERRIDE_WHITELIST)
)
if unknown:
raise ValueError(
f"attribute_overrides keys {unknown} are not allowed; "
f"supported keys: {list(ATTRIBUTE_OVERRIDE_WHITELIST)}"
)
instruct_type = self.effective_override("instruct_type")
if instruct_type not in (None, *SUPPORTED_INSTRUCT_TYPES):
raise ValueError(
f"instruct_type must be one of {list(SUPPORTED_INSTRUCT_TYPES)}, "
f"got {instruct_type!r}"
)
visual_cond = self.effective_override("visual_cond")
if not isinstance(visual_cond, bool):
raise ValueError(f"visual_cond must be a bool, got {visual_cond!r}")
def _validate_sampler_fields(self) -> None:
present = [name for name in _PIFLOW_FIELDS if getattr(self, name) is not None]
if present and len(present) != len(_PIFLOW_FIELDS):
missing = sorted(set(_PIFLOW_FIELDS) - set(present))
raise ValueError(
f"pi-Flow config is incomplete: {missing} missing next to {present}"
)
if self.sr_lq_noise_type not in _LQ_NOISE_TYPES:
raise ValueError(
f"sr_lq_noise_type must be one of {list(_LQ_NOISE_TYPES)}, "
f"got {self.sr_lq_noise_type!r}"
)
@dataclass
class Kandinsky6SRDitConfig(DiTConfig):
arch_config: DiTArchConfig = field(default_factory=Kandinsky6SRArchConfig)
prefix: str = "Kandinsky6SR"
__all__ = [
"ATTRIBUTE_OVERRIDE_WHITELIST",
"Kandinsky6SRArchConfig",
"Kandinsky6SRDitConfig",
"SUPPORTED_INSTRUCT_TYPES",
"WIDE_INPUT_INSTRUCT_TYPES",
]
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
"""Reason1 text encoder, reusing the Qwen2.5-VL runtime without its vision tower.
Defaults describe the 3584-wide language trunk; checkpoint config overrides them."""
from dataclasses import dataclass, field
from typing import Any
from sglang.multimodal_gen.configs.models.encoders.base import (
TextEncoderArchConfig,
TextEncoderConfig,
)
from sglang.multimodal_gen.configs.models.fsdp import (
is_embed_tokens,
is_final_norm,
is_layer,
)
@dataclass
class Reason1ArchConfig(TextEncoderArchConfig):
"""Architecture metadata (defaults match Qwen2.5-VL-7B-Instruct)."""
architectures: list[str] = field(
default_factory=lambda: ["Qwen2_5_VLForConditionalGeneration"]
)
vocab_size: int = 152064
hidden_size: int = 3584
intermediate_size: int = 18944
num_hidden_layers: int = 28
num_attention_heads: int = 28
num_key_value_heads: int = 4
hidden_act: str = "silu"
max_position_embeddings: int = 128000
initializer_range: float = 0.02
rms_norm_eps: float = 1e-6
use_cache: bool = False
tie_word_embeddings: bool = False
attention_dropout: float = 0.0
rope_theta: float = 1_000_000.0
# 3D mRoPE (temporal/height/width) section split; Qwen2.5-VL-specific.
rope_scaling: dict[str, Any] | None = field(
default_factory=lambda: {"type": "mrope", "mrope_section": [16, 24, 24]}
)
# Kandinsky6's text-conditioning stack always consumes the last hidden
# state (see kandinsky6_qwen_postprocess_text in
# configs/pipeline_configs/kandinsky6.py), so hidden states must be kept.
output_hidden_states: bool = True
hidden_state_skip_layer: int = 0
# text_len caps user tokens; tokenizer length also includes the 129-token template
text_len: int = 512
bos_token_id: int = 151643
pad_token_id: int = 151643
eos_token_id: int = 151645
vision_start_token_id: int = 151652
vision_end_token_id: int = 151653
vision_token_id: int = 151654
image_token_id: int = 151655
video_token_id: int = 151656
# Same fused-projection naming as the sglang qwen2_5vl/qwen_image runtime
# encoders (MergedColumnParallelLinear "gate_up_proj", fused "qkv_proj").
stacked_params_mapping: list[tuple[str, str, str]] = field(
default_factory=lambda: [
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0), # type: ignore
(".gate_up_proj", ".up_proj", 1), # type: ignore
]
)
_fsdp_shard_conditions: list = field(
default_factory=lambda: [is_layer, is_embed_tokens, is_final_norm]
)
@dataclass
class Reason1Config(TextEncoderConfig):
arch_config: TextEncoderArchConfig = field(default_factory=Reason1ArchConfig)
prefix: str = "reason1"
__all__ = ["Reason1ArchConfig", "Reason1Config"]
@@ -0,0 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
from sglang.multimodal_gen.configs.models.upsamplers.kandinsky6_sr import (
Kandinsky6SRLatentUpscalerEntryConfig,
)
__all__ = ["Kandinsky6SRLatentUpscalerEntryConfig"]
@@ -0,0 +1,225 @@
# SPDX-License-Identifier: Apache-2.0
"""Configuration of an entry in the SR latent-upscaler bank.
Only released architectures are implemented; reject unsupported settings."""
from __future__ import annotations
from collections.abc import Mapping
from typing import Any
import msgspec
# key -> (required value, value a config that omits the key stands for). The omitted-key values
# are the training code's defaults, so e.g. a config without ``input_skip`` means
# ``input_skip=True`` and is rejected.
_FIXED_KEYS: dict[str, tuple[Any, Any]] = {
"upscale_factor": (4, 4),
"dims": (3, 2),
"temporal_padding": ("replicate", "zeros"),
"upsample_mode": ("pxs_v2", "pixel_shuffle"),
"upsample_padding_mode": ("zeros", "reflect"),
"modulated_norm": (True, False),
"modulated_output_proj": (True, False),
"bare_stem": (True, False),
"input_skip": (False, True),
"global_skip": (False, False),
"grn": (False, False),
"layer_scale_init": (None, None),
"depthwise": (False, False),
"bottleneck_channels": (None, None),
"motion_attention": (None, None),
# Only read by depthwise blocks; the residual convs are always 3x3x3.
"kernel_size": (3, 3),
}
# Fixed for entries built with the x2 entry (``enable_x2_entry=true``).
_X2_FIXED_KEYS: dict[str, tuple[Any, Any]] = {
"x2_tail_mode": ("private_full", "shared"),
"x2_finisher": ("pxs_residual", "none"),
}
# Accepted without effect at inference: training-time settings, and switches that only apply to
# the pixel-shuffle upsample modes (the pxs_v2 upsample has no temporal kernel and no ICNR init).
_IGNORED_KEYS = frozenset(
{
"gradient_checkpointing",
"stochastic_depth_rate",
"loss_weight_2x",
"loss_weight_4x",
"temporal_mix",
"icnr",
"upsample_position",
}
)
_REQUIRED_INT_KEYS = (
"in_channels",
"hidden_channels",
"num_pre_blocks",
"num_mid_blocks",
"num_post_blocks",
"expand_ratio",
)
_SUPPORTED_SCALES = (2, 4)
def _positive_int(where: str, key: str, value: Any) -> int:
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
raise ValueError(f"{where}: `{key}` must be a positive integer, got {value!r}")
return value
def _parse_target_scale(where: str, value: Any) -> int:
text = str(value).removesuffix("x")
if (
isinstance(value, bool)
or not text.isdigit()
or int(text) not in _SUPPORTED_SCALES
):
raise ValueError(
f"{where}: `target_scale` must be one of '2x', '4x', got {value!r}"
)
return int(text)
class Kandinsky6SRLatentUpscalerEntryConfig(msgspec.Struct, frozen=True, kw_only=True):
"""Cascaded 2x+2x upsampler, optionally with a private x2 entry path."""
target_scale: int
in_channels: int
hidden_channels: int
stage_channels: tuple[int, int, int]
num_pre_blocks: int
num_mid_blocks: int
num_post_blocks: int
expand_ratio: int
enable_x2_entry: bool = False
x2_adapter_blocks: int = 0
@classmethod
def from_dict(
cls, spec: Any, index: int = 0
) -> Kandinsky6SRLatentUpscalerEntryConfig:
"""Validate one ``models[index]`` entry of ``latent_upscaler/config.json``."""
where = f"latent upscaler models[{index}]"
if (
not isinstance(spec, Mapping)
or set(spec) != {"target_scale", "model"}
or not isinstance(spec["model"], Mapping)
):
raise ValueError(
f"{where} must be a mapping with exactly `target_scale` and a `model` "
f"mapping, got {spec!r}"
)
target_scale = _parse_target_scale(where, spec["target_scale"])
model = dict(spec["model"])
if model.get("architecture") != "multi_scale":
raise ValueError(
f"{where}: `architecture` must be 'multi_scale', got "
f"{model.get('architecture')!r}"
)
enable_x2_entry = model.get("enable_x2_entry", False)
if not isinstance(enable_x2_entry, bool):
raise ValueError(
f"{where}: `enable_x2_entry` must be a boolean, got {enable_x2_entry!r}"
)
fixed = dict(_FIXED_KEYS)
if enable_x2_entry:
fixed.update(_X2_FIXED_KEYS)
for key, (required, omitted) in fixed.items():
value = model.get(key, omitted)
if value != required or type(value) is not type(required):
raise ValueError(
f"{where}: unsupported `{key}`={value!r}"
f"{' (omitted)' if key not in model else ''}; only {required!r} is "
"implemented"
)
x2_defaults = {
"x2_adapter_blocks": 0,
"x2_adapter_sources": None,
"x2_tail_mode": "shared",
"x2_finisher": "none",
}
known = {
"architecture",
"enable_x2_entry",
"stage_channels",
*_REQUIRED_INT_KEYS,
*fixed,
*x2_defaults,
*_IGNORED_KEYS,
}
unknown = sorted(set(model) - known)
if unknown:
raise ValueError(f"{where}: unknown keys {unknown}")
if not enable_x2_entry:
present = sorted(
key
for key, default in x2_defaults.items()
if model.get(key, default) != default
)
if present:
raise ValueError(f"{where}: {present} require `enable_x2_entry`=true")
if target_scale == 2 and not enable_x2_entry:
raise ValueError(
f"{where}: `target_scale`='2x' requires `enable_x2_entry`=true"
)
missing = [key for key in _REQUIRED_INT_KEYS if key not in model]
if missing:
raise ValueError(f"{where}: missing keys {missing}")
ints = {
key: _positive_int(where, key, model[key]) for key in _REQUIRED_INT_KEYS
}
hidden = ints["hidden_channels"]
raw_stages = model.get("stage_channels")
if raw_stages is None:
stage_channels = (hidden, hidden, hidden)
else:
if not isinstance(raw_stages, (list, tuple)) or len(raw_stages) != 3:
raise ValueError(
f"{where}: `stage_channels` must list 3 widths, got {raw_stages!r}"
)
first, second, third = (
_positive_int(where, "stage_channels", width) for width in raw_stages
)
stage_channels = (first, second, third)
if stage_channels[0] != hidden:
raise ValueError(
f"{where}: `hidden_channels` ({hidden}) must equal `stage_channels[0]` "
f"({stage_channels[0]})"
)
x2_adapter_blocks = 0
if enable_x2_entry:
x2_adapter_blocks = _positive_int(
where, "x2_adapter_blocks", model.get("x2_adapter_blocks", 0)
)
# Training warm-starts the adapter from these pre_blocks; only their consistency
# matters here.
sources = model.get("x2_adapter_sources")
if sources is not None and (
not isinstance(sources, (list, tuple))
or len(sources) != x2_adapter_blocks
or any(
not isinstance(i, int) or not 0 <= i < ints["num_pre_blocks"]
for i in sources
)
):
raise ValueError(
f"{where}: `x2_adapter_sources` must list {x2_adapter_blocks} indices in "
f"[0, num_pre_blocks={ints['num_pre_blocks']}), got {sources!r}"
)
return cls(
target_scale=target_scale,
stage_channels=stage_channels,
enable_x2_entry=enable_x2_entry,
x2_adapter_blocks=x2_adapter_blocks,
**ints,
)
__all__ = ["Kandinsky6SRLatentUpscalerEntryConfig"]
@@ -0,0 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
"""Checkpoint configuration for the bundled MMAudio codec and BigVGAN vocoder."""
from dataclasses import dataclass, field
from typing import Any
from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig
@dataclass
class Kandinsky6AudioVAEArchConfig(VAEArchConfig):
mode: str = "44k"
# The two flags below are not read by the runtime module: the official
# checkpoints carry the encoder half's weights, so ``Kandinsky6AudioVAE``
# always builds both halves of the nested MMAudioVAE.
need_vae_decoder: bool = True
need_vae_encoder: bool = False
scaling_factor: float = 0.5302
# Reverses a checkpoint's own audio-latent shift, matching the diffusers
# reference's postprocess_audio (getattr(audio_vae, "mean_value", 0.0));
# no real checkpoint has been observed setting this to anything but 0.0
# yet.
mean_value: float = 0.0
# BigVGAN-v2 hyperparameters, passed straight through to the runtime
# BigVGANV2(...) constructor unmodified.
vocoder_config: dict[str, Any] = field(default_factory=dict)
@dataclass
class Kandinsky6AudioVAEConfig(VAEConfig):
arch_config: VAEArchConfig = field(default_factory=Kandinsky6AudioVAEArchConfig)
# Generation only decodes audio; the runtime module does not read these.
load_encoder: bool = False
load_decoder: bool = True
__all__ = ["Kandinsky6AudioVAEArchConfig", "Kandinsky6AudioVAEConfig"]
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
"""SR causal KVAE config supporting nested and flat checkpoint layouts.
Normalize flat fields into encoder_config/decoder_config; reject unknown fields."""
from dataclasses import dataclass, field
from typing import Any
from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig
_TOLERATED_EXTRA_KEYS = frozenset({"_class_name", "_diffusers_version"})
@dataclass
class Kandinsky6SRVAEArchConfig(VAEArchConfig):
vae_type: str = "video-kvae"
encoder_config: dict[str, Any] = field(default_factory=dict)
decoder_config: dict[str, Any] = field(default_factory=dict)
scaling_factor: float = 1.0
spatial_factor: int = 16
temporal_factor: int = 4
# Flat top-level KVAE architecture fields (current official vae/config.json layout -- see
# module docstring). Left at their None default (and ignored) when the checkpoint instead
# uses the older nested encoder_config/decoder_config layout.
in_channels: int | None = None
out_channels: int | None = None
z_channels: int | None = None
ch: int | None = None
ch_mult: list[float] | None = None
decoder_ch: int | None = None
decoder_ch_mult: list[float] | None = None
num_res_blocks: int | None = None
padding_mode: str | None = None
temporal_compress_times: int | None = None
temporal_compress_start_level: int | None = None
norm_type: str | None = None
double_z: bool | None = None
downsample_version: int | None = None
fix_pxs: bool | None = None
# Present in the flat layout but unused by Kandinsky6SRVAE (the KVAE is resolution-agnostic);
# declared only so it doesn't trip the unknown-top-level-key check below.
resolution: int | None = None
def __post_init__(self) -> None:
unknown = sorted(set(self.extra_attrs) - _TOLERATED_EXTRA_KEYS)
if unknown:
raise ValueError(f"Kandinsky6SR VAE config has unknown keys {unknown}")
if self.vae_type != "video-kvae":
raise ValueError(
f"Kandinsky6SRVAE supports only 'video-kvae', got {self.vae_type!r}"
)
self.spatial_compression_ratio = self.spatial_factor
self.temporal_compression_ratio = self.temporal_factor
if not self.encoder_config and not self.decoder_config and self.ch is not None:
shared = dict(
num_res_blocks=self.num_res_blocks,
z_channels=self.z_channels,
temporal_compress_times=self.temporal_compress_times,
temporal_compress_start_level=self.temporal_compress_start_level,
norm_type=self.norm_type,
padding_mode=self.padding_mode,
)
self.encoder_config = {
**shared,
"ch": self.ch,
"ch_mult": self.ch_mult,
"in_channels": self.in_channels,
"double_z": self.double_z,
"downsample_version": self.downsample_version,
"fix_pxs": self.fix_pxs,
}
self.decoder_config = {
**shared,
"ch": self.decoder_ch,
"ch_mult": self.decoder_ch_mult,
"out_ch": self.out_channels,
}
@dataclass
class Kandinsky6SRVAEConfig(VAEConfig):
arch_config: VAEArchConfig = field(default_factory=Kandinsky6SRVAEArchConfig)
# SR stages own tile geometry; parallel tiling distributes these existing tiles
# without enabling the generic VAE tiling or spatial-shard decode paths
use_tiling: bool = False
use_temporal_tiling: bool = False
use_parallel_tiling: bool = False
use_parallel_decode: bool = False
def get_vae_scale_factor(self) -> int:
return self.arch_config.spatial_factor
__all__ = ["Kandinsky6SRVAEArchConfig", "Kandinsky6SRVAEConfig"]
@@ -0,0 +1,248 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky 6 text-to-video/audio pipeline with optional image conditioning."""
from dataclasses import dataclass, field
from typing import Any, Callable
import torch
from sglang.multimodal_gen.configs.models import EncoderConfig
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
Kandinsky6VideoAudioConfig,
)
from sglang.multimodal_gen.configs.models.encoders import (
BaseEncoderOutput,
CLIPTextConfig,
)
from sglang.multimodal_gen.configs.models.encoders.kandinsky6_reason1 import (
Reason1Config,
)
from sglang.multimodal_gen.configs.models.vaes.hunyuanvae import HunyuanVAEConfig
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
Kandinsky6AudioVAEConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ModelTaskType,
PipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
_is_kandinsky6_sr,
)
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import (
qwen_image_postprocess_text,
)
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy
# checkpoint-trained template: preserve its exact bytes, including misspellings
KANDINSKY6_PROMPT_TEMPLATE = "\n".join(
[
"<|im_start|>system\nYou are a promt engineer. Describe the video in detail.", # codespell:ignore promt
"Describe how the camera moves or shakes, describe the zoom and view angle, whether it follows the objects.",
"Describe the location of the video, main characters or objects and their action.",
"Describe the dynamism of the video and presented actions.",
"Name the visual style of the video: whether it is a professional footage, user generated content, some kind of animation, video game or scren content.", # codespell:ignore scren
"Describe the visual effects, postprocessing and transitions if they are presented in the video.",
"Pay attention to the order of key actions shown in the scene.<|im_end|>",
"<|im_start|>user\n{}<|im_end|>",
]
)
# Reason1 template prefix length before user tokens
KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX = 129
def kandinsky6_qwen_preprocess_text(prompt: str) -> str:
if not prompt.strip():
prompt = "."
return KANDINSKY6_PROMPT_TEMPLATE.format(prompt)
def kandinsky6_qwen_postprocess_text(
outputs: BaseEncoderOutput,
text_inputs,
return_attention_mask: bool = False,
):
"""Strip padding and the Reason1 template prefix, then repad embeddings and mask."""
return qwen_image_postprocess_text(
outputs,
text_inputs,
drop_idx=KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX,
return_attention_mask=return_attention_mask,
)
def kandinsky6_clip_postprocess_text(
outputs: BaseEncoderOutput, _text_inputs
) -> torch.Tensor:
return outputs.pooler_output
@dataclass
class Kandinsky6TI2VAPipelineConfig(PipelineConfig):
"""Configuration for the Kandinsky6 TI2VA pipeline."""
task_type: ModelTaskType = ModelTaskType.TI2V
native_only_components: tuple[str, ...] = (
"transformer",
"text_encoder",
"text_encoder_2",
"vae",
"audio_vae",
)
# image conditioning is handled by Kandinsky6ImageEncodingStage, not Wan TI2V preprocessing
skip_input_image_preprocess: bool = True
# a single joint DiT consumes video and audio latents
dit_config: Kandinsky6VideoAudioConfig = field(
default_factory=Kandinsky6VideoAudioConfig
)
dit_precision: str = "bf16"
cfg_policy: CFGPolicy = field(
default_factory=lambda: CFGPolicy(parallel_uses_serial_arithmetic=True)
)
# keep tiled decode: whole-clip sharding materializes a quadratic temporal mask
vae_config: HunyuanVAEConfig = field(
default_factory=lambda: HunyuanVAEConfig(parallel_decode_mode="tiled")
)
vae_precision: str = "bf16"
vae_tiling: bool = True
# audio_vae includes both the mel codec and vocoder
audio_vae_config: Kandinsky6AudioVAEConfig = field(
default_factory=Kandinsky6AudioVAEConfig
)
audio_vae_precision: str = "bf16"
# Reason1 provides token embeddings; CLIP provides pooled text conditioning
text_encoder_configs: tuple[EncoderConfig, ...] = field(
default_factory=lambda: (Reason1Config(), CLIPTextConfig())
)
text_encoder_precisions: tuple[str, ...] = field(
default_factory=lambda: ("bf16", "bf16")
)
preprocess_text_funcs: tuple[Callable[[str], str] | None, ...] = field(
default_factory=lambda: (kandinsky6_qwen_preprocess_text, None)
)
postprocess_text_funcs: tuple[Callable[..., Any], ...] = field(
default_factory=lambda: (
kandinsky6_qwen_postprocess_text,
kandinsky6_clip_postprocess_text,
)
)
# Reason1: 129 template + 512 user tokens; CLIP: fixed 77-token context
text_encoder_extra_args: list[dict] = field(
default_factory=lambda: [
dict(
max_length=KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX + 512,
padding=True,
truncation=True,
),
dict(
max_length=77,
padding="max_length",
truncation=True,
add_special_tokens=True,
),
]
)
# Shift of both official checkpoints' schedulers (FlowMatchEulerDiscreteScheduler
# for Pro-sft, PiflowScheduler for Pro-distill: scheduler_config.json "shift").
flow_shift: float | None = 5.0
# audio frames = ceil(pixel_frames / fps * sample_rate / downsample_factor)
sample_fps: float = 24.0
audio_sample_rate: int = 44100
audio_downsample_factor: int = 1024
def __post_init__(self) -> None:
if len(self.text_encoder_configs) != 2:
raise ValueError(
"Kandinsky6 pipeline requires exactly 2 text encoders "
f"(Reason1 and CLIP), but got {len(self.text_encoder_configs)}."
)
if len(self.text_encoder_precisions) != 2:
raise ValueError(
"Kandinsky6 pipeline requires exactly 2 text encoder "
f"precisions, but got {len(self.text_encoder_precisions)}."
)
if len(self.text_encoder_extra_args) != 2:
raise ValueError(
"Kandinsky6 pipeline requires exactly 2 text encoder extra "
f"tokenizer arg dicts, but got {len(self.text_encoder_extra_args)}."
)
# keep the video VAE encoder available for optional image conditioning
self.vae_config.load_encoder = True
self.vae_config.load_decoder = True
def supports_disaggregation(self) -> bool:
# joint denoising currently runs monolithically
return False
def get_text_encoder_pooler_output(self, outputs, encoder_index):
# only CLIP (index 1) returns pooler_output; Reason1 returns causal-LM output
if encoder_index == 1:
return outputs.pooler_output
return None
def get_pos_prompt_embeds(self, batch):
return batch.prompt_embeds[0]
def get_neg_prompt_embeds(self, batch):
return batch.negative_prompt_embeds[0]
def tokenize_prompt(self, prompt, tokenizer, tok_kwargs) -> dict:
# Reason1 uses a multimodal processor: positional input would bind to images,
# not text. The conditioning image is handled separately by the VAE.
return tokenizer(text=prompt, **tok_kwargs)
def _is_kandinsky6_t2va_family(model_id: str) -> bool:
normalized = model_id.lower().replace("-", "").replace("_", "")
return "kandinsky6" in normalized and not _is_kandinsky6_sr(model_id)
def _is_kandinsky6_t2va_distilled(model_id: str) -> bool:
short_name = model_id.lower().rstrip("/").split("/")[-1]
return _is_kandinsky6_t2va_family(model_id) and "distill" in short_name
def _is_kandinsky6_t2va(model_id: str) -> bool:
short_name = model_id.lower().rstrip("/").split("/")[-1]
return _is_kandinsky6_t2va_family(model_id) and "distill" not in short_name
def register():
from sglang.multimodal_gen.configs.sample.kandinsky6 import (
Kandinsky6TI2VADistilledSamplingParams,
Kandinsky6TI2VASamplingParams,
)
from sglang.multimodal_gen.registry import register_configs
register_configs(
sampling_param_cls=Kandinsky6TI2VADistilledSamplingParams,
pipeline_config_cls=Kandinsky6TI2VAPipelineConfig,
hf_model_paths=["kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers"],
model_detectors=[_is_kandinsky6_t2va_distilled],
)
register_configs(
sampling_param_cls=Kandinsky6TI2VASamplingParams,
pipeline_config_cls=Kandinsky6TI2VAPipelineConfig,
hf_model_paths=[
"kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers",
"kandinskylab/Kandinsky-6.0-Pro-sft-5s-Diffusers",
],
model_detectors=[_is_kandinsky6_t2va],
)
__all__ = [
"KANDINSKY6_PROMPT_TEMPLATE",
"KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX",
"Kandinsky6TI2VAPipelineConfig",
"kandinsky6_clip_postprocess_text",
"kandinsky6_qwen_postprocess_text",
"kandinsky6_qwen_preprocess_text",
]
@@ -0,0 +1,76 @@
# SPDX-License-Identifier: Apache-2.0
"""Video-to-video SR without text/audio models; source audio is preserved.
TI2V supplies the API task type, but model-specific stages handle video_path."""
import re
from dataclasses import dataclass, field
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
Kandinsky6SRVAEConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.base import (
ModelTaskType,
PipelineConfig,
)
@dataclass
class Kandinsky6SRPipelineConfig(PipelineConfig):
"""Configuration for ``Kandinsky6SRPipeline``."""
task_type: ModelTaskType = ModelTaskType.TI2V
skip_input_image_preprocess: bool = True
# A missing / unexpected weight, a bad config or an unsupported flag must fail the
# load loudly: the loaders only load strictly and re-raise their errors (instead of
# falling back to a diffusers ``AutoModel``) for these components.
native_only_components: tuple[str, ...] = ("transformer", "vae", "latent_upscaler")
dit_config: Kandinsky6SRDitConfig = field(default_factory=Kandinsky6SRDitConfig)
dit_precision: str = "bf16"
vae_config: Kandinsky6SRVAEConfig = field(default_factory=Kandinsky6SRVAEConfig)
vae_precision: str = "bf16"
vae_tiling: bool = False
vae_sp: bool = False
# Text-free: no text encoders (all four per-encoder tuples must stay equally long).
text_encoder_configs: tuple = ()
text_encoder_precisions: tuple = ()
preprocess_text_funcs: tuple = ()
postprocess_text_funcs: tuple = ()
should_use_guidance: bool = False
enable_autocast: bool = False
def supports_disaggregation(self) -> bool:
# The tiled loop runs in one process over shared tensors.
return False
__all__ = ["Kandinsky6SRPipelineConfig"]
def _is_kandinsky6_sr(model_id: str) -> bool:
normalized = model_id.lower().replace("-", "").replace("_", "")
return re.search(r"kandinsky6(?:\.\d+)?(?:vsr|sr|superres)", normalized) is not None
def register():
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.registry import register_configs
register_configs(
sampling_param_cls=Kandinsky6SRSamplingParams,
pipeline_config_cls=Kandinsky6SRPipelineConfig,
hf_model_paths=[
"kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers",
"kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers",
],
model_detectors=[_is_kandinsky6_sr],
)
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
from dataclasses import dataclass
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
@dataclass
class Kandinsky6TI2VASamplingParams(SamplingParams):
"""Sampling params for Kandinsky6 TI2VA (text[+image] -> video+audio).
Defaults of the Diffusers ``Kandinsky6TI2VAPipeline`` (the undistilled
``Kandinsky-6.0-Pro-sft-5s-Diffusers`` checkpoint): 512x768 @ 24fps, 121
frames ((121 - 1) // 4 + 1 == 31 latent frames at the video VAE's
temporal_compression_ratio=4), guidance_scale=5.0, 50 inference steps.
"""
# Video parameters (Kandinsky6 TI2VA default preset)
height: int = 512
width: int = 768
num_frames: int = 121
fps: int = 24
# SamplingParams._adjust's multi-GPU frame-count rounding
# (`orig_latent_num_frames` up to a multiple of `num_gpus`) is a second,
# independent rounding rule on top of Kandinsky6LatentPreparationStage's
# own `num_frames % 4 == 1` rounding (the diffusers reference's own
# `latent_frames = (num_frames - 1) // 4 + 1` convention -- the single
# source of truth for this model). The two compose for num_gpus > 1 and
# land on a decoded frame count neither rule intended (e.g. the
# documented default num_frames=121 becomes 125/129/125/125 decoded
# frames at 2/3/4/8 GPUs instead of Diffusers' 121). Disabling it here
# leaves Kandinsky6LatentPreparationStage's rounding as the only one
# applied, regardless of num_gpus -- matching this model's precedent for
# opting out of this same mechanism when it owns frame-count alignment
# itself (see `enable_sequence_shard=True`, which disables it for the
# same reason a few lines up in SamplingParams._adjust).
adjust_frames: bool = False
# Denoising stage
guidance_scale: float = 5.0
num_inference_steps: int = 50
negative_prompt: str = (
"Static, 2D cartoon, cartoon, 2d animation, paintings, images, worst "
"quality, low quality, ugly, deformed, walking backwards"
)
@dataclass
class Kandinsky6TI2VADistilledSamplingParams(Kandinsky6TI2VASamplingParams):
"""Sampling params for the pi-Flow distilled Kandinsky6 TI2VA checkpoint
(``Kandinsky-6.0-Pro-distill-5s-Diffusers``).
Distilled for 10 steps (model card; its ``scheduler_config.json`` leaves
``nfe`` unset): its ``PiflowScheduler`` runs one DiT call per step and has
no classifier-free guidance, so any guidance_scale other than 1.0 is
rejected by the denoising stage, and the negative prompt is never encoded
either.
"""
guidance_scale: float = 1.0
num_inference_steps: int = 10
__all__ = ["Kandinsky6TI2VADistilledSamplingParams", "Kandinsky6TI2VASamplingParams"]
@@ -0,0 +1,124 @@
# SPDX-License-Identifier: Apache-2.0
"""Sampling parameters of Kandinsky 6 video super-resolution (video -> video)."""
import os
import time
from dataclasses import dataclass
from typing import Any
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
SUPPORTED_RESOLUTION_SCALES,
resolve_target_hw,
)
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
SR_RESIZE_MODES = ("fit", "exact")
@dataclass
class Kandinsky6SRSamplingParams(SamplingParams):
"""SR request options; dimensions and fps follow video_path, prompt is ignored."""
# No prompt-driven generation: keep the guidance defaults inert.
negative_prompt: str = ""
guidance_scale: float = 1.0
# 2, 4 or 2.25 (= x1.125 pixel pre-upscale, then x2 tiles).
sr_resolution_scale: float = 2.25
# DiT calls per tile (the repo-wide convention; counts network calls, not timestep grid
# points). Flow-matching bundles want 4; a pi-Flow bundle ignores this and runs its own
# ``nfe`` instead (the denoising stage warns if this is set to something else).
num_inference_steps: int = 4
# Tiles denoised together (memory / speed trade-off).
sr_tiles_batch_size: int = 1
# Minimum tile overlap of the tile grid, as a fraction of the tile size.
sr_tile_min_overlap: float = 0.20
# Optional delivery resolution: "hd", "fullhd", "2k" or "WxH" (never upscales).
sr_target_resolution: str | None = None
# "fit" keeps the aspect ratio inside the target bucket; "exact" squeezes to it.
sr_target_resize_mode: str = "fit"
# The SR DiT is text-free: a multipart request needs no ``prompt`` field.
prompt_optional = True
@classmethod
def video_request_extra_fields(cls) -> frozenset[str]:
# num_inference_steps is already in the base schema; only sr_* needs model_extra
return frozenset(
{
"sr_resolution_scale",
"sr_tiles_batch_size",
"sr_tile_min_overlap",
"sr_target_resolution",
"sr_target_resize_mode",
}
)
@classmethod
def lower_video_request_kwargs(
cls, request: Any, kwargs: dict[str, Any]
) -> dict[str, Any]:
"""Lift the ``sr_*`` extras (declared above, kept by the multipart / JSON
request layer as pydantic ``model_extra``) into the kwargs this class is
constructed from; the base class only forwards a fixed, generic field set."""
extra = getattr(request, "model_extra", None) or {}
for name in cls.video_request_extra_fields():
if name in extra and extra[name] is not None:
kwargs[name] = extra[name]
return kwargs
def _validate(self):
super()._validate()
if self.sr_resolution_scale not in SUPPORTED_RESOLUTION_SCALES:
raise ValueError(
f"sr_resolution_scale must be one of {list(SUPPORTED_RESOLUTION_SCALES)}, "
f"got {self.sr_resolution_scale!r}"
)
# num_inference_steps >= 1 is already enforced generically by the base class; the
# denoising stage additionally rejects it there with a message naming the model, and
# warns when a pi-Flow checkpoint is asked to run something other than its own nfe.
if self.sr_tiles_batch_size < 1:
raise ValueError(
f"sr_tiles_batch_size must be >= 1, got {self.sr_tiles_batch_size}"
)
if not 0.0 <= self.sr_tile_min_overlap < 1.0:
raise ValueError(
f"sr_tile_min_overlap must be in [0, 1), got {self.sr_tile_min_overlap}"
)
if self.sr_target_resize_mode not in SR_RESIZE_MODES:
raise ValueError(
f"sr_target_resize_mode must be one of {list(SR_RESIZE_MODES)}, "
f"got {self.sr_target_resize_mode!r}"
)
# "exact" mode only parses the spec: a malformed tier / WxH fails early.
resolve_target_hw(self.sr_target_resolution, (1080, 1920), "exact")
def _validate_with_pipeline_config(self, pipeline_config):
super()._validate_with_pipeline_config(pipeline_config)
if self.source_video_path() is None:
raise ValueError(
"Kandinsky6 SR needs an input video: pass --video-path (a local "
"mp4/mkv file)."
)
def source_video_path(self) -> str | None:
"""The single input video path (a one-element list is accepted)."""
path = self.video_path
if isinstance(path, list):
if len(path) != 1:
raise ValueError(
f"Kandinsky6 SR takes exactly one video, got {len(path)} paths"
)
path = path[0]
return path
def _set_output_file_name(self):
path = self.source_video_path() if self.video_path is not None else None
if self.output_file_name is None and path:
stem = os.path.splitext(os.path.basename(path))[0]
stamp = time.strftime("%Y%m%d-%H%M%S")
self.output_file_name = f"{stem}_sr{self.sr_resolution_scale:g}x_{stamp}"
super()._set_output_file_name()
__all__ = ["Kandinsky6SRSamplingParams"]
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
"""SR resolution validation shared by sampling parameters and output stages.
Keep this module independent of runtime to avoid the registry import cycle."""
from __future__ import annotations
import logging
from typing import Literal
logger = logging.getLogger(__name__)
# from-video total scales (2.25 = x1.125 pixel pre-upscale + x2 tiling).
SUPPORTED_RESOLUTION_SCALES: tuple[float, ...] = (2, 4, 2.25)
# Delivery tiers for the target resolution: (H, W) candidates per tier -- landscape 16:9,
# square, portrait 9:16; the closest-aspect entry wins.
TARGET_RESOLUTIONS: dict[str, tuple[tuple[int, int], ...]] = {
"hd": ((720, 1280), (720, 720), (1280, 720)),
"fullhd": ((1080, 1920), (1080, 1080), (1920, 1080)),
"2k": ((1440, 2560), (1440, 1440), (2560, 1440)),
}
TargetResizeMode = Literal["fit", "exact"]
# Log a warning when the SR and target aspect ratios differ by more than this (relative) --
# a mild anisotropic squeeze is fine, a big one means the wrong tier for the route.
ASPECT_MISMATCH_TOLERANCE = 0.02
def fit_within(
sr_hw: tuple[int, int], bucket_hw: tuple[int, int]
) -> tuple[int, int] | None:
"""Fit within the bucket without upscaling; preserve aspect and codec-even sizes."""
scale = min(bucket_hw[0] / sr_hw[0], bucket_hw[1] / sr_hw[1])
if scale >= 1.0:
logger.warning(
"SR result %sx%s already fits inside the %sx%s bucket -- nothing to "
"downscale, keeping it as is.",
sr_hw[1],
sr_hw[0],
bucket_hw[1],
bucket_hw[0],
)
return None
height, width = (
min(bucket, round(side * scale / 2) * 2)
for bucket, side in zip(bucket_hw, sr_hw, strict=True)
)
return height, width
def resolve_target_hw(
spec: str | None,
sr_hw: tuple[int, int],
mode: TargetResizeMode = "fit",
validate_only: bool = False,
) -> tuple[int, int] | None:
"""Resolve a tier or WxH bucket to (H, W), or None for no resize.
fit preserves aspect; exact returns the bucket dimensions. validate_only
checks syntax without needing an actual SR result."""
if mode not in ("fit", "exact"):
raise ValueError(
f"sr_target_resize_mode must be 'fit' or 'exact', got {mode!r}"
)
if spec is None or spec.lower() == "none":
return None
key = spec.lower()
if key in TARGET_RESOLUTIONS:
source_ratio = sr_hw[1] / sr_hw[0]
bucket = min(
TARGET_RESOLUTIONS[key], key=lambda hw: abs(hw[1] / hw[0] - source_ratio)
)
else:
parts = key.split("x")
if len(parts) != 2 or not all(part.strip().isdigit() for part in parts):
raise ValueError(
f"sr_target_resolution must be one of {sorted(TARGET_RESOLUTIONS)} or "
f"WxH (e.g. 1280x720), got {spec!r}"
)
width, height = (int(part) for part in parts)
bucket = (height, width)
if mode == "exact" or validate_only:
return bucket
return fit_within(sr_hw, bucket)
__all__ = [
"ASPECT_MISMATCH_TOLERANCE",
"SUPPORTED_RESOLUTION_SCALES",
"TARGET_RESOLUTIONS",
"TargetResizeMode",
"fit_within",
"resolve_target_hw",
]
@@ -298,6 +298,10 @@ class SamplingParams:
# clamped to it, so a generic 1-step warmup stays a valid request.
min_num_inference_steps: ClassVar[int] = 1
# A prompt-free pipeline (e.g. a text-free SR / video-to-video DiT) sets this to
# ``True`` so the multipart video-generation route does not require ``prompt``.
prompt_optional: ClassVar[bool] = False
height: int | None = None
width: int | None = None
fps: int = 24
@@ -1442,6 +1446,42 @@ class SamplingParams:
"frames for the generated output."
),
)
# Kandinsky6 video super-resolution (ignored by other models)
add_argument(
"--sr-resolution-scale",
type=float,
help="Kandinsky6 SR total upscale factor: 2, 4 or 2.25 (default: 2.25).",
)
add_argument(
"--sr-num-steps",
type=int,
help="Kandinsky6 SR Euler grid points (>= 2, default: 5); ignored by "
"pi-Flow checkpoints.",
)
add_argument(
"--sr-tiles-batch-size",
type=int,
help="Kandinsky6 SR: number of tiles denoised together (default: 1).",
)
add_argument(
"--sr-tile-min-overlap",
type=float,
help="Kandinsky6 SR: minimum tile overlap fraction in [0, 1) "
"(default: 0.2).",
)
add_argument(
"--sr-target-resolution",
type=str,
help="Kandinsky6 SR delivery resolution: hd, fullhd, 2k or WxH; the "
"result is only ever downscaled to it (default: keep the SR size).",
)
add_argument(
"--sr-target-resize-mode",
type=str,
choices=["fit", "exact"],
help="Kandinsky6 SR: 'fit' keeps the aspect ratio inside the target "
"resolution, 'exact' resizes to it exactly (default: fit).",
)
add_argument(
"--action-mode",
type=str,
@@ -105,15 +105,14 @@ def _patch_cache_dit_similarity():
tp_sp_group = getattr(self, "_sglang_tp_sp_group", None)
target_group = tp_sp_group or sp_group or tp_group
# Averaging over a one-rank group returns the input, so skip the
# collective rather than pay for a round trip that cannot change it.
# CFG/DP peers are independent; never fall back to the global group
if target_group is None or dist.get_world_size(target_group) == 1:
return _original_similarity(
self,
t1,
t2,
threshold=threshold,
parallelized=parallelized,
parallelized=False,
prefix=prefix,
)
@@ -436,7 +435,29 @@ def _build_custom_block_adapter(
) -> Optional[BlockAdapter]:
"""Build a manual BlockAdapter for a model absent from cache-dit's registry,
or None if the class is unknown."""
spec = _CUSTOM_BLOCK_ADAPTER_SPECS.get(transformer.__class__.__name__)
# FSDP adds a runtime subclass; retain the native model's adapter contract.
model_name = next(
(
cls.__name__
for cls in type(transformer).__mro__
if cls.__name__ == "Kandinsky6Transformer3DModel"
or cls.__name__ in _CUSTOM_BLOCK_ADAPTER_SPECS
),
None,
)
if model_name == "Kandinsky6Transformer3DModel":
# the second stream is evolving audio, not static text conditioning
return BlockAdapter(
transformer=transformer,
blocks=transformer.visual_transformer_blocks,
forward_pattern=(
ForwardPattern.Pattern_0
if transformer.config.is_multimodal
else ForwardPattern.Pattern_2
),
has_separate_cfg=has_separate_cfg,
)
spec = _CUSTOM_BLOCK_ADAPTER_SPECS.get(model_name)
if spec is None:
return None
blocks = getattr(transformer, spec.blocks_attr, None)
@@ -608,7 +629,9 @@ def enable_cache_on_transformer(
)
parallelism_config = _build_parallelism_config(sp_group, tp_group)
if parallelism_config is not None:
if parallelism_config is not None or (
dist.is_initialized() and dist.get_world_size() > 1
):
_patch_cache_dit_similarity()
_mark_transformer_parallelized(transformer, parallelism_config, sp_group, tp_group)
@@ -128,10 +128,22 @@ class IpcA2AState:
def init(self, group):
import ctypes
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.platforms.cuda import (
device_id_to_physical_device_id,
)
self.rank = dist.get_rank(group=group)
self.group = group
dev = torch.cuda.current_device()
peer_dev = _peer_cuda_device(group, self.rank, dev)
# Peer access alone also admits PCIe pairs, where this NVLink transport
# can stall in its GPU-side flag wait. Query the same pair on both ranks.
physical_devices = sorted(
device_id_to_physical_device_id(d) for d in (dev, peer_dev)
)
if not current_platform.is_full_nvlink(physical_devices):
raise _Unsupported("requires an NVLink-connected GPU pair")
try:
has_peer_access = torch.cuda.can_device_access_peer(dev, peer_dev)
except RuntimeError as e:
@@ -398,7 +398,11 @@ class DiffGenerator:
save_outputs(
output_batch.output,
requests[0].data_type,
requests[0].fps,
(
output_batch.fps
if output_batch.fps is not None
else requests[0].fps
),
requests[0].save_output,
lambda idx: output_requests[idx].output_file_path(),
audio=output_batch.audio,
@@ -526,14 +526,15 @@ async def create_video(
# Parse model-specific multipart metadata before creating request-owned
# directories or saving uploads, so malformed JSON leaves no resources.
if is_multipart:
if not prompt:
sampling_params_cls = resolve_sampling_params_cls(server_args)
if not prompt and not sampling_params_cls.prompt_optional:
raise HTTPException(status_code=400, detail="prompt is required")
raw_form = await request.form()
extra_from_form = _multipart_video_extras(
raw_form,
extra_body=extra_body,
extra_params=extra_params,
sampling_params_cls=resolve_sampling_params_cls(server_args),
sampling_params_cls=sampling_params_cls,
)
# Resolve input upload directory (may be a temp dir when saving is disabled)
@@ -613,7 +614,11 @@ async def create_video(
num_frames_val = form_value("num_frames", num_frames)
req = VideoGenerationsRequest(
prompt=prompt,
# ``prompt`` is a required str field on VideoGenerationsRequest; it is only
# ``None`` here for a prompt-optional pipeline (the gate above already
# rejected a missing prompt for every other one), so an empty string is the
# correct substitute, not a real (ignored) prompt value.
prompt=prompt or "",
enhance_prompt=form_value("enhance_prompt", enhance_prompt) or False,
input_reference=input_path,
video_path=form_value("video_path", video_input_path),
@@ -710,6 +715,8 @@ async def create_video(
detail=f"Failed to process image source: {str(e)}",
)
payload["input_reference"] = input_path
if resolve_sampling_params_cls(server_args).prompt_optional:
payload.setdefault("prompt", "")
req = VideoGenerationsRequest(**payload)
except Exception as e:
raise HTTPException(status_code=400, detail=f"Invalid request body: {e}")
@@ -1245,13 +1245,13 @@ class USPAttention(nn.Module):
if get_ring_parallel_world_size() > 1:
if (
meta_only_pad
and q.shape[0] == 1
and self.backend == AttentionBackendEnum.FA
and not self.causal
):
return self._forward_ring_tail_pad(q, k, v, attn_mask_meta)
raise NotImplementedError(
"USPAttention masked path supports ring parallelism only "
"for batch-1 tail-pad metadata on the FA backend."
"for non-causal tail-pad metadata on the FA backend."
)
if attn_mask is not None and attn_mask.dim() != 2:
raise NotImplementedError(
@@ -1505,15 +1505,15 @@ class USPAttention(nn.Module):
attn_impl=self.attn_impl,
real_seq_len=int(attn_mask_meta["pad_start"]),
ring_ws=get_ring_parallel_world_size(),
)
).reshape_as(q)
# Match the Ulysses tail path: masked query rows read as zeros. This
# rank's chunk covers global rows [rank*chunk, (rank+1)*chunk).
pad_from = (
int(attn_mask_meta["pad_start"]) - get_ring_parallel_rank() * out.shape[0]
int(attn_mask_meta["pad_start"]) - get_ring_parallel_rank() * out.shape[1]
)
if pad_from < out.shape[0]:
out[max(pad_from, 0) :].zero_()
return _usp_output_all_to_all(out.unsqueeze(0), head_dim=2)
if pad_from < out.shape[1]:
out[:, max(pad_from, 0) :].zero_()
return _usp_output_all_to_all(out, head_dim=2)
@staticmethod
def _gather_sharded_sequence(
@@ -351,7 +351,7 @@ class BaseLayerWithLoRA(nn.Module):
"""An in-place merge is about to write the base storage; if the
snapshot is a zero-copy view into it, materialize the clone now."""
if self._base_is_view:
self.cpu_weight = self.cpu_weight.clone()
self.cpu_weight = self.cpu_weight.to("cpu", copy=True)
self._base_is_view = False
@torch.no_grad()
@@ -495,11 +495,10 @@ class BaseLayerWithLoRA(nn.Module):
if self.disable_lora:
return
# Zero-copy views unmerge by subtracting B@A; do not clone the base.
if not self._base_is_view:
self._ensure_base_snapshot_owned()
if self.merged:
self.unmerge_lora_weights()
# Only layers actually merged need a snapshot of the mutable backing store.
self._ensure_base_snapshot_owned()
lora_list = self._active_lora_list()
if not lora_list:
@@ -612,67 +611,14 @@ class BaseLayerWithLoRA(nn.Module):
)
self.base_layer.weight = nn.Parameter(new_weight_data)
del old_weight
elif self._base_is_view:
self._unmerge_by_inverse()
else:
current_device = self.base_layer.weight.data.device
cpu_weight_on_device = self.cpu_weight.to(current_device, non_blocking=True)
if self.base_layer.weight.data.is_inference():
self.base_layer.weight.data = self._as_mutable_tensor(
cpu_weight_on_device
)
else:
self.base_layer.weight.data.copy_(cpu_weight_on_device)
if (
cpu_weight_on_device.data_ptr()
!= self.base_layer.weight.data.data_ptr()
):
del cpu_weight_on_device
# Update the offload backing store too, not a transient GPU placeholder.
# A copy keeps inference-tensor rebinding from aliasing the snapshot.
restored = self.cpu_weight.clone()
write_dense_weight(self.base_layer, self._as_mutable_tensor(restored))
self.merged = False
@torch.no_grad()
def _unmerge_by_inverse(self) -> None:
lora_list = self._active_lora_list()
if not lora_list:
raise ValueError(
"LoRA weights not set; cannot unmerge a zero-copy view by inverse."
)
src, current_device = self._materialized_weight_src()
data = self._as_mutable_tensor(src.to(get_local_torch_device()))
target_dtype = data.dtype
if (
self._should_merge_in_fp32(lora_list)
and data.is_floating_point()
and data.dtype != torch.float32
):
data = data.to(torch.float32)
inverted = [
(
lora_A,
lora_B,
lora_path,
-lora_strength,
lora_rank,
lora_alpha,
output_offset,
)
for (
lora_A,
lora_B,
lora_path,
lora_strength,
lora_rank,
lora_alpha,
output_offset,
) in reversed(lora_list)
]
self._merge_lora_into_data(data, inverted)
write_dense_weight(
self.base_layer,
self._as_mutable_tensor(data.to(dtype=target_dtype)),
)
@torch.no_grad()
def commit_merged_as_base(self) -> None:
"""Promote the currently merged weights to the permanent base.
@@ -823,12 +823,12 @@ def _ring_merge_attention(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Online-softmax combine of one more ring step's partial attention.
`step_out` is `[T, H, D]`; `step_lse` is FlashAttention's varlen LSE
layout `[H, T]`. Both are already self-normalized over their own KV
`step_out` is `[T, H, D]` or `[B, T, H, D]`; `step_lse` is
`[H, T]` or `[B, H, T]`. Both are already self-normalized over their own KV
chunk, so combining chunks is the standard two-term logsumexp merge
(exact up to float rounding); done in fp32 for stability.
"""
step_lse = step_lse.transpose(0, 1).unsqueeze(-1).to(torch.float32)
step_lse = step_lse.transpose(-2, -1).unsqueeze(-1).to(torch.float32)
step_out = step_out.to(torch.float32)
if out_acc is None:
return step_out, step_lse
@@ -850,7 +850,8 @@ def _ring_attention_varlen(
) -> torch.Tensor:
"""Ring-rotated varlen attention over one rank's local packed chunk.
`q, k, v` are this rank's full local ring chunk (`ring_chunk_len` rows,
`q, k, v` are `[T, H, D]` or `[B, T, H, D]`, with equal lengths per batch.
They contain this rank's full local ring chunk (`ring_chunk_len` rows,
real rows followed by however many of this chunk's rows are padding).
KV is P2P-rotated around the ring one hop per step (send this step's
buffer to the next rank, receive the following step's buffer from the
@@ -864,7 +865,7 @@ def _ring_attention_varlen(
"""
ring_pg = get_sp_group().ring_group
assert ring_pg is not None, "Ring process group is not initialized."
ring_chunk_len = q.shape[0]
ring_chunk_len = q.shape[-3]
_, ring_rank = get_ring_ctx()
# `isend`/`irecv` (unlike collectives) address peers by global rank even
@@ -913,11 +914,14 @@ def _ring_attention_varlen(
max(real_seq_len - src_rank * ring_chunk_len, 0), ring_chunk_len
)
if remote_used > 0:
step_out, step_lse = attn_impl.forward_ring_kv_chunk(
q,
kv_bufs[cur][0, :remote_used],
kv_bufs[cur][1, :remote_used],
)
key = kv_bufs[cur][0, ..., :remote_used, :, :]
value = kv_bufs[cur][1, ..., :remote_used, :, :]
if q.ndim == 3:
step_out, step_lse = attn_impl.forward_ring_kv_chunk(q, key, value)
else:
step_out, step_lse, *_ = attn_impl.forward(
q, key, value, attn_metadata=None, return_softmax_lse=True
)
out_acc, lse_acc = _ring_merge_attention(
out_acc, lse_acc, step_out, step_lse
)
@@ -146,8 +146,8 @@ class ComponentLoader(ABC):
# the list of possible name of the component in model_index.json, e.g., scheduler
component_names: list[str] = []
# diffusers or transformers
expected_library: str = ""
# diffusers or transformers (a tuple lists several accepted spellings)
expected_library: str | tuple[str, ...] = ""
_loaders_registered = False
@@ -681,7 +681,12 @@ class ComponentLoader(ABC):
if loader_cls is not None:
expected_library = loader_cls.expected_library
# Assert that the library matches what's expected for this component type
assert transformers_or_diffusers == expected_library, (
allowed = (
(expected_library,)
if isinstance(expected_library, str)
else tuple(expected_library)
)
assert transformers_or_diffusers in allowed, (
f"{loader_type} must be loaded from {expected_library}, got {transformers_or_diffusers}"
)
loader = loader_cls()
@@ -0,0 +1,80 @@
# SPDX-License-Identifier: Apache-2.0
"""Loader of the ``latent_upscaler`` component of an official Kandinsky 6 SR repo."""
import torch
from safetensors.torch import load_file as safetensors_load_file
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
ComponentLoader,
)
from sglang.multimodal_gen.runtime.loader.utils import _list_safetensors_files
from sglang.multimodal_gen.runtime.models.registry import ModelRegistry
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
get_diffusers_component_config,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
class LatentUpscalerLoader(ComponentLoader):
"""Builds the bank from ``config.json`` and loads its weights strictly.
Every failure (unparsable model config, missing or unexpected keys) surfaces as an
error: this component has no native diffusers fallback, so the base class must not
swallow the customized-load error and try ``AutoModel``.
"""
component_names = ["latent_upscaler"]
# The official ``save_pretrained`` writes "kandinsky6", the Hub repo "diffusers".
expected_library = ("diffusers", "kandinsky6")
def should_raise_customized_load_error(
self, server_args: ServerArgs, component_name: str
) -> bool:
return True
def load_customized(
self, component_model_path: str, server_args: ServerArgs, component_name: str
):
config = get_diffusers_component_config(component_path=component_model_path)
class_name = config.pop("_class_name", None)
if class_name is None:
raise ValueError(f"{component_model_path}/config.json has no _class_name")
bank_cls, _ = ModelRegistry.resolve_model_cls(class_name)
bank = self._build_bank(bank_cls, config, component_model_path)
state_dict: dict[str, torch.Tensor] = {}
for path in _list_safetensors_files(component_model_path):
state_dict.update(safetensors_load_file(path))
if not state_dict:
raise FileNotFoundError(f"no safetensors files in {component_model_path}")
bank.load_state_dict(state_dict, strict=True)
target_device = self.target_device(
server_args.should_start_component_on_cpu(component_name)
)
# The reference runs the LU in bf16 (weights and non-persistent buffers).
return (
bank.eval()
.requires_grad_(False)
.to(device=target_device, dtype=torch.bfloat16)
)
@staticmethod
def _build_bank(bank_cls, config: dict, component_model_path: str):
for key in ("models", "scaling_factor"):
if key not in config:
raise ValueError(f"{component_model_path}/config.json lacks {key!r}")
try:
return bank_cls(
models=config["models"],
scaling_factor=config["scaling_factor"],
scales=config.get("scales", (2, 4)),
)
except (ValueError, KeyError, TypeError) as error:
raise ValueError(
f"invalid latent upscaler config in {component_model_path}: "
f"{type(error).__name__}: {error}"
) from error
@@ -1435,6 +1435,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
output=result.output,
audio=getattr(result, "audio", None),
audio_sample_rate=getattr(result, "audio_sample_rate", None),
fps=getattr(result, "fps", None),
metrics=result.metrics,
usage=getattr(result, "usage", None),
trajectory_timesteps=getattr(result, "trajectory_timesteps", None),
@@ -1450,6 +1451,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
) -> OutputBatch:
"""Merge per-output batches produced by grouped execution."""
merged = OutputBatch()
merged.fps = output_batches[0].fps
parts = _ExpandedOutputParts()
for output_batch in output_batches:
@@ -53,6 +53,7 @@ VAE_COMPONENT_NAMES = frozenset(
"audio_vae",
"vocoder",
"spatial_upsampler",
"latent_upscaler",
"condition_image_encoder",
"delight_vae",
"diffusion_decoder",
@@ -1122,6 +1122,7 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
output_batch.audio, start, end, total_items
),
audio_sample_rate=output_batch.audio_sample_rate,
fps=output_batch.fps,
action_pred=self._slice_batched_value(
output_batch.action_pred, start, end, total_items
),
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,430 @@
# SPDX-License-Identifier: Apache-2.0
"""Text-free Kandinsky 6 SR DiT, reusing the joint model's TP/SP building blocks.
Latents are [B, T, H, W, C]. Residuals, norms and modulation remain fp32;
linear inputs use the parameter dtype. A pi-Flow DX head emits n_grid velocities
per latent channel. VAE, upscaler and tile scheduling remain replicated."""
from collections.abc import Iterable, Iterator
from typing import Any
import torch
import torch.nn as nn
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
ATTRIBUTE_OVERRIDE_WHITELIST,
Kandinsky6SRArchConfig,
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_ring_ctx,
get_ulysses_ctx,
)
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import (
gather_seq,
shard_like,
shard_seq,
tail_attn_meta,
)
from sglang.multimodal_gen.runtime.layers.layernorm import LayerNormScaleShift
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
QuantizationConfig,
)
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
Kandinsky6Attention,
Kandinsky6FeedForward,
Kandinsky6Modulation,
Kandinsky6OutLayer,
Kandinsky6RoPE3D,
Kandinsky6TimeEmbeddings,
Kandinsky6VisualEmbeddings,
_build_rotary_freqs,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.srt.utils import add_prefix
logger = init_logger(__name__)
_ARCH_CONFIG_DEFAULTS = Kandinsky6SRDitConfig().arch_config
_BLOCK_CONTAINERS = ("visual_transformer_blocks",)
_TIME_EMBEDDING_NAMES = ("time_embeddings", "motion_embeddings", "lq_noise_embeddings")
def _is_sr_transformer_block(name: str, module: object) -> bool:
return is_module_list_entry_in(name, _BLOCK_CONTAINERS)
def _modulate(
norm: LayerNormScaleShift, x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor
) -> torch.Tensor:
"""fp32 ``LN(x) * (scale + 1) + shift`` (no cast back: the caller picks the dtype)."""
return norm(x.float(), shift=shift, scale=scale)
class Kandinsky6SRRoPE3D(Kandinsky6RoPE3D):
"""Keep fp32 RoPE angle tables across residency-manager dtype casts."""
def _apply(self, fn, recurse=True):
device = fn(torch.zeros(1)).device
for name, buffer in self._buffers.items():
if buffer is not None:
self._buffers[name] = buffer.to(device=device)
return self
class Kandinsky6SRDecoderBlock(nn.Module):
"""Modulated self-attention and FFN with fp32 residuals and parameter-dtype linears."""
def __init__(
self,
model_dim: int,
time_dim: int,
ff_dim: int,
head_dim: int,
supported_attention_backends: set[AttentionBackendEnum] | None = None,
prefix: str = "",
quant_config: QuantizationConfig | None = None,
):
super().__init__()
self.visual_modulation = Kandinsky6Modulation(time_dim, model_dim, 6)
self.self_attention_norm = LayerNormScaleShift(
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
)
self.self_attention = Kandinsky6Attention(
model_dim,
head_dim,
supported_attention_backends=supported_attention_backends,
prefix=add_prefix("self_attention", prefix),
quant_config=quant_config,
)
self.feed_forward_norm = LayerNormScaleShift(
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
)
self.feed_forward = Kandinsky6FeedForward(
model_dim,
ff_dim,
prefix=add_prefix("feed_forward", prefix),
quant_config=quant_config,
)
def forward(
self,
visual_embed: torch.Tensor,
time_embed: torch.Tensor,
rope: torch.Tensor,
compute_dtype: torch.dtype,
attn_mask_meta: dict | None = None,
) -> torch.Tensor:
modulation = self.visual_modulation(time_embed).unsqueeze(dim=1)
self_attn_params, ff_params = torch.chunk(modulation, 2, dim=-1)
shift, scale, gate = torch.chunk(self_attn_params, 3, dim=-1)
out = _modulate(self.self_attention_norm, visual_embed, shift, scale)
out = self.self_attention(
out.to(compute_dtype), rotary_emb=rope, attn_mask_meta=attn_mask_meta
)
visual_embed = visual_embed.float() + gate.float() * out.float()
shift, scale, gate = torch.chunk(ff_params, 3, dim=-1)
out = _modulate(self.feed_forward_norm, visual_embed, shift, scale)
out = self.feed_forward(out.to(compute_dtype))
return visual_embed + gate.float() * out.float()
class Kandinsky6SROutLayer(Kandinsky6OutLayer):
"""K6 ``OutLayer`` for an fp32 residual stream (norm in fp32, linear in weight dtype)."""
def forward(
self, visual_embed: torch.Tensor, time_embed: torch.Tensor
) -> torch.Tensor:
shift, scale = torch.chunk(
self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1
)
x = (
self.norm(visual_embed.float()) * (scale.float()[:, None, None] + 1.0)
+ shift.float()[:, None, None]
)
x, _ = self.out_layer(x.to(self.out_layer.weight.dtype))
batch_size, duration, height, width, _ = x.shape
pt, ph, pw = self.patch_size
return (
x.view(batch_size, duration, height, width, -1, pt, ph, pw)
.permute(0, 1, 5, 2, 6, 3, 7, 4)
.flatten(1, 2)
.flatten(2, 3)
.flatten(3, 4)
)
class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
"""Text-free Kandinsky 6 SR DiT (optionally with a DX / pi-Flow head)."""
_fsdp_shard_conditions = [_is_sr_transformer_block]
_compile_conditions = [_is_sr_transformer_block]
param_names_mapping = _ARCH_CONFIG_DEFAULTS.param_names_mapping
reverse_param_names_mapping: dict = {}
lora_param_names_mapping: dict = {}
# Dense attention only: the sparse (NABLA) path of the reference is not ported.
_supported_attention_backends = {
AttentionBackendEnum.FA,
AttentionBackendEnum.TORCH_SDPA,
}
@staticmethod
def _validate_tp_config(*, arch: Kandinsky6SRArchConfig, tp_size: int) -> None:
if tp_size <= 0:
raise ValueError("Kandinsky6SR TP size must be positive.")
for name, value in (
("num_attention_heads", arch.num_attention_heads),
("ff_dim", arch.ff_dim),
):
if value % tp_size:
raise ValueError(
f"Kandinsky6SR {name}={value} must be divisible by TP size {tp_size}."
)
@staticmethod
def _validate_sequence_parallel_config(
*, arch: Kandinsky6SRArchConfig, tp_size: int, ulysses_size: int, ring_size: int
) -> None:
if ulysses_size <= 0:
raise ValueError("Kandinsky6SR Ulysses size must be positive.")
if ring_size <= 0:
raise ValueError("Kandinsky6SR ring size must be positive.")
if ulysses_size == 1 and ring_size == 1:
return
# Ring Attention rotates whole K/V shards between ranks rather than
# splitting heads, so only Ulysses constrains head divisibility.
local_heads = arch.num_attention_heads // tp_size
if local_heads % ulysses_size:
raise ValueError(
f"Kandinsky6SR TP-local attention heads {local_heads} must be "
f"divisible by Ulysses size {ulysses_size} (total heads="
f"{arch.num_attention_heads}, TP={tp_size})."
)
def __init__(
self,
config: Kandinsky6SRDitConfig,
hf_config: dict[str, Any],
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__(config=config, hf_config=hf_config)
arch: Kandinsky6SRArchConfig = self.config
self.quant_config = quant_config
head_dim = sum(arch.axes_dims)
tp_size = get_tp_world_size()
ulysses_size, _ = get_ulysses_ctx()
ring_size, _ = get_ring_ctx()
self._validate_tp_config(arch=arch, tp_size=tp_size)
self._validate_sequence_parallel_config(
arch=arch, tp_size=tp_size, ulysses_size=ulysses_size, ring_size=ring_size
)
self.in_visual_dim = arch.in_visual_dim
self.base_out_visual_dim = arch.base_out_visual_dim
self.n_grid = arch.n_grid
self.model_dim = arch.model_dim
self.patch_size = arch.patch_size
self.use_motion_score = arch.use_motion_score
self.use_lq_noise_cond = arch.use_lq_noise_cond
# The input width is fixed by the *trained* configuration ...
self.visual_embed_dim = (
2 * arch.in_visual_dim + 1
if arch.trained_wide_input
else arch.in_visual_dim
)
self._build_layers(arch, head_dim=head_dim, quant_config=quant_config)
self.hidden_size = arch.hidden_size
self.num_attention_heads = arch.num_attention_heads
self.num_channels_latents = arch.num_channels_latents
self.layer_names = list(_BLOCK_CONTAINERS)
# ... while inference behaviour comes from the post-load overrides.
self._apply_attribute_overrides(arch)
self._require_all_checkpoint_keys()
self.__post_init__()
def _build_layers(
self,
arch: Kandinsky6SRArchConfig,
*,
head_dim: int,
quant_config: QuantizationConfig | None,
) -> None:
self.time_embeddings = Kandinsky6TimeEmbeddings(arch.model_dim, arch.time_dim)
if arch.use_motion_score:
self.motion_embeddings = Kandinsky6TimeEmbeddings(
arch.model_dim, arch.time_dim
)
if arch.use_lq_noise_cond:
# Present in the checkpoint; never used at inference (kept for loading).
self.lq_noise_embeddings = Kandinsky6TimeEmbeddings(
arch.model_dim, arch.time_dim
)
# Constant that replaces the pooled empty-caption embedding of the text DiT.
self.pooled_bias = nn.Parameter(torch.zeros(arch.time_dim))
self.visual_embeddings = Kandinsky6VisualEmbeddings(
self.visual_embed_dim,
arch.model_dim,
arch.patch_size,
prefix=add_prefix("visual_embeddings", self.prefix),
)
self.visual_rope_embeddings = Kandinsky6SRRoPE3D(arch.axes_dims)
self.visual_transformer_blocks = nn.ModuleList(
[
Kandinsky6SRDecoderBlock(
arch.model_dim,
arch.time_dim,
arch.ff_dim,
head_dim,
self._supported_attention_backends,
prefix=add_prefix(f"visual_transformer_blocks.{i}", self.prefix),
quant_config=quant_config,
)
for i in range(arch.num_visual_blocks)
]
)
self.out_layer = Kandinsky6SROutLayer(
arch.model_dim,
arch.time_dim,
arch.head_width,
arch.patch_size,
)
def _apply_attribute_overrides(self, arch: Kandinsky6SRArchConfig) -> None:
self.instruct_type = arch.instruct_type
self.visual_cond = arch.visual_cond
self.attention_params = arch.attention_params
for name in ATTRIBUTE_OVERRIDE_WHITELIST:
if name in arch.attribute_overrides:
setattr(self, name, arch.attribute_overrides[name])
self._check_input_width_matches_overrides()
sparse_type = arch.requested_sparse_attention()
if sparse_type is not None:
logger.warning(
"Kandinsky6SR: attention_params request %r sparse attention; this port "
"runs dense attention, so outputs differ from the sparse reference.",
sparse_type,
)
def _check_input_width_matches_overrides(self) -> None:
run_wide = self.visual_cond or self.instruct_type in (
"channel",
"hybrid",
"hybrid_anchor",
)
run_width = 2 * self.in_visual_dim + 1 if run_wide else self.in_visual_dim
if run_width != self.visual_embed_dim:
raise ValueError(
f"Kandinsky6SR: the trained input layer takes {self.visual_embed_dim} "
f"channels but instruct_type={self.instruct_type!r} with "
f"visual_cond={self.visual_cond} feeds {run_width}; set "
"attribute_overrides.visual_cond=true to run a wide-input checkpoint "
"under instruct_type='noise'."
)
def _require_all_checkpoint_keys(self) -> None:
"""A missing tensor must fail loudly instead of being zero-filled by the loader."""
for param in self.parameters():
param.missing_param_init = "error"
def preprocess_loaded_state_dict(
self, weight_iterator: Iterable[tuple[str, torch.Tensor]]
) -> Iterator[tuple[str, torch.Tensor]]:
"""Pass tensors through, then raise if the checkpoint held unexpected keys."""
map_name = get_param_names_mapping(self.param_names_mapping)
expected = set(self.state_dict())
unexpected: list[str] = []
for name, tensor in weight_iterator:
if map_name(name)[0] not in expected:
unexpected.append(name)
yield name, tensor
if unexpected:
shown = ", ".join(unexpected[:10])
raise ValueError(
f"Kandinsky6SR checkpoint has {len(unexpected)} unexpected keys "
f"(first: {shown}); the transformer config does not match the weights."
)
def forward(
self,
hidden_states: torch.Tensor,
timestep: torch.Tensor,
visual_rope_pos: (
tuple[torch.Tensor, torch.Tensor, torch.Tensor] | list[torch.Tensor]
),
scale_factor: tuple[float, float, float] = (1.0, 1.0, 1.0),
motion_score: torch.Tensor | None = None,
) -> torch.Tensor:
"""Predict [B, T, H, W, head_width] velocities from channel-last latents.
timesteps are scaled by 1000. DX channels are ordered grid-major;
the scheduler reshapes them into n_grid velocity predictions."""
self._check_latent_shape(hidden_states)
compute_dtype = self.visual_embeddings.in_layer.weight.dtype
time_embed = self.time_embeddings(timestep)
if motion_score is not None and self.use_motion_score:
time_embed = time_embed + self.motion_embeddings(motion_score)
time_embed = time_embed + self.pooled_bias
visual_embed = self.visual_embeddings(hidden_states.to(compute_dtype))
visual_shape = visual_embed.shape[:-1]
visual_rope = self.visual_rope_embeddings(
visual_shape, visual_rope_pos, scale_factor
)
visual_embed = visual_embed.flatten(1, 3)
visual_rope = visual_rope.flatten(1, 3)
visual_embed, shard = shard_seq(visual_embed)
visual_rope = shard_like(visual_rope, shard, pad_mode="repeat_last")
attn_meta = tail_attn_meta(shard, visual_embed.shape[0], visual_embed.device)
for block in self.visual_transformer_blocks:
visual_embed = block(
visual_embed,
time_embed,
visual_rope,
compute_dtype,
attn_mask_meta=attn_meta,
)
visual_embed = gather_seq(visual_embed, shard.orig_len)
return self.out_layer(visual_embed.reshape(*visual_shape, -1), time_embed)
def _check_latent_shape(self, hidden_states: torch.Tensor) -> None:
if hidden_states.ndim != 5 or hidden_states.shape[-1] != self.visual_embed_dim:
raise ValueError(
"Kandinsky6SR expects a [B, T, H, W, "
f"{self.visual_embed_dim}] latent, got {tuple(hidden_states.shape)}"
)
height, width = hidden_states.shape[2:4]
if height % self.patch_size[1] or width % self.patch_size[2]:
raise ValueError(
f"latent {height}x{width} is not divisible by patch {self.patch_size}"
)
def post_load_weights(self) -> None:
"""Rebuild RoPE / time-embedding tables that a meta-device init left on meta."""
device = next(self.parameters()).device
rope = self.visual_rope_embeddings
if any(buf.is_meta for buf in rope.buffers()):
self.visual_rope_embeddings = Kandinsky6SRRoPE3D(
rope.axes_dims, rope.max_pos, rope.max_period
).to(device)
for name in _TIME_EMBEDDING_NAMES:
embeddings = self._modules.get(name)
if embeddings is not None and embeddings.freqs.is_meta:
embeddings.freqs = _build_rotary_freqs(
embeddings.model_dim // 2, embeddings.max_period
).to(device=device)
EntryClass = Kandinsky6SRTransformer3DModel
@@ -183,6 +183,9 @@ class CLIPTextModel(TextEncoder):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
# standalone CLIP text-transformer checkpoints omit the model wrapper
if name.startswith(("embeddings.", "encoder.", "final_layer_norm.")):
name = f"text_model.{name}"
name = _srt_clip_param_name(name)
# Handle q_proj, k_proj, v_proj -> qkv_proj mapping
for param_name, weight_name, shard_id in stacked_params_mapping:
@@ -0,0 +1,314 @@
# Copyright 2025 The Kandinsky Team and The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Diffusers scheduler for distilled Kandinsky 6 PiFlow checkpoints."""
from __future__ import annotations
import torch
from diffusers.configuration_utils import register_to_config
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
FlowMatchEulerDiscreteSchedulerOutput,
)
class DXPolicy:
def __init__( # noqa: PLR0913
self,
denoising_output: torch.Tensor,
x_t_src: torch.Tensor,
sigma_t_src: torch.Tensor,
segment_size: float | torch.Tensor = 1.0,
shift: float = 1.0,
mode: str = "grid",
eps: float = 1e-4,
) -> None:
self.x_t_src = x_t_src
self.ndim = x_t_src.dim()
self.shift = shift
self.eps = eps
if mode not in ("grid", "polynomial"):
raise ValueError(f"Unknown mode: {mode}")
self.mode = mode
self.sigma_t_src = sigma_t_src.reshape(
*sigma_t_src.size(), *((self.ndim - sigma_t_src.dim()) * [1])
)
self.raw_t_src = self._unwarp_t(self.sigma_t_src)
segment = segment_size
if isinstance(segment, torch.Tensor) and segment.dim() < self.raw_t_src.dim():
segment = segment.reshape(
*segment.size(), *((self.raw_t_src.dim() - segment.dim()) * [1])
)
self.raw_t_dst = (self.raw_t_src - segment).clamp(min=0)
self.segment_size = (self.raw_t_src - self.raw_t_dst).clamp(min=eps)
self.denoising_output_x_0 = self._u_to_x_0(
denoising_output, self.x_t_src, self.sigma_t_src
)
@staticmethod
def _interpolate(x: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
n = x.size(1)
if n < 2: # noqa: PLR2004
return x.squeeze(1)
t = t.clamp(min=0, max=1) * (n - 1)
t0 = t.floor().to(torch.long).clamp(min=0, max=n - 2)
t1 = t0 + 1
indices = torch.stack([t0, t1], dim=1)
values = torch.gather(x, dim=1, index=indices.expand(-1, -1, *x.shape[2:]))
return (t1 - t) * values[:, 0] + (t - t0) * values[:, 1]
def _unwarp_t(self, sigma_t: torch.Tensor) -> torch.Tensor:
return sigma_t / (self.shift + (1 - self.shift) * sigma_t)
@staticmethod
def _u_to_x_0(
denoising_output: torch.Tensor,
x_t: torch.Tensor,
sigma_t: torch.Tensor,
) -> torch.Tensor:
return x_t.unsqueeze(1) - sigma_t.unsqueeze(1) * denoising_output
def pi(self, x_t: torch.Tensor, sigma_t: torch.Tensor) -> torch.Tensor:
sigma_t = sigma_t.reshape(*sigma_t.size(), *((self.ndim - sigma_t.dim()) * [1]))
raw_t = self._unwarp_t(sigma_t)
if self.mode == "grid":
x_0 = self._interpolate(
self.denoising_output_x_0,
(raw_t - self.raw_t_dst) / self.segment_size,
)
else:
p_order = self.denoising_output_x_0.size(1)
diff_t = self.raw_t_src - raw_t
basis = torch.stack([diff_t**i for i in range(p_order)], dim=1)
x_0 = torch.sum(basis * self.denoising_output_x_0, dim=1)
return (x_t - x_0) / sigma_t.clamp(min=self.eps)
def shift_timesteps(t: torch.Tensor, shift: float) -> torch.Tensor:
return shift * t / (1 + (shift - 1) * t)
def policy_rollout_fm( # noqa: PLR0913
x_t_start: torch.Tensor,
sigma_t_start: torch.Tensor,
raw_t_start: torch.Tensor,
raw_t_end: torch.Tensor,
total_substeps: int,
policy: DXPolicy,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
num_batches = x_t_start.size(0)
ndim = x_t_start.dim()
shape = (num_batches, *((ndim - 1) * [1]))
raw_t_start = raw_t_start.reshape(shape)
raw_t_end = raw_t_end.reshape(shape)
sigma_t = sigma_t_start.reshape(shape)
delta_raw_t = raw_t_start - raw_t_end
num_substeps = (delta_raw_t * total_substeps).round().to(torch.long).clamp(min=1)
substep_size = delta_raw_t / num_substeps
max_num_substeps = num_substeps.max()
raw_t = raw_t_start
x_t = x_t_start
for substep_id in range(max_num_substeps.item()):
velocity = policy.pi(x_t, sigma_t)
raw_t_minus = (raw_t - substep_size).clamp(min=0)
sigma_t_minus = shift_timesteps(raw_t_minus, policy.shift)
x_t_minus = x_t + velocity * (sigma_t_minus - sigma_t)
active_mask = num_substeps > substep_id
x_t = torch.where(active_mask, x_t_minus, x_t)
sigma_t = torch.where(active_mask, sigma_t_minus, sigma_t)
raw_t = torch.where(active_mask, raw_t_minus, raw_t)
return x_t, sigma_t, sigma_t.flatten() * 1_000
class PiflowScheduler(FlowMatchEulerDiscreteScheduler):
is_piflow = True
@register_to_config
def __init__(
self,
num_train_timesteps: int = 1000,
shift: float = 5.0,
n_grid: int = 10,
nfe: int | None = None,
eps: float = 1e-6,
final_step_size_scale: float = 0.5,
num_policy_substeps: int = 128,
) -> None:
if n_grid < 2:
raise ValueError(f"PiflowScheduler requires n_grid >= 2, got {n_grid}")
if eps <= 0:
raise ValueError(f"PiflowScheduler requires eps > 0, got {eps}")
if not 0 < final_step_size_scale <= 1:
raise ValueError("PiflowScheduler requires 0 < final_step_size_scale <= 1")
if num_policy_substeps < 1:
raise ValueError("PiflowScheduler requires num_policy_substeps >= 1")
super().__init__(
num_train_timesteps=num_train_timesteps,
shift=shift,
)
self.n_grid = int(n_grid)
self.eps = float(eps)
self.final_step_size_scale = float(final_step_size_scale)
self.num_policy_substeps = int(num_policy_substeps)
self._piflow_raw_timesteps = torch.empty(0)
def set_timesteps(
self,
num_inference_steps: int | None = None,
device: str | torch.device | None = None,
sigmas: list[float] | None = None,
mu: float | None = None,
timesteps: list[float] | None = None,
) -> None:
if sigmas is not None or mu is not None or timesteps is not None:
raise ValueError(
"PiflowScheduler only supports its configured distilled timestep schedule"
)
if num_inference_steps is None or num_inference_steps < 1:
raise ValueError(
f"num_inference_steps must be positive, got {num_inference_steps}"
)
one_minus_final = 1.0 - self.final_step_size_scale
segment = 1.0 / (num_inference_steps - one_minus_final)
raw = (
1.0
- torch.arange(num_inference_steps, dtype=torch.float32, device=device)
* segment
)
sigmas = shift_timesteps(raw, float(self.config.shift))
self.num_inference_steps = int(num_inference_steps)
self._piflow_raw_timesteps = raw
self.timesteps = sigmas * self.config.num_train_timesteps
self.sigmas = torch.cat([sigmas, sigmas.new_zeros(1)])
self._step_index = None
self._begin_index = None
def _to_grid(
self, model_output: torch.Tensor, sample: torch.Tensor
) -> torch.Tensor:
if (
model_output.ndim != sample.ndim
or model_output.shape[:-1] != sample.shape[:-1]
):
raise ValueError(
"Piflow model output must match sample shape except for the output channels: "
f"got {tuple(model_output.shape)} for sample {tuple(sample.shape)}"
)
if model_output.shape[-1] % self.n_grid != 0:
raise ValueError(
f"Piflow model output channels {model_output.shape[-1]} are not divisible by n_grid={self.n_grid}"
)
output_dim = model_output.shape[-1] // self.n_grid
if output_dim != sample.shape[-1]:
raise ValueError(
"Piflow model output channels do not match the sample: "
f"expected {sample.shape[-1] * self.n_grid}, got {model_output.shape[-1]}"
)
return model_output.reshape(
*model_output.shape[:-1], self.n_grid, output_dim
).movedim(-2, 1)
def _policy_step(
self, model_output: torch.Tensor, sample: torch.Tensor, step_index: int
) -> torch.Tensor:
model_output = self._to_grid(model_output, sample)
raw_src = self._piflow_raw_timesteps[step_index].to(device=sample.device)
raw_dst = (
self._piflow_raw_timesteps[step_index + 1]
if step_index + 1 < self._piflow_raw_timesteps.numel()
else self._piflow_raw_timesteps.new_full((), self.eps)
).to(device=sample.device)
sigma_src = self.sigmas[step_index].to(device=sample.device)
token_shape = (sample.shape[0], *((sample.ndim - 1) * [1]))
sigma = sigma_src.expand(sample.shape[0]).reshape(token_shape)
segment = (raw_src - raw_dst).expand(sample.shape[0])
policy = DXPolicy(
model_output,
sample,
sigma,
segment,
shift=float(self.config.shift),
mode="grid",
eps=self.eps,
)
updated, _, _ = policy_rollout_fm(
sample,
sigma,
raw_src.expand(sample.shape[0]),
raw_dst.expand(sample.shape[0]),
self.num_policy_substeps,
policy,
)
return updated.to(dtype=sample.dtype)
def _step_index_for(self, timestep: torch.Tensor | float) -> int:
if self.step_index is None:
if self.begin_index is not None:
self._step_index = self.begin_index
else:
schedule_timesteps = self.timesteps
if not isinstance(schedule_timesteps, torch.Tensor):
schedule_timesteps = torch.as_tensor(
schedule_timesteps, dtype=torch.float32
)
timestep = torch.as_tensor(
timestep,
device=schedule_timesteps.device,
dtype=schedule_timesteps.dtype,
)
indices = torch.nonzero(schedule_timesteps == timestep).flatten()
if not indices.numel():
raise ValueError(
f"timestep {timestep.item()} is not in the Piflow schedule"
)
position = 1 if indices.numel() > 1 else 0
self._step_index = int(indices[position].item())
if self.step_index is None or self.step_index >= self.num_inference_steps:
raise RuntimeError(
"PiflowScheduler.step called after the schedule was exhausted"
)
return int(self.step_index)
def step(
self,
model_output: torch.FloatTensor,
timestep: float | torch.FloatTensor,
sample: torch.FloatTensor,
return_dict: bool = True,
) -> FlowMatchEulerDiscreteSchedulerOutput | tuple:
if isinstance(timestep, int) or isinstance(
timestep, (torch.IntTensor, torch.LongTensor)
):
raise ValueError(
"Passing integer indices as timesteps to PiflowScheduler.step() is not supported; "
"pass a value from scheduler.timesteps instead"
)
step_index = self._step_index_for(timestep)
# PiFlow's policy rollout performs its update in float32. Keep that
# precision across outer steps, matching the native sampler.
updated = self._policy_step(model_output, sample.to(torch.float32), step_index)
self._step_index += 1
if return_dict:
return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=updated)
return (updated,)
EntryClass = PiflowScheduler
@@ -0,0 +1,336 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky 6 SR x2/x4 upsampler bank for scaled [B, C, T, H, W] KVAE latents.
Norms are modulated by the source latent; temporal conv padding repeats edge frames.
_models entries follow checkpoint scale order and retain checkpoint module names."""
from __future__ import annotations
from collections.abc import Mapping, Sequence
from typing import Any
import torch
import torch.nn.functional as F
from torch import Tensor, nn
from sglang.multimodal_gen.configs.models.upsamplers.kandinsky6_sr import (
Kandinsky6SRLatentUpscalerEntryConfig,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
class ReplicateTimeConv3d(nn.Conv3d):
"""Conv3d with explicit edge-repeated temporal padding and zero spatial padding."""
def __init__(self, in_channels: int, out_channels: int, kernel_size: int) -> None:
pad = kernel_size // 2
super().__init__(in_channels, out_channels, kernel_size, padding=(0, pad, pad))
self.temporal_pad = pad
def forward(self, x: Tensor) -> Tensor:
x = F.pad(
x, (0, 0, 0, 0, self.temporal_pad, self.temporal_pad), mode="replicate"
)
return super().forward(x)
class RMSNorm(nn.Module):
"""Channel RMS norm of ``[B, C, T, H, W]`` features with a learnable per-channel gain."""
def __init__(self, dim: int) -> None:
super().__init__()
self.scale = dim**0.5
self.gamma = nn.Parameter(torch.ones(dim, 1, 1, 1))
def forward(self, x: Tensor) -> Tensor:
# Upcast to float32 around the normalize (matches the Diffusers reference's
# Kandinsky6SRLatentUpscalerRMSNorm): without it, bf16 activations drift from the
# reference across the many ResidualBlock/X2Branch calls that route through this norm.
normalized = F.normalize(x.float(), dim=1).to(x.dtype)
return normalized * self.scale * self.gamma
class ModulatedRMSNorm(nn.Module):
"""``RMSNorm(x) * conv_y(zq) + conv_b(zq)`` with 1x1x1 convs of the conditioning latent ``zq``."""
def __init__(self, dim: int, zq_dim: int) -> None:
super().__init__()
self.norm = RMSNorm(dim)
self.conv_y = nn.Conv3d(zq_dim, dim, kernel_size=1)
self.conv_b = nn.Conv3d(zq_dim, dim, kernel_size=1)
def forward(self, x: Tensor, zq: Tensor) -> Tensor:
if zq.shape[2:] != x.shape[2:]:
zq = F.interpolate(zq, size=x.shape[2:], mode="nearest")
return self.norm(x) * self.conv_y(zq) + self.conv_b(zq)
class ResidualBlock(nn.Module):
"""Pre-activation block ``norm1 -> SiLU -> conv1 -> norm2 -> SiLU -> conv2`` plus a (1x1x1 if
narrowing) skip."""
def __init__(
self, in_channels: int, out_channels: int, mid_channels: int, zq_dim: int
) -> None:
super().__init__()
self.norm1 = ModulatedRMSNorm(in_channels, zq_dim)
self.conv1 = ReplicateTimeConv3d(in_channels, mid_channels, kernel_size=3)
self.norm2 = ModulatedRMSNorm(mid_channels, zq_dim)
self.conv2 = ReplicateTimeConv3d(mid_channels, out_channels, kernel_size=3)
self.shortcut: nn.Module = (
nn.Identity()
if in_channels == out_channels
else ReplicateTimeConv3d(in_channels, out_channels, kernel_size=1)
)
def forward(self, x: Tensor, zq: Tensor) -> Tensor:
h = F.silu(self.norm1(x, zq))
h = self.conv1(h)
h = F.silu(self.norm2(h, zq))
h = self.conv2(h)
return self.shortcut(x) + h
def nearest_2x(x: Tensor) -> Tensor:
"""Nearest-neighbour 2x resize of H and W of ``[B, C, T, H, W]`` (T folded into the batch)."""
b, c, t, h, w = x.shape
x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w)
x = F.interpolate(x, scale_factor=2, mode="nearest")
return x.reshape(b, t, c, 2 * h, 2 * w).permute(0, 2, 1, 3, 4)
class PXSUpsample(nn.Module):
"""2x spatial upsample ``linear(up + spatial_conv(up))`` with ``up = nearest_2x(x)``."""
def __init__(self, channels: int) -> None:
super().__init__()
self.spatial_conv = nn.Conv3d(
channels, channels, kernel_size=(1, 3, 3), padding=(0, 1, 1)
)
self.linear = nn.Conv3d(channels, channels, kernel_size=1)
def forward(self, x: Tensor) -> Tensor:
up = nearest_2x(x)
return self.linear(up + self.spatial_conv(up))
class X2Finisher(nn.Module):
"""The ``PXSUpsample`` conv pair applied on the unchanged grid (no resize)."""
def __init__(self, channels: int) -> None:
super().__init__()
self.spatial_conv = nn.Conv3d(
channels, channels, kernel_size=(1, 3, 3), padding=(0, 1, 1)
)
self.linear = nn.Conv3d(channels, channels, kernel_size=1)
def forward(self, x: Tensor) -> Tensor:
return self.linear(x + self.spatial_conv(x))
def _stem(in_channels: int, out_channels: int) -> nn.Sequential:
return nn.Sequential(ReplicateTimeConv3d(in_channels, out_channels, kernel_size=3))
def _mid_output_head(channels: int, out_channels: int) -> nn.Sequential:
"""Unused training head whose checkpoint layout remains positional."""
return nn.Sequential(
RMSNorm(channels),
nn.SiLU(),
ReplicateTimeConv3d(channels, out_channels, kernel_size=3),
)
class Kandinsky6SRLatentUpscalerOutputHead(nn.Module):
"""Named ``norm -> activation -> conv`` projection used by current checkpoints."""
def __init__(self, channels: int, out_channels: int, zq_dim: int) -> None:
super().__init__()
self.norm = ModulatedRMSNorm(channels, zq_dim)
self.activation = nn.SiLU()
self.conv = ReplicateTimeConv3d(channels, out_channels, kernel_size=3)
def forward(self, x: Tensor, zq: Tensor) -> Tensor:
return self.conv(self.activation(self.norm(x, zq)))
def _residual_stack(
count: int, in_channels: int, channels: int, expand_ratio: int, zq_dim: int
) -> nn.Sequential:
"""``count`` blocks at width ``channels``; the first one narrows from ``in_channels``."""
return nn.Sequential(
*(
ResidualBlock(
in_channels if index == 0 else channels,
channels,
channels * expand_ratio,
zq_dim,
)
for index in range(count)
)
)
def _run_blocks(blocks: nn.Sequential, x: Tensor, zq: Tensor) -> Tensor:
for block in blocks:
x = block(x, zq)
return x
def _second_stage(
upsample: nn.Module,
blocks: nn.Sequential,
head: Kandinsky6SRLatentUpscalerOutputHead,
x: Tensor,
zq: Tensor,
) -> Tensor:
"""``upsample -> blocks -> head``, with the head's first (norm) layer conditioned on ``zq``."""
x = _run_blocks(blocks, upsample(x), zq)
return head(x, zq)
class X2Branch(nn.Module):
"""Weights exclusive to the x2 path."""
def __init__(self, config: Kandinsky6SRLatentUpscalerEntryConfig) -> None:
super().__init__()
c = config.in_channels
w1, w2, w3 = config.stage_channels
er = config.expand_ratio
self.adapter = _residual_stack(config.x2_adapter_blocks, w1, w1, er, c)
self.finisher = X2Finisher(w1)
self.mid_blocks = _residual_stack(config.num_mid_blocks, w1, w2, er, c)
self.upsample = PXSUpsample(w2)
self.blocks = _residual_stack(config.num_post_blocks, w2, w3, er, c)
self.output_proj = Kandinsky6SRLatentUpscalerOutputHead(w3, c, zq_dim=c)
def forward(self, x: Tensor, zq: Tensor) -> Tensor:
x = _run_blocks(self.adapter, x, zq)
x = self.finisher(x)
x = _run_blocks(self.mid_blocks, x, zq)
return _second_stage(self.upsample, self.blocks, self.output_proj, x, zq)
class Kandinsky6SRLatentUpscaler(nn.Module):
"""One bank entry: ``[B, C, T, h, w]`` -> ``[B, C, T, s*h, s*w]`` for ``s`` = 4, or 2 with
``enable_x2_entry``."""
def __init__(self, config: Kandinsky6SRLatentUpscalerEntryConfig) -> None:
super().__init__()
c = config.in_channels
w1, w2, w3 = config.stage_channels
er = config.expand_ratio
self.input_proj = _stem(c, w1)
# The 2x deep-supervision head of training: part of the checkpoint, never run at inference.
self.mid_output_head = _mid_output_head(w2, c)
self.output_proj = Kandinsky6SRLatentUpscalerOutputHead(w3, c, zq_dim=c)
self.upsample_1 = PXSUpsample(w1)
self.upsample_2 = PXSUpsample(w2)
self.pre_blocks = _residual_stack(config.num_pre_blocks, w1, w1, er, c)
self.mid_blocks = _residual_stack(config.num_mid_blocks, w1, w2, er, c)
self.post_blocks = _residual_stack(config.num_post_blocks, w2, w3, er, c)
self.mid_input_proj: nn.Sequential | None = None
self.x2_branch: X2Branch | None = None
if config.enable_x2_entry:
self.mid_input_proj = _stem(c, config.hidden_channels)
self.x2_branch = X2Branch(config)
def forward(self, z: Tensor, scale: int) -> Tensor:
zq = z
if scale == 2:
if self.x2_branch is None or self.mid_input_proj is None:
raise ValueError(
"this latent-upscaler entry has no x2 path (enable_x2_entry=false)"
)
return self.x2_branch(self.mid_input_proj(z), zq)
if scale != 4:
raise ValueError(
f"latent-upscaler entries upsample by 2 or 4, got scale={scale!r}"
)
x = _run_blocks(self.pre_blocks, self.input_proj(z), zq)
x = _run_blocks(self.mid_blocks, self.upsample_1(x), zq)
return _second_stage(self.upsample_2, self.post_blocks, self.output_proj, x, zq)
class Kandinsky6SRLatentUpscalerBank(nn.Module, LayerwiseOffloadableModuleMixin):
"""Select the checkpoint bank entry to upscale scaled KVAE latents in H/W."""
layerwise_offload_dit_group_enabled = False
def __init__(
self,
models: Sequence[Mapping[str, Any]],
scaling_factor: float = 1.0,
scales: Sequence[int] = (2, 4),
) -> None:
super().__init__()
if not models:
raise ValueError("latent upscaler config declares no `models` entries")
entries = [
Kandinsky6SRLatentUpscalerEntryConfig.from_dict(spec, index)
for index, spec in enumerate(models)
]
entries_by_scale = {entry.target_scale: entry for entry in entries}
if len(entries_by_scale) != len(entries):
raise ValueError("latent upscaler `models` has duplicate target scales")
self._scales = tuple(int(scale) for scale in scales)
if len(set(self._scales)) != len(self._scales) or set(self._scales) != set(
entries_by_scale
):
raise ValueError(
"latent upscaler `scales` must match the target scales in `models`: "
f"{self._scales} vs {tuple(entries_by_scale)}"
)
self.scaling_factor = float(scaling_factor)
self._models = nn.ModuleList(
[
Kandinsky6SRLatentUpscaler(entries_by_scale[scale])
for scale in self._scales
]
)
self.layer_names = []
for index, entry in enumerate(self._models):
self.layer_names += [
f"_models.{index}.pre_blocks",
f"_models.{index}.mid_blocks",
f"_models.{index}.post_blocks",
]
if entry.x2_branch is not None:
self.layer_names += [
f"_models.{index}.x2_branch.adapter",
f"_models.{index}.x2_branch.mid_blocks",
f"_models.{index}.x2_branch.blocks",
]
@property
def scales(self) -> tuple[int, ...]:
"""The upscale factors this bank serves, ascending."""
return tuple(sorted(self._scales))
def for_scale(self, scale: int) -> nn.Module | None:
"""Return the bank entry matching ``scale``, or ``None`` if the bank does not serve it."""
scale = int(scale)
return (
self._models[self._scales.index(scale)] if scale in self._scales else None
)
def upscale(self, latent: Tensor, *, scale: int) -> Tensor:
"""Run the entry for ``scale`` on a *scaled* latent ``[B, C, T, h, w]``."""
entry = self.for_scale(scale)
if entry is None:
raise ValueError(
f"the latent-upscaler bank has no x{scale} entry (scales: {self.scales})"
)
return entry(latent, scale)
def forward(self, z: Tensor, *, scale: int) -> Tensor:
return self.upscale(z, scale=scale)
EntryClass = Kandinsky6SRLatentUpscalerBank
@@ -1,5 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
"""Alias-free-resampling DSP primitives shared by the BigVGAN-lineage vocoders.
Generic, architecture-agnostic building blocks (Kaiser-windowed-sinc low-pass
filtering, 2x up/down resampling, and the up-activate-down ``Activation1d``
wrapper) -- not specific to any one model. Previously duplicated
byte-for-byte between ``kandinsky6_audio_vae/alias_free.py`` and
``minimax_h3_audio_vae/alias_free.py``; both ``BigVGANV2``/``BigVGAN``
implementations now import ``Activation1d`` from here instead, matching this
package's existing cross-model-shared-VAE-code convention (see ``common.py``
in this same directory, reused by ``dac.py``, ``ltx_2_audio.py``,
``wanvae.py``, ``hunyuanvae.py``, and others).
"""
import math
@@ -25,6 +25,7 @@ from sglang.multimodal_gen.runtime.cache.conditioning import (
from sglang.multimodal_gen.runtime.distributed import (
get_decode_parallel_group_coordinator,
get_decode_parallel_world_size,
get_sp_group,
get_sp_parallel_rank,
get_sp_world_size,
model_parallel_is_initialized,
@@ -286,7 +287,8 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
"""
Parallel version of tiled_decode that distributes both temporal and spatial computation across GPUs
"""
world_size, rank = get_sp_world_size(), get_sp_parallel_rank()
sp_group = get_sp_group()
world_size, rank = sp_group.world_size, sp_group.rank_in_group
_, _, T, H, W = z.shape
tile_latent_min_height = (
@@ -361,7 +363,7 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
torch.zeros(1, device=results.device, dtype=torch.int64)
for _ in range(world_size)
]
dist.all_gather(all_sizes, local_size)
dist.all_gather(all_sizes, local_size, group=sp_group.device_group)
max_size = max(size.item() for size in all_sizes)
padded_results = torch.zeros(
@@ -375,8 +377,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
.repeat(world_size, *[1] * len(padded_results.shape))
.contiguous()
)
all_gather_single(gathered_results, padded_results)
dist.all_gather_object(gathered_dim_metadata, local_dim_metadata)
all_gather_single(
gathered_results.view(-1), padded_results, group=sp_group.device_group
)
dist.all_gather_object(
gathered_dim_metadata, local_dim_metadata, group=sp_group.cpu_group
)
gathered_dim_metadata = cast(list[list[torch.Size]], gathered_dim_metadata)
data: list = [
@@ -35,6 +35,7 @@ from sglang.multimodal_gen.runtime.layers.parallel_conv import (
disable_spatial_parallel_decode,
gather_and_trim_height,
gather_variable_height,
spatial_parallel_decode_disabled,
split_height_for_parallel_decode,
)
from sglang.multimodal_gen.runtime.models.vaes.common import (
@@ -125,7 +126,11 @@ class HunyuanVAEAttention(nn.Module):
)
def forward(
self, hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
*,
num_frames: int | None = None,
) -> torch.Tensor:
residual = hidden_states
@@ -145,10 +150,30 @@ class HunyuanVAEAttention(nn.Module):
key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
# Perform scaled dot-product attention
hidden_states = F.scaled_dot_product_attention(
query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
)
if num_frames is None:
hidden_states = F.scaled_dot_product_attention(
query,
key,
value,
attn_mask=attention_mask,
dropout_p=0.0,
is_causal=False,
)
else:
# each frame sees all spatial tokens in itself and earlier frames
# slicing the key prefix avoids a quadratic whole-video mask
tokens_per_frame = sequence_length // num_frames
hidden_states = torch.cat(
[
F.scaled_dot_product_attention(
query[:, :, start : start + tokens_per_frame],
key[:, :, : start + tokens_per_frame],
value[:, :, : start + tokens_per_frame],
)
for start in range(0, sequence_length, tokens_per_frame)
],
dim=2,
)
# Reshape back
hidden_states = hidden_states.transpose(1, 2).reshape(
@@ -278,6 +303,8 @@ class HunyuanVideoUpsampleCausal3D(nn.Module):
else:
hidden_states = first_frame
# release interpolation buffers before allocating convolution workspace
del first_frame, other_frames
hidden_states = self.conv(hidden_states)
return hidden_states
@@ -422,19 +449,25 @@ class HunyuanVideoMidBlock3D(nn.Module):
self, attn: HunyuanVAEAttention, hidden_states: torch.Tensor
) -> torch.Tensor:
heights = None
if self.spatial_parallel:
spatial_parallel = (
self.spatial_parallel and not spatial_parallel_decode_disabled()
)
if spatial_parallel:
hidden_states, heights = gather_variable_height(hidden_states)
batch_size, num_channels, num_frames, height, width = hidden_states.shape
hidden_states = hidden_states.permute(0, 2, 3, 4, 1).flatten(1, 3)
attention_mask = prepare_causal_attention_mask(
num_frames,
height * width,
hidden_states.dtype,
hidden_states.device,
batch_size=batch_size,
)
hidden_states = attn(hidden_states, attention_mask=attention_mask)
if spatial_parallel:
hidden_states = attn(hidden_states, num_frames=num_frames)
else:
attention_mask = prepare_causal_attention_mask(
num_frames,
height * width,
hidden_states.dtype,
hidden_states.device,
batch_size=batch_size,
)
hidden_states = attn(hidden_states, attention_mask=attention_mask)
hidden_states = hidden_states.unflatten(1, (num_frames, height, width)).permute(
0, 4, 1, 2, 3
)
@@ -0,0 +1,120 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky 6 mel VAE and waveform vocoder.
The vae.*, vocoder.* and mel_converter.* names match the official checkpoint,
allowing the generic VAELoader to load the full tree without key remapping."""
from __future__ import annotations
import torch
import torch.nn as nn
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
Kandinsky6AudioVAEConfig,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.bigvgan import (
BigVGANV2,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.mel_converter import (
MelConverter,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.mmaudio_vae import (
MMAudioVAE,
)
class Kandinsky6AudioVAE(nn.Module, LayerwiseOffloadableModuleMixin):
"""MMAudio mel codec and BigVGAN vocoder with checkpoint-compatible nesting.
decode: [B, embed_dim, A] -> [B, num_mels, T]
vocode: [B, num_mels, T] -> [B, 1, samples]
wrapped_decode and wrapped_encode compose the codec and waveform stages."""
# official checkpoint class name
_aliases = ["MMAudioVAE"]
# Decode-only small model relative to the DiT; no benefit from DiT-group
# offload tuning knobs. Mirrors MiniMaxH3AudioVAE's choice.
layerwise_offload_dit_group_enabled = False
def __init__(self, config: Kandinsky6AudioVAEConfig) -> None:
super().__init__()
arch = config.arch_config
self.scaling_factor = float(arch.scaling_factor)
self.mean_value = float(arch.mean_value)
# audio latent stride; frame-count alignment is owned by the pipeline config
self.downsample_factor = 1024
self.mel_converter = MelConverter(
sampling_rate=44100,
n_fft=2048,
num_mels=128,
hop_size=512,
win_size=2048,
fmin=0,
fmax=22050,
)
# checkpoint weights include the encoder regardless of need_vae_encoder
self.vae = MMAudioVAE(mode=arch.mode, need_encoder=True)
# normalize random initialization before loading; checkpoint weights are
# already normalized and must never be renormalized
self.vae.remove_weight_norm()
self.vocoder = BigVGANV2(dict(arch.vocoder_config))
# strip PyTorch weight_norm before loading plain .weight checkpoint keys;
# this differs from MMAudio's RMS renormalization above
self.vocoder.remove_weight_norm()
# resolution containers are not called; hooks belong on their blocks
self.layer_names = (
[
f"{path}.{index}.{group}"
for path, levels in (
("vae.encoder.down", self.vae.encoder.down),
("vae.decoder.up", self.vae.decoder.up),
)
for index in range(len(levels))
for group in ("block", "attn")
]
+ [f"vocoder.ups.{index}" for index in range(len(self.vocoder.ups))]
+ ["vocoder.resblocks"]
)
def decode(self, latents: torch.Tensor) -> torch.Tensor:
"""latents: [B, embed_dim, A] (1D-conv channel-first) -> mel: [B, num_mels, T]."""
return self.vae.decode(latents, unnormalize_output=True)
def vocode(self, mel: torch.Tensor) -> torch.Tensor:
"""mel: [B, num_mels, T] -> waveform: [B, 1, samples]."""
# mel denormalization returns fp32; cast to the loaded vocoder's dtype
vocoder_dtype = next(self.vocoder.parameters()).dtype
return self.vocoder(mel.to(dtype=vocoder_dtype))
def wrapped_decode(self, latents: torch.Tensor) -> torch.Tensor:
"""latents -> waveform in one call, matching the diffusers
reference's MMAudioVAE.wrapped_decode."""
return self.vocode(self.decode(latents))
def encode_audio(self, waveform: torch.Tensor):
"""waveform -> mel -> VAE posterior. See MelConverter's module
docstring: not exercised by / verified against the current
(decode-only) inference path."""
mel = self.mel_converter(waveform)
return self.vae.encode(mel)
def wrapped_encode(self, waveform: torch.Tensor) -> torch.Tensor:
"""waveform -> mean audio latent in one call, matching the
diffusers reference's MMAudioVAE.wrapped_encode."""
return self.encode_audio(waveform).mean
def remove_weight_norm(self) -> Kandinsky6AudioVAE:
"""Idempotent MMAudio weight normalization, normally applied before loading."""
self.vae.remove_weight_norm()
return self
EntryClass = Kandinsky6AudioVAE
__all__ = ["Kandinsky6AudioVAE"]
@@ -0,0 +1,31 @@
This directory contains code adapted from the following MIT-licensed projects:
- NVIDIA BigVGAN, bigvgan.py
https://github.com/NVIDIA/BigVGAN
Copyright (c) 2024 NVIDIA CORPORATION.
- MMAudio, mmaudio_vae.py
https://github.com/hkchengrex/MMAudio
Copyright (c) 2024 Sony Research Inc.
The following license applies to those portions of the code:
MIT License
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
"""Audio codec building blocks; the registry entry lives in ../kandinsky6_audio.py."""
from .bigvgan import BigVGANV2
from .mel_converter import MelConverter
from .mmaudio_vae import DiagonalGaussianDistribution, MMAudioVAE
__all__ = ["BigVGANV2", "DiagonalGaussianDistribution", "MMAudioVAE", "MelConverter"]
@@ -0,0 +1,350 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2024 NVIDIA CORPORATION.
# Adapted from NVIDIA BigVGAN-v2 (MIT license) via FastVideo's
# fastvideo/models/audio/bigvgan.py. The CUDA activation kernel is
# intentionally excluded; this module uses the portable PyTorch path for
# deterministic loading and parity across platforms.
# Original license terms are retained in THIRD_PARTY_LICENSES.
#
# This top-level generator class (BigVGANV2) and its Snake/SnakeBeta/
# AMPBlock1/AMPBlock2 building blocks are freshly ported here rather than
# reused from this codebase's sibling ``minimax_h3_audio_vae/bigvgan.py`` --
# that module's ``BigVGAN``/``AMPBlock1`` only support resblock="1" and
# activation="snakebeta" (MiniMax's own DAC-vocoder config never needs more),
# while this implementation supports either resblock variant or activation. The
# alias-free resampling primitives (LowPassFilter1d/UpSample1d/DownSample1d/
# Activation1d) ARE architecture-agnostic DSP building blocks shared with
# that same sibling module, though -- both import ``Activation1d`` from
# ``runtime/models/vaes/alias_free.py`` (sibling of this package, alongside
# ``common.py``) instead of each keeping its own copy.
from __future__ import annotations
from typing import Any
import torch
import torch.nn as nn
from torch.nn import Conv1d, ConvTranspose1d
from torch.nn.utils.parametrizations import weight_norm
from torch.nn.utils.parametrize import remove_parametrizations
from sglang.multimodal_gen.runtime.models.vaes.alias_free import Activation1d
class AttrDict(dict):
def __init__(self, *args, **kwargs) -> None:
super().__init__(*args, **kwargs)
self.__dict__ = self
def get_padding(kernel_size: int, dilation: int = 1) -> int:
return int((kernel_size * dilation - dilation) / 2)
def init_weights(module: nn.Module, mean: float = 0.0, std: float = 0.01) -> None:
if "Conv" in module.__class__.__name__:
module.weight.data.normal_(mean, std)
class Snake(nn.Module):
def __init__(
self,
in_features: int,
alpha: float = 1.0,
alpha_trainable: bool = True,
alpha_logscale: bool = False,
) -> None:
super().__init__()
self.in_features = in_features
self.alpha_logscale = alpha_logscale
initial = (
torch.zeros(in_features) * alpha
if alpha_logscale
else torch.ones(in_features) * alpha
)
self.alpha = nn.Parameter(initial, requires_grad=alpha_trainable)
self.no_div_by_zero = 1e-9
def forward(self, x: torch.Tensor) -> torch.Tensor:
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
if self.alpha_logscale:
alpha = torch.exp(alpha)
return x + (1.0 / (alpha + self.no_div_by_zero)) * torch.pow(
torch.sin(x * alpha), 2
)
class SnakeBeta(nn.Module):
def __init__(
self,
in_features: int,
alpha: float = 1.0,
alpha_trainable: bool = True,
alpha_logscale: bool = False,
) -> None:
super().__init__()
self.in_features = in_features
self.alpha_logscale = alpha_logscale
initial = (
torch.zeros(in_features) * alpha
if alpha_logscale
else torch.ones(in_features) * alpha
)
self.alpha = nn.Parameter(initial.clone(), requires_grad=alpha_trainable)
self.beta = nn.Parameter(initial.clone(), requires_grad=alpha_trainable)
self.no_div_by_zero = 1e-9
def forward(self, x: torch.Tensor) -> torch.Tensor:
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
beta = self.beta.unsqueeze(0).unsqueeze(-1)
if self.alpha_logscale:
alpha = torch.exp(alpha)
beta = torch.exp(beta)
return x + (1.0 / (beta + self.no_div_by_zero)) * torch.pow(
torch.sin(x * alpha), 2
)
def _activation(name: str, channels: int, logscale: bool) -> Activation1d:
if name == "snake":
activation = Snake(channels, alpha_logscale=logscale)
elif name == "snakebeta":
activation = SnakeBeta(channels, alpha_logscale=logscale)
else:
raise ValueError(f"Unsupported BigVGAN activation: {name}")
return Activation1d(activation)
class AMPBlock1(nn.Module):
def __init__(
self,
config: AttrDict,
channels: int,
kernel_size: int = 3,
dilation: tuple[int, ...] = (1, 3, 5),
activation: str = "snake",
) -> None:
super().__init__()
self.convs1 = nn.ModuleList(
[
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
stride=1,
dilation=rate,
padding=get_padding(kernel_size, rate),
)
)
for rate in dilation
]
)
self.convs1.apply(init_weights)
self.convs2 = nn.ModuleList(
[
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
stride=1,
dilation=1,
padding=get_padding(kernel_size, 1),
)
)
for _ in dilation
]
)
self.convs2.apply(init_weights)
self.activations = nn.ModuleList(
[
_activation(activation, channels, config.snake_logscale)
for _ in range(len(self.convs1) + len(self.convs2))
]
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
acts1, acts2 = self.activations[::2], self.activations[1::2]
for conv1, conv2, act1, act2 in zip(
self.convs1, self.convs2, acts1, acts2, strict=True
):
residual = conv2(act2(conv1(act1(x))))
x = residual + x
return x
def remove_weight_norm(self) -> None:
for layer in self.convs1:
remove_parametrizations(layer, "weight")
for layer in self.convs2:
remove_parametrizations(layer, "weight")
class AMPBlock2(nn.Module):
def __init__(
self,
config: AttrDict,
channels: int,
kernel_size: int = 3,
dilation: tuple[int, ...] = (1, 3, 5),
activation: str = "snake",
) -> None:
super().__init__()
self.convs = nn.ModuleList(
[
weight_norm(
Conv1d(
channels,
channels,
kernel_size,
stride=1,
dilation=rate,
padding=get_padding(kernel_size, rate),
)
)
for rate in dilation
]
)
self.convs.apply(init_weights)
self.activations = nn.ModuleList(
[
_activation(activation, channels, config.snake_logscale)
for _ in self.convs
]
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
for conv, activation in zip(self.convs, self.activations, strict=True):
x = conv(activation(x)) + x
return x
def remove_weight_norm(self) -> None:
for layer in self.convs:
remove_parametrizations(layer, "weight")
class BigVGANV2(nn.Module):
"""BigVGAN-v2 using checkpoint vocoder_config and the portable PyTorch path."""
def __init__(self, config: dict[str, Any]) -> None:
super().__init__()
config = dict(config)
config.pop("_class_name", None)
config.pop("architectures", None)
weight_norm_removed = bool(config.pop("weight_norm_removed", False))
self.config = AttrDict(config)
if self.config.get("use_cuda_kernel", False):
raise ValueError(
"Kandinsky6 BigVGANV2 supports only the portable PyTorch path"
)
self.config["use_cuda_kernel"] = False
self.num_kernels = len(self.config.resblock_kernel_sizes)
self.num_upsamples = len(self.config.upsample_rates)
self.conv_pre = weight_norm(
Conv1d(
self.config.num_mels,
self.config.upsample_initial_channel,
7,
1,
padding=3,
)
)
if self.config.resblock == "1":
block_class = AMPBlock1
elif self.config.resblock == "2":
block_class = AMPBlock2
else:
raise ValueError(f"Unsupported BigVGAN resblock: {self.config.resblock}")
self.ups = nn.ModuleList()
for index, (rate, kernel) in enumerate(
zip(
self.config.upsample_rates,
self.config.upsample_kernel_sizes,
strict=True,
)
):
self.ups.append(
nn.ModuleList(
[
weight_norm(
ConvTranspose1d(
self.config.upsample_initial_channel // (2**index),
self.config.upsample_initial_channel
// (2 ** (index + 1)),
kernel,
rate,
padding=(kernel - rate) // 2,
)
)
]
)
)
self.resblocks = nn.ModuleList()
for index in range(len(self.ups)):
channels = self.config.upsample_initial_channel // (2 ** (index + 1))
for kernel, dilation in zip(
self.config.resblock_kernel_sizes,
self.config.resblock_dilation_sizes,
strict=True,
):
self.resblocks.append(
block_class(
self.config,
channels,
kernel,
tuple(dilation),
activation=self.config.activation,
)
)
channels = self.config.upsample_initial_channel // (2 ** len(self.ups))
self.activation_post = _activation(
self.config.activation, channels, self.config.snake_logscale
)
self.use_bias_at_final = self.config.get("use_bias_at_final", True)
self.conv_post = weight_norm(
Conv1d(channels, 1, 7, 1, padding=3, bias=self.use_bias_at_final)
)
for upsampler in self.ups:
upsampler.apply(init_weights)
self.conv_post.apply(init_weights)
self.use_tanh_at_final = self.config.get("use_tanh_at_final", True)
if weight_norm_removed:
self.remove_weight_norm()
def forward(self, mel: torch.Tensor) -> torch.Tensor:
hidden = self.conv_pre(mel)
for index in range(self.num_upsamples):
for upsampler in self.ups[index]:
hidden = upsampler(hidden)
accumulated = None
for kernel in range(self.num_kernels):
block_output = self.resblocks[index * self.num_kernels + kernel](hidden)
if accumulated is None:
accumulated = block_output
else:
# Preserve BigVGAN's published sequential accumulation
# order. Tree reduction drifts through later nonlinear
# upsampling stages with the full checkpoint.
accumulated += block_output
assert accumulated is not None
hidden = accumulated / self.num_kernels
hidden = self.conv_post(self.activation_post(hidden))
if self.use_tanh_at_final:
return torch.tanh(hidden)
return torch.clamp(hidden, min=-1.0, max=1.0)
def remove_weight_norm(self) -> None:
try:
for upsamplers in self.ups:
for upsampler in upsamplers:
remove_parametrizations(upsampler, "weight")
for block in self.resblocks:
block.remove_weight_norm()
remove_parametrizations(self.conv_pre, "weight")
remove_parametrizations(self.conv_post, "weight")
except ValueError:
# Idempotent for pipeline setup and converted checkpoints.
return
@@ -0,0 +1,69 @@
# SPDX-License-Identifier: Apache-2.0
# Adapted from FastVideo's fastvideo/models/audio/kandinsky6_audio_vae.py.
"""Checkpoint-compatible waveform-to-log-mel frontend.
Included for state-dict completeness; generation only decodes audio latents,
so waveform encoding has not been validated end-to-end."""
from __future__ import annotations
import torch
import torch.nn as nn
import torch.nn.functional as F
class MelConverter(nn.Module):
"""Waveform -> log-mel-spectrogram STFT frontend. See module docstring:
not exercised by the current (decode-only) inference path."""
def __init__(
self,
*,
sampling_rate: int = 44100,
n_fft: int = 2048,
num_mels: int = 128,
hop_size: int = 512,
win_size: int = 2048,
fmin: int = 0,
fmax: int | None = 22050,
) -> None:
super().__init__()
self.n_fft = n_fft
self.hop_size = hop_size
self.win_size = win_size
self.register_buffer(
"hann_window", torch.hann_window(win_size), persistent=True
)
try:
from librosa.filters import mel as librosa_mel_fn
mel_basis = torch.from_numpy(
librosa_mel_fn(
sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax
)
).float()
except Exception:
# generation does not use this buffer; the checkpoint supplies its real values
mel_basis = torch.zeros(num_mels, n_fft // 2 + 1)
self.register_buffer("mel_basis", mel_basis, persistent=True)
def forward(self, waveform: torch.Tensor, center: bool = False) -> torch.Tensor:
waveform = waveform.clamp(min=-1.0, max=1.0)
pad = (self.n_fft - self.hop_size) // 2
padded = F.pad(waveform.unsqueeze(1), (pad, pad), mode="reflect").squeeze(1)
spec = torch.stft(
padded,
self.n_fft,
hop_length=self.hop_size,
win_length=self.win_size,
window=self.hann_window,
center=center,
pad_mode="reflect",
normalized=False,
onesided=True,
return_complex=True,
)
spec = torch.view_as_real(spec)
magnitude = torch.sqrt(spec.pow(2).sum(-1) + 1e-9).float()
mel = torch.matmul(self.mel_basis, magnitude)
return torch.log(torch.clamp(mel, min=1e-5))
@@ -0,0 +1,854 @@
# SPDX-License-Identifier: Apache-2.0
#
# Adapted from MMAudio (https://github.com/hkchengrex/MMAudio), MIT License,
# Copyright (c) 2024 Sony Research Inc., via FastVideo's
# fastvideo/models/audio/mmaudio_vae.py. This is the mel-spectrogram
# <-> latent 1D-conv VAE codec nested at ``Kandinsky6AudioVAE.vae``
# (see kandinsky6_audio.py in the parent directory).
# Original license terms are retained in THIRD_PARTY_LICENSES.
#
# Ported verbatim except: FastVideo's ``MMAudioVAE.load_weights()`` (and its
# ``fastvideo.models.loader.weight_utils.default_weight_loader`` import) is
# dropped -- sglang's generic VAELoader never calls a per-submodule
# ``load_weights()`` hook, it does a single plain ``nn.Module.load_state_dict``
# over the whole ``Kandinsky6AudioVAE`` tree, so that method would be dead code
# here.
"""Native 1D audio VAE used by MMAudio (mel-spectrogram <-> latent codec)."""
from __future__ import annotations
import logging
import math
from typing import Any
import torch
import torch.nn as nn
import torch.nn.functional as F
from einops import rearrange
logger = logging.getLogger(__name__)
DATA_MEAN_80D = [
-1.6058,
-1.3676,
-1.2520,
-1.2453,
-1.2078,
-1.2224,
-1.2419,
-1.2439,
-1.2922,
-1.2927,
-1.3170,
-1.3543,
-1.3401,
-1.3836,
-1.3907,
-1.3912,
-1.4313,
-1.4152,
-1.4527,
-1.4728,
-1.4568,
-1.5101,
-1.5051,
-1.5172,
-1.5623,
-1.5373,
-1.5746,
-1.5687,
-1.6032,
-1.6131,
-1.6081,
-1.6331,
-1.6489,
-1.6489,
-1.6700,
-1.6738,
-1.6953,
-1.6969,
-1.7048,
-1.7280,
-1.7361,
-1.7495,
-1.7658,
-1.7814,
-1.7889,
-1.8064,
-1.8221,
-1.8377,
-1.8417,
-1.8643,
-1.8857,
-1.8929,
-1.9173,
-1.9379,
-1.9531,
-1.9673,
-1.9824,
-2.0042,
-2.0215,
-2.0436,
-2.0766,
-2.1064,
-2.1418,
-2.1855,
-2.2319,
-2.2767,
-2.3161,
-2.3572,
-2.3954,
-2.4282,
-2.4659,
-2.5072,
-2.5552,
-2.6074,
-2.6584,
-2.7107,
-2.7634,
-2.8266,
-2.8981,
-2.9673,
]
DATA_STD_80D = [
1.0291,
1.0411,
1.0043,
0.9820,
0.9677,
0.9543,
0.9450,
0.9392,
0.9343,
0.9297,
0.9276,
0.9263,
0.9242,
0.9254,
0.9232,
0.9281,
0.9263,
0.9315,
0.9274,
0.9247,
0.9277,
0.9199,
0.9188,
0.9194,
0.9160,
0.9161,
0.9146,
0.9161,
0.9100,
0.9095,
0.9145,
0.9076,
0.9066,
0.9095,
0.9032,
0.9043,
0.9038,
0.9011,
0.9019,
0.9010,
0.8984,
0.8983,
0.8986,
0.8961,
0.8962,
0.8978,
0.8962,
0.8973,
0.8993,
0.8976,
0.8995,
0.9016,
0.8982,
0.8972,
0.8974,
0.8949,
0.8940,
0.8947,
0.8936,
0.8939,
0.8951,
0.8956,
0.9017,
0.9167,
0.9436,
0.9690,
1.0003,
1.0225,
1.0381,
1.0491,
1.0545,
1.0604,
1.0761,
1.0929,
1.1089,
1.1196,
1.1176,
1.1156,
1.1117,
1.1070,
]
DATA_MEAN_128D = [
-3.3462,
-2.6723,
-2.4893,
-2.3143,
-2.2664,
-2.3317,
-2.1802,
-2.4006,
-2.2357,
-2.4597,
-2.3717,
-2.4690,
-2.5142,
-2.4919,
-2.6610,
-2.5047,
-2.7483,
-2.5926,
-2.7462,
-2.7033,
-2.7386,
-2.8112,
-2.7502,
-2.9594,
-2.7473,
-3.0035,
-2.8891,
-2.9922,
-2.9856,
-3.0157,
-3.1191,
-2.9893,
-3.1718,
-3.0745,
-3.1879,
-3.2310,
-3.1424,
-3.2296,
-3.2791,
-3.2782,
-3.2756,
-3.3134,
-3.3509,
-3.3750,
-3.3951,
-3.3698,
-3.4505,
-3.4509,
-3.5089,
-3.4647,
-3.5536,
-3.5788,
-3.5867,
-3.6036,
-3.6400,
-3.6747,
-3.7072,
-3.7279,
-3.7283,
-3.7795,
-3.8259,
-3.8447,
-3.8663,
-3.9182,
-3.9605,
-3.9861,
-4.0105,
-4.0373,
-4.0762,
-4.1121,
-4.1488,
-4.1874,
-4.2461,
-4.3170,
-4.3639,
-4.4452,
-4.5282,
-4.6297,
-4.7019,
-4.7960,
-4.8700,
-4.9507,
-5.0303,
-5.0866,
-5.1634,
-5.2342,
-5.3242,
-5.4053,
-5.4927,
-5.5712,
-5.6464,
-5.7052,
-5.7619,
-5.8410,
-5.9188,
-6.0103,
-6.0955,
-6.1673,
-6.2362,
-6.3120,
-6.3926,
-6.4797,
-6.5565,
-6.6511,
-6.8130,
-6.9961,
-7.1275,
-7.2457,
-7.3576,
-7.4663,
-7.6136,
-7.7469,
-7.8815,
-8.0132,
-8.1515,
-8.3071,
-8.4722,
-8.7418,
-9.3975,
-9.6628,
-9.7671,
-9.8863,
-9.9992,
-10.0860,
-10.1709,
-10.5418,
-11.2795,
-11.3861,
]
DATA_STD_128D = [
2.3804,
2.4368,
2.3772,
2.3145,
2.2803,
2.2510,
2.2316,
2.2083,
2.1996,
2.1835,
2.1769,
2.1659,
2.1631,
2.1618,
2.1540,
2.1606,
2.1571,
2.1567,
2.1612,
2.1579,
2.1679,
2.1683,
2.1634,
2.1557,
2.1668,
2.1518,
2.1415,
2.1449,
2.1406,
2.1350,
2.1313,
2.1415,
2.1281,
2.1352,
2.1219,
2.1182,
2.1327,
2.1195,
2.1137,
2.1080,
2.1179,
2.1036,
2.1087,
2.1036,
2.1015,
2.1068,
2.0975,
2.0991,
2.0902,
2.1015,
2.0857,
2.0920,
2.0893,
2.0897,
2.0910,
2.0881,
2.0925,
2.0873,
2.0960,
2.0900,
2.0957,
2.0958,
2.0978,
2.0936,
2.0886,
2.0905,
2.0845,
2.0855,
2.0796,
2.0840,
2.0813,
2.0817,
2.0838,
2.0840,
2.0917,
2.1061,
2.1431,
2.1976,
2.2482,
2.3055,
2.3700,
2.4088,
2.4372,
2.4609,
2.4731,
2.4847,
2.5072,
2.5451,
2.5772,
2.6147,
2.6529,
2.6596,
2.6645,
2.6726,
2.6803,
2.6812,
2.6899,
2.6916,
2.6931,
2.6998,
2.7062,
2.7262,
2.7222,
2.7158,
2.7041,
2.7485,
2.7491,
2.7451,
2.7485,
2.7233,
2.7297,
2.7233,
2.7145,
2.6958,
2.6788,
2.6439,
2.6007,
2.4786,
2.2469,
2.1877,
2.1392,
2.0717,
2.0107,
1.9676,
1.9140,
1.7102,
0.9101,
0.7164,
]
_NORMALIZATION_EPSILON = 1e-4
_SILU_DIVISOR = 0.596
_RESIDUAL_BLEND = 0.3
_RESIDUAL_DIVISOR = math.hypot(1.0 - _RESIDUAL_BLEND, _RESIDUAL_BLEND)
def _rms_normalize(x: torch.Tensor, dim: int | tuple[int, ...]) -> torch.Tensor:
dims = (dim,) if isinstance(dim, int) else dim
element_count = math.prod(x.shape[axis] for axis in dims)
l2_norm = torch.linalg.vector_norm(x, dim=dims, keepdim=True, dtype=torch.float32)
rms = torch.add(_NORMALIZATION_EPSILON, l2_norm, alpha=element_count**-0.5)
return x / rms.to(x.dtype)
def _conv1d(in_channels: int, out_channels: int, kernel_size: int) -> nn.Conv1d:
return nn.Conv1d(
in_channels, out_channels, kernel_size, padding=kernel_size // 2, bias=False
)
def _conv1d_with_gain(
conv: nn.Conv1d, x: torch.Tensor, gain: torch.Tensor | float
) -> torch.Tensor:
return F.conv1d(
x,
conv.weight * gain,
stride=conv.stride,
padding=conv.padding,
dilation=conv.dilation,
groups=conv.groups,
)
class DiagonalGaussianDistribution:
def __init__(self, parameters: torch.Tensor, deterministic: bool = False) -> None:
self.parameters = parameters
self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)
self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
self.deterministic = deterministic
self.std = torch.exp(0.5 * self.logvar)
self.var = torch.exp(self.logvar)
if deterministic:
self.var = self.std = torch.zeros_like(self.mean)
def sample(self, generator: torch.Generator | None = None) -> torch.Tensor:
noise = torch.empty_like(self.mean).normal_(generator=generator)
return self.mean + self.std * noise
def mode(self) -> torch.Tensor:
return self.mean
class ResnetBlock1D(nn.Module):
def __init__(
self,
*,
in_dim: int,
out_dim: int | None = None,
conv_shortcut: bool = False,
kernel_size: int = 3,
use_norm: bool = True,
) -> None:
super().__init__()
self.in_dim = in_dim
self.out_dim = in_dim if out_dim is None else out_dim
self.use_conv_shortcut = conv_shortcut
self.use_norm = use_norm
self.conv1 = _conv1d(in_dim, self.out_dim, kernel_size)
self.conv2 = _conv1d(self.out_dim, self.out_dim, kernel_size)
if self.in_dim != self.out_dim:
if conv_shortcut:
self.conv_shortcut = _conv1d(in_dim, self.out_dim, kernel_size)
else:
self.nin_shortcut = _conv1d(in_dim, self.out_dim, 1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.use_norm:
x = _rms_normalize(x, dim=1)
hidden = self.conv1(F.silu(x) / _SILU_DIVISOR)
hidden = self.conv2(F.silu(hidden) / _SILU_DIVISOR)
if self.in_dim != self.out_dim:
shortcut = (
self.conv_shortcut if self.use_conv_shortcut else self.nin_shortcut
)
x = shortcut(x)
return torch.lerp(x, hidden, _RESIDUAL_BLEND) / _RESIDUAL_DIVISOR
class AttnBlock1D(nn.Module):
def __init__(self, in_channels: int, num_heads: int = 1) -> None:
super().__init__()
self.in_channels = in_channels
self.num_heads = num_heads
self.qkv = _conv1d(in_channels, in_channels * 3, 1)
self.proj_out = _conv1d(in_channels, in_channels, 1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
qkv = self.qkv(x).reshape(x.shape[0], self.num_heads, -1, 3, x.shape[-1])
query, key, value = _rms_normalize(qkv, dim=2).unbind(3)
query = rearrange(query, "b h c l -> b h l c")
key = rearrange(key, "b h c l -> b h l c")
value = rearrange(value, "b h c l -> b h l c")
hidden = F.scaled_dot_product_attention(query, key, value)
hidden = rearrange(hidden, "b h l c -> b (h c) l")
return torch.lerp(x, self.proj_out(hidden), _RESIDUAL_BLEND) / _RESIDUAL_DIVISOR
class Upsample1D(nn.Module):
def __init__(self, in_channels: int, with_conv: bool) -> None:
super().__init__()
self.with_conv = with_conv
if with_conv:
self.conv = _conv1d(in_channels, in_channels, 3)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = F.interpolate(x, scale_factor=2.0, mode="nearest-exact")
return self.conv(x) if self.with_conv else x
class Downsample1D(nn.Module):
def __init__(self, in_channels: int, with_conv: bool) -> None:
super().__init__()
self.with_conv = with_conv
if with_conv:
self.conv1 = _conv1d(in_channels, in_channels, 1)
self.conv2 = _conv1d(in_channels, in_channels, 1)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.with_conv:
x = self.conv1(x)
x = F.avg_pool1d(x, kernel_size=2, stride=2)
return self.conv2(x) if self.with_conv else x
class Encoder1D(nn.Module):
def __init__(
self,
*,
dim: int,
ch_mult: tuple[int, ...],
num_res_blocks: int,
attn_layers: list[int],
down_layers: list[int],
in_dim: int,
embed_dim: int,
resamp_with_conv: bool = True,
double_z: bool = True,
kernel_size: int = 3,
clip_act: float = 256.0,
) -> None:
super().__init__()
self.dim = dim
self.num_layers = len(ch_mult)
self.num_res_blocks = num_res_blocks
self.in_channels = in_dim
self.clip_act = clip_act
self.down_layers = down_layers
self.attn_layers = attn_layers
self.conv_in = _conv1d(in_dim, dim, kernel_size)
in_ch_mult = (1,) + ch_mult
self.in_ch_mult = in_ch_mult
self.down = nn.ModuleList()
for level in range(self.num_layers):
block = nn.ModuleList()
attn = nn.ModuleList()
block_in = dim * in_ch_mult[level]
block_out = dim * ch_mult[level]
for _ in range(num_res_blocks):
block.append(
ResnetBlock1D(
in_dim=block_in,
out_dim=block_out,
kernel_size=kernel_size,
use_norm=True,
)
)
block_in = block_out
if level in attn_layers:
attn.append(AttnBlock1D(block_in))
down = nn.Module()
down.block = block
down.attn = attn
if level in down_layers:
down.downsample = Downsample1D(block_in, resamp_with_conv)
self.down.append(down)
self.mid = nn.Module()
self.mid.block_1 = ResnetBlock1D(
in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True
)
self.mid.attn_1 = AttnBlock1D(block_in)
self.mid.block_2 = ResnetBlock1D(
in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True
)
output_dim = 2 * embed_dim if double_z else embed_dim
self.conv_out = _conv1d(block_in, output_dim, kernel_size)
self.learnable_gain = nn.Parameter(torch.zeros([]))
def forward(self, x: torch.Tensor) -> torch.Tensor:
states = [self.conv_in(x)]
for level in range(self.num_layers):
for block_index in range(self.num_res_blocks):
hidden = self.down[level].block[block_index](states[-1])
if len(self.down[level].attn) > 0:
hidden = self.down[level].attn[block_index](hidden)
states.append(hidden.clamp(-self.clip_act, self.clip_act))
if level in self.down_layers:
states.append(self.down[level].downsample(states[-1]))
hidden = self.mid.block_1(states[-1])
hidden = self.mid.attn_1(hidden)
hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act)
return _conv1d_with_gain(
self.conv_out, F.silu(hidden) / _SILU_DIVISOR, self.learnable_gain + 1
)
class Decoder1D(nn.Module):
def __init__(
self,
*,
dim: int,
out_dim: int,
ch_mult: tuple[int, ...],
num_res_blocks: int,
attn_layers: list[int],
down_layers: list[int],
in_dim: int,
embed_dim: int,
kernel_size: int = 3,
resamp_with_conv: bool = True,
clip_act: float = 256.0,
) -> None:
super().__init__()
self.ch = dim
self.num_layers = len(ch_mult)
self.num_res_blocks = num_res_blocks
self.in_channels = in_dim
self.clip_act = clip_act
self.down_layers = [level + 1 for level in down_layers]
block_in = dim * ch_mult[-1]
self.conv_in = _conv1d(embed_dim, block_in, kernel_size)
self.mid = nn.Module()
self.mid.block_1 = ResnetBlock1D(
in_dim=block_in, out_dim=block_in, use_norm=True
)
self.mid.attn_1 = AttnBlock1D(block_in)
self.mid.block_2 = ResnetBlock1D(
in_dim=block_in, out_dim=block_in, use_norm=True
)
self.up = nn.ModuleList()
for level in reversed(range(self.num_layers)):
block = nn.ModuleList()
attn = nn.ModuleList()
block_out = dim * ch_mult[level]
for _ in range(num_res_blocks + 1):
block.append(
ResnetBlock1D(in_dim=block_in, out_dim=block_out, use_norm=True)
)
block_in = block_out
if level in attn_layers:
attn.append(AttnBlock1D(block_in))
up = nn.Module()
up.block = block
up.attn = attn
if level in self.down_layers:
up.upsample = Upsample1D(block_in, resamp_with_conv)
self.up.insert(0, up)
self.conv_out = _conv1d(block_in, out_dim, kernel_size)
self.learnable_gain = nn.Parameter(torch.zeros([]))
def forward(self, z: torch.Tensor) -> torch.Tensor:
hidden = self.conv_in(z)
hidden = self.mid.block_1(hidden)
hidden = self.mid.attn_1(hidden)
hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act)
for level in reversed(range(self.num_layers)):
for block_index in range(self.num_res_blocks + 1):
hidden = self.up[level].block[block_index](hidden)
if len(self.up[level].attn) > 0:
hidden = self.up[level].attn[block_index](hidden)
hidden = hidden.clamp(-self.clip_act, self.clip_act)
if level in self.down_layers:
hidden = self.up[level].upsample(hidden)
return _conv1d_with_gain(
self.conv_out, F.silu(hidden) / _SILU_DIVISOR, self.learnable_gain + 1
)
class MMAudioVAE(nn.Module):
"""MMAudio mel-spectrogram VAE for 16 kHz or 44.1 kHz audio."""
def __init__(
self, mode: str | dict[str, Any] = "44k", need_encoder: bool = False
) -> None:
super().__init__()
if isinstance(mode, dict):
config = mode
mode = config.get("mode", "44k")
need_encoder = config.get("need_encoder", need_encoder)
if mode == "16k":
data_dim, embed_dim, hidden_dim = 80, 20, 384
data_mean, data_std = DATA_MEAN_80D, DATA_STD_80D
elif mode == "44k":
data_dim, embed_dim, hidden_dim = 128, 40, 512
data_mean, data_std = DATA_MEAN_128D, DATA_STD_128D
else:
raise ValueError(f"Unknown MMAudio VAE mode: {mode}")
self.mode = mode
self.embed_dim = embed_dim
self._weights_normalized = False
self.register_buffer(
"data_mean", torch.tensor(data_mean, dtype=torch.float32).view(1, -1, 1)
)
self.register_buffer(
"data_std", torch.tensor(data_std, dtype=torch.float32).view(1, -1, 1)
)
if need_encoder:
self.encoder = Encoder1D(
dim=hidden_dim,
ch_mult=(1, 2, 4),
num_res_blocks=2,
attn_layers=[3],
down_layers=[0],
in_dim=data_dim,
embed_dim=embed_dim,
)
self.decoder = Decoder1D(
dim=hidden_dim,
ch_mult=(1, 2, 4),
num_res_blocks=2,
attn_layers=[3],
down_layers=[0],
in_dim=data_dim,
out_dim=data_dim,
embed_dim=embed_dim,
)
def encode(
self, mel: torch.Tensor, normalize_input: bool = True
) -> DiagonalGaussianDistribution:
self._require_normalized_weights()
if not hasattr(self, "encoder"):
raise RuntimeError("This MMAudio VAE was loaded decoder-only")
if normalize_input:
mel = self.normalize(mel)
return DiagonalGaussianDistribution(self.encoder(mel))
def decode(
self, latent: torch.Tensor, unnormalize_output: bool = True
) -> torch.Tensor:
self._require_normalized_weights()
mel = self.decoder(latent)
return self.unnormalize(mel) if unnormalize_output else mel
def forward(self, latent: torch.Tensor) -> torch.Tensor:
return self.decode(latent)
def normalize(self, mel: torch.Tensor) -> torch.Tensor:
return (mel - self.data_mean) / self.data_std
def unnormalize(self, mel: torch.Tensor) -> torch.Tensor:
return mel * self.data_std + self.data_mean
def _require_normalized_weights(self) -> None:
if not self._weights_normalized:
raise RuntimeError("call remove_weight_norm() before inference")
@torch.no_grad()
def remove_weight_norm(self) -> MMAudioVAE:
# Idempotent: the diffusers reference calls this exactly once, in
# ``MMAudioVAE.__init__``, BEFORE ``from_pretrained`` overwrites the
# freshly-constructed (random) Conv1d weights with a checkpoint's
# real, already-final values -- so a real checkpoint's own weights
# are never renormalized. Mirroring that timing here (see
# ``Kandinsky6AudioVAE.__init__``) means this method only ever runs
# against pre-load weights too; the guard below additionally makes a
# second, redundant call (e.g. the wrapper's own public
# ``remove_weight_norm()``) a true no-op instead of re-mutating an
# already-normalized (possibly checkpoint-loaded) weight a second
# time.
if self._weights_normalized:
return self
for name, module in self.named_modules():
if isinstance(module, nn.Conv1d):
weight = _rms_normalize(module.weight.to(torch.float32), dim=(1, 2))
weight = weight / math.sqrt(weight[0].numel())
module.weight.copy_(weight.to(module.weight.dtype))
logger.debug("Removed weight norm from %s", name)
self._weights_normalized = True
return self
@@ -0,0 +1,656 @@
# SPDX-License-Identifier: Apache-2.0
"""Causal Kandinsky 6 SR KVAE: spatial x16 and temporal x4.
Encode/decode segments preserve causal-convolution caches; segmentation affects
numerics. Pixels normalize as x/128-1, encode returns an unscaled posterior mean,
and decode returns normalized pixels. encoder.* and decoder.* match the checkpoint."""
from __future__ import annotations
import math
from typing import Any
import torch
import torch.nn as nn
import torch.nn.functional as F
from diffusers.models.autoencoders.vae import DecoderOutput
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
Kandinsky6SRVAEArchConfig,
Kandinsky6SRVAEConfig,
)
from sglang.multimodal_gen.runtime.distributed import (
get_decode_parallel_rank,
get_decode_parallel_world_size,
)
from sglang.multimodal_gen.runtime.layers.parallel_conv import (
SpatialParallelConv3d,
disable_spatial_parallel_decode,
gather_and_trim_height,
split_height_for_parallel_decode,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.models.vaes.common import (
can_install_spatial_shard_parallel_decode,
should_run_spatial_shard_parallel_decode,
)
from sglang.multimodal_gen.runtime.models.vaes.wanvae import WanRMS_norm
# Pixel frames per encode segment after the first (which has one more: the causal first frame).
_SEGMENT_FRAMES = 16
# Inputs above this many elements are convolved in temporal chunks: at 4x SR decode tiles
# (~1536x832) the full-resolution 256-channel convs of the last decoder level see ~5.6e9
# elements per segment, beyond 32-bit indexing, and chunking also keeps the transient conv
# workspace to a fraction of the activation.
_MAX_CONV_NUMEL = 2 * 10**9
class _SegmentCache:
"""Causal state carried from one temporal segment to the next during a single encode / decode call."""
def __init__(self) -> None:
self.first = True
self.padding: dict[nn.Module, torch.Tensor] = {}
def _silu(x: torch.Tensor) -> torch.Tensor:
# The output norms use this form and the resblocks use F.silu; the two are not bitwise
# equal in bf16, so this distinction (inherited from the Diffusers / FastVideo ports) is
# deliberate, not a simplification opportunity.
return x * torch.sigmoid(x)
class _K6RMSNorm(WanRMS_norm):
"""``WanRMS_norm`` with an explicit float32 upcast around the normalize, matching the
Diffusers reference's ``Kandinsky6SRRMSNorm`` (which upcasts unconditionally, not only for
fp16/bf16/fp8 inputs): at bf16/fp16 activations, normalizing without upcasting measurably
drifts from the reference over many resnet blocks."""
def forward(self, x: torch.Tensor) -> torch.Tensor:
normed = F.normalize(x.float(), dim=1).to(x.dtype)
return normed * self.scale * self.gamma + self.bias
class _ChunkedConv3d(nn.Conv3d):
"""``nn.Conv3d`` that convolves inputs larger than ``_MAX_CONV_NUMEL`` in overlapping
temporal chunks (plain ``F.conv3d`` silently corrupts most elements once the input exceeds
~2**31 elements: it appears to do 32-bit index arithmetic internally)."""
def forward(self, x: torch.Tensor) -> torch.Tensor:
num_parts = math.ceil(x.numel() / _MAX_CONV_NUMEL)
if num_parts <= 1:
return super().forward(x)
k, frames = self.kernel_size[0], x.size(2)
if self.padding[0] != 0:
raise ValueError(
f"temporal chunking needs an unpadded time axis, got kernel {self.kernel_size}, "
f"padding {self.padding}"
)
if self.stride[0] != 1:
# A strided conv can't reuse the stride-1 path's "prefix each chunk with the
# previous chunk's tail" trick below (a strided conv's output doesn't align with
# input chunk boundaries), so slide one kernel-width window at a time instead, only
# raising if even a single window would overflow.
stride = self.stride[0]
window_numel = x.shape[0] * x.shape[1] * k * x.shape[3] * x.shape[4]
if window_numel >= _MAX_CONV_NUMEL:
raise ValueError(
f"frames are too big for Conv3d even one stride-{stride} window at a time "
f"({window_numel} elements per window)"
)
# Bare super() doesn't resolve inside a list comprehension's own implicit scope,
# hence the loop.
window_outputs = []
for i in range(0, frames - k + 1, stride):
window_outputs.append(super().forward(x[:, :, i : i + k]))
return torch.cat(window_outputs, dim=2)
step = math.ceil(frames / num_parts)
last = frames - step * (math.ceil(frames / step) - 1)
if k > 1 and min(step, last) < k:
# A chunk would be shorter than the kernel: convolve one output frame at a time instead.
windows = [(i, i + k) for i in range(frames - k + 1)]
else:
# Each input chunk [start, end) is prefixed with the previous chunk's last k - 1 frames.
windows = [
(max(start - k + 1, 0), min(start + step, frames))
for start in range(0, frames, step)
]
out: torch.Tensor | None = None
for lo, hi in windows:
y = super().forward(x[:, :, lo:hi])
if out is None:
out = y.new_empty((*y.shape[:2], frames - k + 1, *y.shape[3:]))
out[:, :, lo : lo + y.size(2)] = y
assert out is not None
return out
class _SpatialChunkedConv3d(SpatialParallelConv3d, _ChunkedConv3d):
# exchange halos once before temporal chunking; uneven height shards may need
# different chunk counts, so collectives must stay outside that loop
@classmethod
def from_conv(cls, conv: _ChunkedConv3d, height_padding=None):
with torch.device("meta"):
parallel = cls(
conv.in_channels,
conv.out_channels,
conv.kernel_size,
stride=conv.stride,
padding=conv.padding,
dilation=conv.dilation,
groups=conv.groups,
bias=conv.bias is not None,
height_padding=height_padding,
)
parallel.weight, parallel.bias = conv.weight, conv.bias
return parallel
def _direct_forward(self, x):
x = F.pad(x, (0, 0, self.height_pad_top, self.height_pad_bottom, 0, 0))
return _ChunkedConv3d.forward(self, x)
class _CausalConv3d(nn.Module):
"""Conv3d that is causal in time: the first segment is left-padded with copies of its
first frame, later segments with the tail of the previous one; height / width are zero
padded."""
def __init__(
self,
chan_in: int,
chan_out: int,
kernel_size: int | tuple[int, int, int],
stride: tuple[int, int, int] = (1, 1, 1),
) -> None:
super().__init__()
if isinstance(kernel_size, int):
kernel_size = (kernel_size,) * 3
time_kernel, height_kernel, width_kernel = kernel_size
self.height_pad = height_kernel // 2
self.width_pad = width_kernel // 2
self.time_pad = time_kernel - 1
self.time_kernel = time_kernel
self.time_stride = stride[0]
self.conv = _ChunkedConv3d(chan_in, chan_out, kernel_size, stride=stride)
def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
s, k = self.time_stride, self.time_kernel
batch, _, frames, height, width = x.shape
x = F.pad(
x, (self.width_pad, self.width_pad, self.height_pad, self.height_pad, 0, 0)
)
if cache.first:
first_frame = x[:, :, :1]
padding = first_frame.expand(-1, -1, self.time_pad, -1, -1)
else:
padding = cache.padding[self]
# conv(cat([padding, x])) without materializing the concatenation of the whole segment:
# only the outputs whose window overlaps ``padding`` are computed on a small
# concatenated head.
out_frames = frames if s == 1 else (frames + 1) // 2
output = x.new_empty((batch, self.conv.out_channels, out_frames, height, width))
head_out = math.ceil(padding.size(2) / s)
head_in = head_out * s - padding.size(2)
if head_out > 0:
output[:, :, :head_out] = self.conv(
torch.cat([padding, x[:, :, : head_in + k - s]], dim=2)
)
if head_out < out_frames:
output[:, :, head_out:] = self.conv(x[:, :, head_in:])
# Keep exactly the input frames the next segment's first output window needs.
pad_offset = head_in + s * math.trunc((frames - head_in - k) / s) + s
if pad_offset < 0: # segment shorter than the kernel
cache.padding[self] = torch.cat([padding[:, :, pad_offset:], x], dim=2)
else:
cache.padding[self] = x[:, :, pad_offset:].clone()
return output
class _ResnetBlock3D(nn.Module):
"""norm -> SiLU -> causal conv, twice, plus a 1x1 shortcut when the width changes. Decoder
blocks use the latent-conditioned ``_SpatialNorm3D`` (``zq_channels`` set)."""
def __init__(
self, in_channels: int, out_channels: int, zq_channels: int | None = None
) -> None:
super().__init__()
if zq_channels is None:
self.norm1: nn.Module = _K6RMSNorm(in_channels, images=False)
self.norm2: nn.Module = _K6RMSNorm(out_channels, images=False)
else:
self.norm1 = _SpatialNorm3D(in_channels, zq_channels)
self.norm2 = _SpatialNorm3D(out_channels, zq_channels)
self.conv1 = _CausalConv3d(in_channels, out_channels, kernel_size=3)
self.conv2 = _CausalConv3d(out_channels, out_channels, kernel_size=3)
if in_channels != out_channels:
self.nin_shortcut = _ChunkedConv3d(in_channels, out_channels, kernel_size=1)
def _norm(
self,
norm: nn.Module,
h: torch.Tensor,
zq: torch.Tensor | None,
cache: _SegmentCache,
) -> torch.Tensor:
return norm(h) if zq is None else norm(h, zq, cache)
def forward(
self, x: torch.Tensor, cache: _SegmentCache, zq: torch.Tensor | None = None
) -> torch.Tensor:
h = F.silu(self._norm(self.norm1, x, zq, cache), inplace=True)
h = self.conv1(h, cache)
h = F.silu(self._norm(self.norm2, h, zq, cache), inplace=True)
h = self.conv2(h, cache)
if hasattr(self, "nin_shortcut"):
x = self.nin_shortcut(x)
return x + h
def _chunked_interpolate_nearest(
x: torch.Tensor, size: tuple[int, int, int], channels: int = 32
) -> torch.Tensor:
"""Nearest interpolation in channel chunks, bounding memory and int32 indexing."""
if x.shape[1] <= channels:
return F.interpolate(x, size=size, mode="nearest")
return torch.cat(
[
F.interpolate(chunk, size=size, mode="nearest")
for chunk in torch.split(x, channels, dim=1)
],
dim=1,
)
class _SpatialNorm3D(nn.Module):
"""RMS norm modulated by the latent: ``norm(f) * conv_y(zq) + conv_b(zq)`` with ``zq``
nearest-resized to ``f``."""
def __init__(self, f_channels: int, zq_channels: int) -> None:
super().__init__()
self.norm_layer = _K6RMSNorm(f_channels, images=False)
self.conv_y = _ChunkedConv3d(zq_channels, f_channels, kernel_size=1)
self.conv_b = _ChunkedConv3d(zq_channels, f_channels, kernel_size=1)
def forward(
self, f: torch.Tensor, zq: torch.Tensor, cache: _SegmentCache
) -> torch.Tensor:
frames, height, width = f.shape[-3:]
if cache.first:
# The causal first latent frame maps to exactly one pixel frame; the rest cover the
# remaining frames.
zq_first = _chunked_interpolate_nearest(zq[:, :, :1], (1, height, width))
if zq.size(2) > 1:
zq_rest = _chunked_interpolate_nearest(
zq[:, :, 1:], (frames - 1, height, width)
)
zq = torch.cat([zq_first, zq_rest], dim=2)
else:
zq = zq_first
else:
zq = _chunked_interpolate_nearest(zq, (frames, height, width))
norm_f = self.norm_layer(f)
norm_f.mul_(self.conv_y(zq))
norm_f.add_(self.conv_b(zq))
return norm_f
class _PXSDownsample(nn.Module):
"""x2 spatial (strided conv + channel-averaged pixel-unshuffle) and optional x2 causal
temporal (two causal convs + average pooling) downsample; doubles the channels."""
def __init__(self, in_channels: int, compress_time: bool) -> None:
super().__init__()
out_channels = 2 * in_channels
self.spatial_conv = _ChunkedConv3d(
in_channels,
out_channels,
kernel_size=(1, 3, 3),
stride=(1, 2, 2),
padding=(0, 1, 1),
)
self.compress_time = compress_time
if compress_time:
self.temporal_conv = nn.Sequential(
_CausalConv3d(out_channels, out_channels, kernel_size=(2, 1, 1)),
_CausalConv3d(
out_channels, out_channels, kernel_size=(2, 1, 1), stride=(2, 1, 1)
),
)
self.linear = _ChunkedConv3d(out_channels, out_channels, kernel_size=1)
def _spatial(self, x: torch.Tensor) -> torch.Tensor:
b, c, t, h, w = x.shape
pxs = F.pixel_unshuffle(x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w), 2)
pxs = pxs.view(b * t, 2 * c, 2, h // 2, w // 2).mean(dim=2)
pxs = pxs.reshape(b, t, 2 * c, h // 2, w // 2).permute(0, 2, 1, 3, 4)
return self.spatial_conv(x) + pxs
def _temporal(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
b, c, t, h, w = x.shape
pooled = x.permute(0, 3, 4, 1, 2).reshape(b * h * w, c, t)
if cache.first: # the causal first frame is kept as is
first, rest = pooled[..., :1], pooled[..., 1:]
pooled = (
torch.cat([first, F.avg_pool1d(rest, kernel_size=2, stride=2)], dim=-1)
if t > 1
else first
)
else:
pooled = F.avg_pool1d(pooled, kernel_size=2, stride=2)
pooled = pooled.reshape(b, h, w, c, -1).permute(0, 3, 4, 1, 2)
conv = self.temporal_conv[1](self.temporal_conv[0](x, cache), cache)
return conv + pooled
def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
out = self._spatial(x)
if self.compress_time:
out = self._temporal(out, cache)
return self.linear(out)
class _PXSUpsample(nn.Module):
"""Optional x2 causal temporal (frame repeat + causal conv residual) then x2 spatial
(nearest + conv residual) upsample."""
def __init__(self, in_channels: int, compress_time: bool) -> None:
super().__init__()
self.spatial_conv = _ChunkedConv3d(
in_channels, in_channels, kernel_size=(1, 3, 3), padding=(0, 1, 1)
)
self.compress_time = compress_time
if compress_time:
self.temporal_conv = _CausalConv3d(
in_channels, in_channels, kernel_size=(3, 1, 1)
)
self.linear = _ChunkedConv3d(in_channels, in_channels, kernel_size=1)
def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
if self.compress_time:
x = x.repeat_interleave(2, dim=2)
if cache.first: # the causal first frame is not duplicated
x = x[:, :, 1:]
x = self.temporal_conv(x, cache) + x
frames, height, width = x.shape[-3:]
# Chunked along channels: see _chunked_interpolate_nearest's docstring for the overflow
# this guards against (the old vendored code did not chunk this specific call).
x = _chunked_interpolate_nearest(x, (frames, height * 2, width * 2))
x.add_(self.spatial_conv(x))
return self.linear(x)
def _level_module(**children: nn.Module) -> nn.Module:
module = nn.Module()
for name, child in children.items():
setattr(module, name, child)
return module
class _Encoder3D(nn.Module):
def __init__(
self,
*,
ch: int,
ch_mult: list[float],
num_res_blocks: int,
in_channels: int,
z_channels: int,
temporal_compress_times: int,
temporal_compress_start_level: int,
) -> None:
super().__init__()
self.num_res_blocks = num_res_blocks
time_levels = range(
temporal_compress_start_level,
temporal_compress_start_level + int(math.log2(temporal_compress_times)),
)
self.conv_in = _CausalConv3d(in_channels, round(ch * ch_mult[0]), kernel_size=3)
self.down = nn.ModuleList()
block_in = round(ch * ch_mult[0])
for level, mult in enumerate(ch_mult):
block_out = round(ch * mult)
blocks = nn.ModuleList()
for _ in range(num_res_blocks):
blocks.append(_ResnetBlock3D(block_in, block_out))
block_in = block_out
down = _level_module(block=blocks)
if level != len(ch_mult) - 1:
down.downsample = _PXSDownsample(
block_in, compress_time=level in time_levels
)
block_in *= 2
self.down.append(down)
self.mid = _level_module(
block_1=_ResnetBlock3D(block_in, block_in),
block_2=_ResnetBlock3D(block_in, block_in),
)
self.norm_out = _K6RMSNorm(block_in, images=False)
self.conv_out = _CausalConv3d(block_in, 2 * z_channels, kernel_size=3)
def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
h = self.conv_in(x, cache)
for down in self.down:
for block in down.block:
h = block(h, cache)
if hasattr(down, "downsample"):
h = down.downsample(h, cache)
h = self.mid.block_2(self.mid.block_1(h, cache), cache)
return self.conv_out(_silu(self.norm_out(h)), cache)
class _Decoder3D(nn.Module):
def __init__(
self,
*,
ch: int,
ch_mult: list[float],
num_res_blocks: int,
out_ch: int,
z_channels: int,
temporal_compress_times: int,
temporal_compress_start_level: int,
) -> None:
super().__init__()
num_levels = len(ch_mult)
# Mirror of the encoder's temporally compressing levels.
time_levels = range(
num_levels
- temporal_compress_start_level
- int(math.log2(temporal_compress_times)),
num_levels - temporal_compress_start_level,
)
block_in = round(ch * ch_mult[-1])
self.conv_in = _CausalConv3d(z_channels, block_in, kernel_size=3)
self.mid = _level_module(
block_1=_ResnetBlock3D(block_in, block_in, z_channels),
block_2=_ResnetBlock3D(block_in, block_in, z_channels),
)
up_levels: list[nn.Module] = []
for level in reversed(range(num_levels)):
block_out = round(ch * ch_mult[level])
blocks = nn.ModuleList()
for _ in range(num_res_blocks + 1):
blocks.append(_ResnetBlock3D(block_in, block_out, z_channels))
block_in = block_out
up = _level_module(block=blocks)
if level != 0:
up.upsample = _PXSUpsample(block_in, compress_time=level in time_levels)
up_levels.insert(0, up)
self.up = nn.ModuleList(up_levels)
self.norm_out = _SpatialNorm3D(block_in, z_channels)
self.conv_out = _CausalConv3d(block_in, out_ch, kernel_size=3)
def forward(self, z: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
h = self.conv_in(z, cache)
h = self.mid.block_2(self.mid.block_1(h, cache, z), cache, z)
for up in reversed(self.up):
for block in up.block:
h = block(h, cache, z)
if hasattr(up, "upsample"):
h = up.upsample(h, cache)
return self.conv_out(_silu(self.norm_out(h, z, cache)), cache)
# The structural knobs every Encoder3D / Decoder3D constructor kwarg comes from; everything
# else in a checkpoint's encoder_config / decoder_config (``checkpoint_list``, ``resolution``,
# the placeholder ``"None"`` channel field, ...) is ignored, the same way the old vendored
# module silently dropped unknown keys via its own ``**ignore_kwargs`` / unused params.
_ARCH_KEYS = (
"ch",
"ch_mult",
"num_res_blocks",
"z_channels",
"temporal_compress_times",
)
def _arch_kwargs(conf: dict[str, Any], extra: str, which: str) -> dict[str, Any]:
# norm_type is the one legacy knob this port still checks: unlike padding_mode /
# downsample_version / fix_pxs / double_z (which the new Encoder3D / Decoder3D no longer
# branch on at all -- they always run the "zeros" / v2-pixel-unshuffle / fixed-stride /
# double-z behavior those knobs used to select), a group_norm checkpoint would silently
# get rms_norm layers instead of a clear error, which is the one mismatch worth failing
# loudly on here.
norm_type = conf.get("norm_type", "rms_norm")
if norm_type != "rms_norm":
raise ValueError(
f"Kandinsky6SRVAE supports only {which}_config['norm_type'] == 'rms_norm' "
f"(group-norm KVAEs are not implemented by this port), got {norm_type!r}"
)
missing = [key for key in (*_ARCH_KEYS, extra) if key not in conf]
if missing:
raise ValueError(f"Kandinsky6SRVAE {which}_config is missing {missing}")
kwargs = {key: conf[key] for key in (*_ARCH_KEYS, extra)}
kwargs["temporal_compress_start_level"] = conf.get(
"temporal_compress_start_level", 0
)
return kwargs
class Kandinsky6SRVAE(nn.Module, LayerwiseOffloadableModuleMixin):
"""Segment-cached causal video KVAE used by the SR pipeline."""
layerwise_offload_dit_group_enabled = False
def __init__(self, config: Kandinsky6SRVAEConfig) -> None:
super().__init__()
self.config = config
arch = config.arch_config
assert isinstance(arch, Kandinsky6SRVAEArchConfig)
if not arch.encoder_config or not arch.decoder_config:
raise ValueError(
"Kandinsky6SRVAE needs `encoder_config` and `decoder_config` (KVAE "
"architecture) in the component config.json."
)
enc = _arch_kwargs(dict(arch.encoder_config), "in_channels", "encoder")
dec = _arch_kwargs(dict(arch.decoder_config), "out_ch", "decoder")
self.temporal_compression = enc["temporal_compress_times"]
self.encoder = _Encoder3D(**enc)
self.decoder = _Decoder3D(**dec)
self._spatial_parallel_decode_enabled = (
can_install_spatial_shard_parallel_decode(config)
)
if self._spatial_parallel_decode_enabled:
for module in self.decoder.modules():
if isinstance(module, _CausalConv3d) and module.height_pad:
module.conv = _SpatialChunkedConv3d.from_conv(
module.conv, (module.height_pad, module.height_pad)
)
module.height_pad = 0
elif isinstance(module, _PXSUpsample):
module.spatial_conv = _SpatialChunkedConv3d.from_conv(
module.spatial_conv
)
# encode/decode invoke each block, not the surrounding level container
self.layer_names = [
f"{path}.{index}.block"
for path, levels in (
("encoder.down", self.encoder.down),
("decoder.up", self.decoder.up),
)
for index in range(len(levels))
]
@property
def scaling_factor(self) -> float:
return float(self.config.arch_config.scaling_factor)
@property
def spatial_factor(self) -> int:
return int(self.config.arch_config.spatial_factor)
@property
def temporal_factor(self) -> int:
return int(self.config.arch_config.temporal_factor)
@staticmethod
def normalize_data(data: torch.Tensor) -> torch.Tensor:
return data / 128 - 1.0
@staticmethod
def denormalize_data(data: torch.Tensor) -> torch.Tensor:
return (data + 1) * 128
def encode(self, x: torch.Tensor) -> tuple[torch.Tensor, list[int]]:
"""``[B, C, T, H, W]`` normalized pixels -> ``(raw latent mean, pixel-frame segment sizes)``."""
split_list = [_SEGMENT_FRAMES + 1]
remaining = x.size(2) - split_list[0]
while remaining > 0:
split_list.append(_SEGMENT_FRAMES)
remaining -= _SEGMENT_FRAMES
split_list[-1] += remaining
cache = _SegmentCache()
latents = []
for segment in torch.split(x, split_list, dim=2):
moments = self.encoder(segment, cache)
cache.first = False
latents.append(
moments.chunk(2, dim=1)[0]
) # deterministic posterior: the mean
return torch.cat(latents, dim=2), split_list
def decode(self, z: torch.Tensor) -> DecoderOutput:
"""``[B, C, T', h, w]`` raw latent -> ``.sample`` ``[B, 3, T, H, W]`` in the normalized pixel range."""
if self._spatial_parallel_decode_enabled:
world_size = get_decode_parallel_world_size()
if (
should_run_spatial_shard_parallel_decode(self.config, z)
and z.shape[-2] >= world_size
):
z, height = split_height_for_parallel_decode(
z,
expected_height=z.shape[-2] * self.spatial_factor,
world_size=world_size,
rank=get_decode_parallel_rank(),
)
return DecoderOutput(
sample=gather_and_trim_height(self._decode(z), height)
)
with disable_spatial_parallel_decode():
return DecoderOutput(sample=self._decode(z))
return DecoderOutput(sample=self._decode(z))
def _decode(self, z: torch.Tensor) -> torch.Tensor:
segment = _SEGMENT_FRAMES // self.temporal_compression
num_frames = z.size(2)
if num_frames == 1:
split_list = [1]
else:
split_list = [segment] * ((num_frames - 1) // segment)
if (num_frames - 1) % segment:
split_list.append((num_frames - 1) % segment)
split_list[0] += 1
cache = _SegmentCache()
samples = []
for chunk in torch.split(z, split_list, dim=2):
samples.append(self.decoder(chunk, cache))
cache.first = False
return torch.cat(samples, dim=2)
EntryClass = Kandinsky6SRVAE
__all__ = ["Kandinsky6SRVAE"]
@@ -3,13 +3,19 @@
# Licensed under the MIT license.
# Adapted from https://github.com/jik876/hifi-gan under the MIT license.
#
# The alias-free resampling primitives (LowPassFilter1d/UpSample1d/
# DownSample1d/Activation1d) are architecture-agnostic DSP building blocks
# shared with the sibling ``kandinsky6_audio_vae/bigvgan.py`` -- both import
# ``Activation1d`` from ``runtime/models/vaes/alias_free.py`` (sibling of
# this package, alongside ``common.py``) instead of each keeping its own copy.
import torch
import torch.nn as nn
from torch.nn import Conv1d, ConvTranspose1d, Parameter
from torch.nn.utils.parametrizations import weight_norm
from .alias_free import Activation1d
from sglang.multimodal_gen.runtime.models.vaes.alias_free import Activation1d
def get_padding(kernel_size, dilation=1):
@@ -0,0 +1,122 @@
# SPDX-License-Identifier: Apache-2.0
"""Joint video/audio generation with optional image conditioning.
One stage chain handles both T2VA and IT2VA; the image stage is a no-op for T2VA."""
from __future__ import annotations
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
Kandinsky6TI2VAPipelineConfig,
)
from sglang.multimodal_gen.configs.sample.kandinsky6 import (
Kandinsky6TI2VASamplingParams,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
ComposedPipelineBase,
)
from sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline import LoRAPipeline
from sglang.multimodal_gen.runtime.pipelines_core.stages import InputValidationStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6 import (
Kandinsky6AudioDecodingStage,
Kandinsky6DecodingStage,
Kandinsky6DenoisingStage,
Kandinsky6ImageEncodingStage,
Kandinsky6LatentPreparationStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
TextEncodingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation import (
TimestepPreparationStage,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
class Kandinsky6TI2VAPipeline(LoRAPipeline, ComposedPipelineBase):
pipeline_name = "Kandinsky6TI2VAPipeline"
is_video_pipeline = True
pipeline_config_cls = Kandinsky6TI2VAPipelineConfig
sampling_params_cls = Kandinsky6TI2VASamplingParams
# Kandinsky6AudioVAE bundles the mel-VAE decoder *and* the BigVGAN-v2
# vocoder in one checkpoint component -- there is no separate "vocoder"
# module, unlike LTX-2's separately-loaded audio_vae/vocoder pair.
_required_config_modules = [
"scheduler",
"text_encoder",
"text_encoder_2",
"tokenizer",
"tokenizer_2",
"transformer",
"vae",
"audio_vae",
]
def validate_disagg_role(self, role: RoleType) -> None:
# coupled denoising and reference-frame state are not yet disaggregation-aware
if role != RoleType.MONOLITHIC:
raise ValueError(
"Kandinsky6TI2VAPipeline only supports monolithic deployment; "
f"disaggregation role {role.value!r} is not supported"
)
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
self.add_stage(
stage_name="input_validation_stage", stage=InputValidationStage()
)
self.add_stage(
stage_name="text_encoding_stage",
stage=TextEncodingStage(
text_encoders=[
self.get_module("text_encoder"),
self.get_module("text_encoder_2"),
],
tokenizers=[
self.get_module("tokenizer"),
self.get_module("tokenizer_2"),
],
),
)
self.add_stage(
stage_name="timestep_preparation_stage",
stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")),
)
self.add_stage(
stage_name="latent_preparation_stage",
stage=Kandinsky6LatentPreparationStage(),
)
# No-op unless the request supplies a conditioning image; see
# Kandinsky6ImageEncodingStage's docstring.
self.add_stage(
stage_name="image_encoding_stage",
stage=Kandinsky6ImageEncodingStage(vae=self.get_module("vae")),
)
self.add_stage(
stage_name="denoising_stage",
stage=Kandinsky6DenoisingStage(
transformer=self.get_module("transformer"),
scheduler=self.get_module("scheduler"),
pipeline=self,
),
)
# Audio decodes before video, matching the diffusers reference and
# FastVideo's own stage order -- do not reorder.
self.add_stage(
stage_name="audio_decoding_stage",
stage=Kandinsky6AudioDecodingStage(audio_vae=self.get_module("audio_vae")),
)
self.add_stage(
stage_name="decoding_stage",
stage=Kandinsky6DecodingStage(vae=self.get_module("vae"), pipeline=self),
)
EntryClass = [Kandinsky6TI2VAPipeline]
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky 6 video SR: prepare, encode/upscale, denoise, decode and blend.
Work is grouped by component to avoid weight transfers per tile. The bundle's
scheduler drives denoising through the shared DenoisingStage lifecycle.
num_inference_steps counts DiT calls; pi-Flow uses its checkpoint-defined nfe."""
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
ComposedPipelineBase,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
Kandinsky6SRDecodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
Kandinsky6SRDenoisingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import (
Kandinsky6SREncodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
Kandinsky6SRInputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
Kandinsky6SRLatentPrepStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import (
Kandinsky6SROutputStage,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
class Kandinsky6SRPipeline(ComposedPipelineBase):
pipeline_name = "Kandinsky6SRPipeline"
is_video_pipeline = True
pipeline_config_cls = Kandinsky6SRPipelineConfig
sampling_params_cls = Kandinsky6SRSamplingParams
_required_config_modules = ["transformer", "vae", "scheduler", "latent_upscaler"]
# Repos without a latent-upscaler bank run the pixel path only.
_optional_config_modules = ("latent_upscaler",)
def validate_disagg_role(self, role: RoleType) -> None:
# The tiled loop keeps tile latents / decoded tiles in one process.
if role != RoleType.MONOLITHIC:
raise ValueError(
"Kandinsky6SRPipeline only supports monolithic deployment; "
f"disaggregation role {role.value!r} is not supported"
)
def load_modules(self, server_args: ServerArgs, loaded_modules=None):
model_index = self._load_config()
if "dit" in model_index and "transformer" not in model_index:
raise ValueError(
f"{self.model_path} is a k6_video SR bundle (it has a 'dit' component), "
"not an official Diffusers Kandinsky6SRPipeline repo. Use "
"kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers or "
"kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers"
)
for name in self._optional_config_modules:
if model_index.get(name) is None and name in self._required_config_modules:
self._required_config_modules.remove(name)
model_index.pop(name, None)
return super().load_modules(server_args, loaded_modules)
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
latent_upscaler = self.get_module("latent_upscaler")
vae = self.get_module("vae")
transformer = self.get_module("transformer")
scheduler = self.get_module("scheduler")
self.add_stage(stage_name="input_stage", stage=Kandinsky6SRInputStage())
self.add_stage(
stage_name="encode_stage",
stage=Kandinsky6SREncodeStage(vae=vae, latent_upscaler=latent_upscaler),
)
self.add_stage(
stage_name="latent_prep_stage",
stage=Kandinsky6SRLatentPrepStage(
vae=vae,
transformer=transformer,
latent_upscaler=latent_upscaler,
scheduler=scheduler,
),
)
self.add_stage(
stage_name="denoising_stage",
stage=Kandinsky6SRDenoisingStage(
transformer=transformer, scheduler=scheduler, pipeline=self
),
)
self.add_stage(
stage_name="decode_stage", stage=Kandinsky6SRDecodeStage(vae=vae)
)
self.add_stage(stage_name="output_stage", stage=Kandinsky6SROutputStage())
EntryClass = [Kandinsky6SRPipeline]
@@ -488,6 +488,12 @@ class OutputBatch:
raw_frame_metadata: dict[str, Any] | None = None
audio: torch.Tensor | None = None
audio_sample_rate: int | None = None
# The effective fps the worker actually produced ``output`` at (e.g. a model whose input
# stage resamples / resolves the request's nominal fps to a source-derived one, such as
# Kandinsky6 SR). ``None`` means the worker did not resolve its own fps; callers that save
# ``output`` client-side (return_file_paths_only=False) then fall back to the originating
# request's own fps field, as before this field existed.
fps: int | None = None
action_pred: torch.Tensor | None = None
action_mode: str | None = None
action_domain_id: int | None = None
@@ -1035,8 +1035,16 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
self._cache_dit_request_overrides = resolve_cache_dit_request_overrides(
batch.sampling_params.cache_dit_params
)
has_separate_cfg = (
requested
and batch.do_classifier_free_guidance
and not self.server_args.enable_cfg_parallel
)
desired_key = (
cache_dit_overrides_key(self._cache_dit_request_overrides)
(
cache_dit_overrides_key(self._cache_dit_request_overrides),
has_separate_cfg,
)
if requested
else None
)
@@ -1063,11 +1071,11 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
steps_computation_policy=scm_policy,
)
else:
scm_preset = None if scm_preset == "none" else scm_preset
refresh_context_on_transformer(
self.transformer,
primary_num_steps,
scm_preset=scm_preset,
steps_computation_mask=steps_computation_mask,
steps_computation_policy=scm_policy,
)
return
@@ -1156,7 +1164,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
model_name="transformer",
sp_group=sp_group,
tp_group=tp_group,
has_separate_cfg=batch.do_classifier_free_guidance,
has_separate_cfg=has_separate_cfg,
)
logger.info(
"cache-dit enabled on transformer (steps=%d, Fn=%d, Bn=%d, rdt=%.3f)",
@@ -0,0 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky6-specific pipeline stages."""
from .decoding import Kandinsky6AudioDecodingStage, Kandinsky6DecodingStage
from .denoising import Kandinsky6DenoisingStage
from .image_encoding import Kandinsky6ImageEncodingStage
from .latent_preparation import Kandinsky6LatentPreparationStage
__all__ = [
"Kandinsky6AudioDecodingStage",
"Kandinsky6DecodingStage",
"Kandinsky6DenoisingStage",
"Kandinsky6ImageEncodingStage",
"Kandinsky6LatentPreparationStage",
]
@@ -0,0 +1,94 @@
# SPDX-License-Identifier: Apache-2.0
"""Decode audio before video, preserving the reference component order."""
from __future__ import annotations
import torch
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
StageValidators as V,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
VerificationResult,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
class Kandinsky6DecodingStage(DecodingStage):
"""Convert video to channel-first for shared VAE decode; attach the decoded audio."""
def __init__(self, vae, pipeline=None) -> None:
super().__init__(vae=vae, pipeline=pipeline)
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
if batch.latents is None:
raise ValueError("latents must be available before Kandinsky6 decoding.")
batch.latents = batch.latents.permute(0, 4, 1, 2, 3).contiguous()
output_batch = super().forward(batch, server_args)
output_batch.audio = batch.audio
output_batch.audio_sample_rate = batch.audio_sample_rate
return output_batch
class Kandinsky6AudioDecodingStage(PipelineStage):
"""Decode audio through the bundled mel VAE/vocoder into batch.audio."""
def __init__(self, audio_vae) -> None:
super().__init__()
self.audio_vae = audio_vae
@property
def role_affinity(self) -> RoleType:
return RoleType.DECODER
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
stage_name = self._component_stage_name(stage_name)
return [ComponentUse(stage_name, "audio_vae")]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
if batch.audio_latents is None:
return batch
device = get_local_torch_device()
with self.use_declared_component(
component_name="audio_vae", module=self.audio_vae
) as audio_vae:
assert audio_vae is not None
self.audio_vae = audio_vae
# Cast to the module's own loaded weight dtype directly -- no
# autocast / precision-config knob, matching the existing
# MMAudio-lineage audio decoding stages in this codebase.
decoder_dtype = next(audio_vae.parameters()).dtype
# apply checkpoint latent denormalization before the VAE's mel denormalization
scaling_factor = audio_vae.scaling_factor
mean_value = audio_vae.mean_value
latents = (batch.audio_latents / scaling_factor) + mean_value
# [B, A, D] -> [B, D, A] to match the audio VAE's 1D-conv
# (channel, length) convention.
latents = latents.to(device=device, dtype=decoder_dtype).transpose(1, 2)
waveform = audio_vae.wrapped_decode(latents) # [B, 1, samples]
# preserve [B, samples] so the save path selects the audio track for each output
batch.audio = waveform[:, 0].clamp(-1.0, 1.0).float().cpu()
batch.audio_sample_rate = int(server_args.pipeline_config.audio_sample_rate)
return batch
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("audio_latents", batch.audio_latents, V.none_or_tensor)
return result
@@ -0,0 +1,442 @@
# SPDX-License-Identifier: Apache-2.0
"""Joint video/audio denoising using the shared stage lifecycle, BCG and CFG.
Flow-Euler advances the video scheduler once per step; audio uses the same
sigma delta without advancing that scheduler again. pi-Flow uses independent
video/audio schedulers and retains fp32 state between bf16 DiT calls.
Adapted from FastVideo's Kandinsky6DenoisingStage."""
from __future__ import annotations
import dataclasses
import math
from copy import deepcopy
from typing import Any
import torch
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.distributed.cfg_parallel_utils import (
run_cfg_parallel,
run_two_branch_cfg_parallel,
)
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGBranch, CFGPolicy
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_classifier_free_guidance_world_size,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import (
TAIL_COND_ACTIVE_EXTRA_KEY,
VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
StageValidators as V,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
VerificationResult,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYPE
class Kandinsky6DenoisingStage(DenoisingStage):
"""Run the Kandinsky6 joint video+audio denoising loop."""
def __init__(self, transformer, scheduler, pipeline=None) -> None:
super().__init__(
transformer=transformer, scheduler=scheduler, pipeline=pipeline
)
def _owns_compile_warmup_lifecycle(self) -> bool:
# forward() overrides DenoisingStage.forward (the joint video+audio
# loop cannot reuse the parent's single-tensor _denoise()), but it
# still wraps the loop in `_offload_for_torch_compile_warmup` itself
# below -- same opt-in as MiniMaxH3DenoisingStage, the codebase's
# other joint-modality denoising stage.
return True
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
stage_name = self._component_stage_name(stage_name)
return [
ComponentUse(
stage_name,
"transformer",
phase="transformer",
preferred_ready_after_request=True,
memory_intensive=True,
)
]
@staticmethod
def _text_rope_pos(mask: torch.Tensor, device: torch.device) -> torch.Tensor:
seq_len = int(mask.sum(1).max().item())
return torch.arange(seq_len, device=device)
def _call_transformer(self, transformer, **call_kwargs) -> Any:
"""Dispatch through BCG when enabled, otherwise call the transformer directly."""
runner = self._maybe_get_bcg_runner(transformer)
if runner is not None:
return self._bcg_run(runner, call_kwargs, transformer)
return transformer(**call_kwargs)
def _build_cfg_policy(
self,
batch: Req,
server_args: ServerArgs,
*,
prompt_embeds: torch.Tensor,
pooled: torch.Tensor,
text_rope_pos: torch.Tensor,
) -> CFGPolicy:
"""Reuse the pipeline CFG policy with joint-modality conditioning kwargs."""
branches = [
CFGBranch(
"conditional",
True,
{
"encoder_hidden_states": prompt_embeds,
"pooled_projections": pooled,
"text_rope_pos": text_rope_pos,
},
)
]
if batch.do_classifier_free_guidance and batch.negative_prompt_embeds:
neg_prompt_embeds = server_args.pipeline_config.get_neg_prompt_embeds(
batch
).to(device=prompt_embeds.device, dtype=prompt_embeds.dtype)
if not batch.neg_pooled_embeds:
raise ValueError(
"Kandinsky6 requires CLIP negative pooled projections for CFG."
)
neg_pooled = batch.neg_pooled_embeds[0].to(
device=prompt_embeds.device, dtype=prompt_embeds.dtype
)
if not batch.negative_attention_mask:
raise ValueError(
"Kandinsky6 requires Qwen (Reason1) negative attention masks for CFG."
)
negative_text_rope_pos = self._text_rope_pos(
batch.negative_attention_mask[0].to(prompt_embeds.device),
prompt_embeds.device,
)
branches.append(
CFGBranch(
"unconditional",
False,
{
"encoder_hidden_states": neg_prompt_embeds,
"pooled_projections": neg_pooled,
"text_rope_pos": negative_text_rope_pos,
},
)
)
return dataclasses.replace(
server_args.pipeline_config.cfg_policy, branches=branches
)
def _predict_joint_velocity(
self,
*,
transformer,
cfg_policy: CFGPolicy,
batch: Req,
server_args: ServerArgs,
step_index: int,
video_input: torch.Tensor,
audio_input: torch.Tensor,
t_expand: torch.Tensor,
visual_rope_pos: list[torch.Tensor],
scale_factor: tuple[float, ...],
sparse_params: dict[str, Any] | None,
visual_token_type_ids: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Run sequential or distributed CFG branches and combine video/audio velocity."""
def predict_fn(branch: CFGBranch) -> tuple[torch.Tensor, torch.Tensor]:
branch.configure_batch(batch)
with set_forward_context(
current_timestep=step_index, attn_metadata=None, forward_batch=batch
):
model_output = self._call_transformer(
transformer,
hidden_states=video_input,
hidden_states_audio=audio_input,
timestep=t_expand,
visual_rope_pos=visual_rope_pos,
scale_factor=scale_factor,
sparse_params=sparse_params,
visual_token_type_ids=visual_token_type_ids,
# BCG clones tuple leaves so the next CFG replay cannot
# overwrite the previous branch's predictions
return_dict=False,
**branch.kwargs,
)
return model_output
cfg_scale = server_args.pipeline_config.get_classifier_free_guidance_scale(
batch, batch.guidance_scale
)
if server_args.enable_cfg_parallel:
if (
len(cfg_policy.branches) == 2
and get_classifier_free_guidance_world_size() == 2
and not cfg_policy.parallel_uses_serial_arithmetic
):
video_vel, audio_vel = run_two_branch_cfg_parallel(
cfg_policy,
predict_fn,
cfg_scale,
batch,
server_args.pipeline_config,
)
return video_vel, audio_vel
predictions = run_cfg_parallel(cfg_policy, predict_fn)
else:
predictions = [predict_fn(branch) for branch in cfg_policy.branches]
video_vel, audio_vel = cfg_policy.combine(
predictions,
batch,
cfg_scale,
server_args.pipeline_config,
cfg_parallel=server_args.enable_cfg_parallel,
)
return video_vel, audio_vel
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
with self._offload_for_torch_compile_warmup(batch):
return self._run_joint_denoise_loop(batch, server_args)
def _run_joint_denoise_loop(self, batch: Req, server_args: ServerArgs) -> Req:
if batch.timesteps is None:
raise ValueError("timesteps must be prepared before Kandinsky6 denoising.")
if batch.latents is None or batch.audio_latents is None:
raise ValueError(
"video and audio latents must be prepared before Kandinsky6 denoising."
)
scheduler = batch.scheduler
if scheduler is None:
raise ValueError("scheduler must be set for Kandinsky6 denoising.")
use_piflow = bool(getattr(scheduler, "is_piflow", False))
if use_piflow and (
not math.isfinite(batch.guidance_scale)
or abs(batch.guidance_scale - 1.0) > 1e-6
):
raise ValueError("Kandinsky6 PiFlow requires guidance_scale=1.0.")
audio_scheduler = deepcopy(scheduler) if use_piflow else None
pipeline_config = server_args.pipeline_config
arch = pipeline_config.dit_config.arch_config
# The runtime transformer already rejects attention_engine="nabla"
# at construction time (NABLA sparse attention is not yet ported),
# so sparse_params stays None on every currently-reachable
# configuration. A future NABLA-capable DiT would fill this in.
if arch.attention_engine == "nabla":
raise NotImplementedError(
"Kandinsky6 NABLA sparse-attention metadata construction is not implemented; "
"the runtime transformer also rejects attention_engine='nabla' at construction "
"time. Use attention_engine='auto' or 'sdpa'."
)
sparse_params = None
device = get_local_torch_device()
target_dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision]
# .clone(): batch.latents was created by Kandinsky6LatentPreparationStage
# under an active torch.inference_mode() context further up the
# pipeline, which marks it as an "inference tensor" -- such tensors
# can only be mutated in place while still inside an inference_mode
# context. This stage runs under plain @torch.no_grad() instead, and
# mutates `video` in place every denoising step (video[..., :num_channels]
# = ...), so clone once here to get an ordinary, freely-mutable tensor
# before the loop starts. audio is never mutated in place (only
# reassigned via out-of-place `+`), so it needs no clone.
video = batch.latents.clone()
audio = batch.audio_latents
num_channels = int(arch.in_visual_dim)
tail_cond = bool(batch.extra.get(TAIL_COND_ACTIVE_EXTRA_KEY, False))
visual_token_type_ids = batch.extra.get(VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY)
prompt_embeds = pipeline_config.get_pos_prompt_embeds(batch).to(
device=device, dtype=target_dtype
)
if not batch.pooled_embeds:
raise ValueError("Kandinsky6 requires CLIP pooled projections.")
pooled = batch.pooled_embeds[0].to(device=device, dtype=target_dtype)
if not batch.prompt_attention_mask:
raise ValueError(
"Kandinsky6 requires Qwen (Reason1) prompt attention masks."
)
text_rope_pos = self._text_rope_pos(
batch.prompt_attention_mask[0].to(device), device
)
cfg_policy = self._build_cfg_policy(
batch,
server_args,
prompt_embeds=prompt_embeds,
pooled=pooled,
text_rope_pos=text_rope_pos,
)
height = int(batch.height)
width = int(batch.width)
spatial_ratio = pipeline_config.vae_config.arch_config.spatial_compression_ratio
patch_size = arch.patch_size
# Constant per-request geometry -- computed once outside the loop,
# not per-step.
num_video_frames = video.shape[1] - (1 if tail_cond else 0)
t_positions = torch.arange(num_video_frames, device=device)
if tail_cond:
# The appended reference frame reuses T-position 0's rope row
# (RoPE3D is a pure position -> table lookup, so duplicating the
# first row is equivalent to the diffusers reference's
# ``torch.cat([rope, rope[:1]])``).
t_positions = torch.cat([t_positions, t_positions.new_zeros(1)])
visual_rope_pos = [
t_positions,
torch.arange(height // spatial_ratio // patch_size[1], device=device),
torch.arange(width // spatial_ratio // patch_size[2], device=device),
]
# Fixed per-checkpoint RoPE frequency scaling read from the DiT's
# arch config (transformer/config.json's "scale_factor", both real
# Pro checkpoints ship [1.0, 2.0, 2.0]) -- matches the diffusers
# reference, which resolves this once in
# ``Kandinsky6TI2VAPipeline.__init__`` and reuses it for every
# request regardless of the request's own height/width. NOT a
# function of the request's resolution.
scale_factor = arch.scale_factor
image_latent = batch.image_latent
total_steps = int(batch.timesteps.shape[0])
self._maybe_enable_cache_dit_and_torch_compile(total_steps, batch)
# PiFlow's noisy video state is tracked in fp32 across steps here,
# separately from the `video` buffer's bf16 storage dtype -- see the
# module docstring. Always a Tensor (not `Tensor | None`) even on the
# flow-Euler path, where it is simply unused: the initial cast is one
# cheap op, and keeping it non-Optional avoids re-narrowing it after
# every `if use_piflow:` branch re-entry in the loop below.
video_state = video[..., :num_channels].to(torch.float32)
with self.use_declared_component(
component_name="transformer", module=self.transformer
) as transformer:
assert transformer is not None
self.transformer = transformer
with self.progress_bar(
total=total_steps, batch=batch, desc="Kandinsky6 Denoising"
) as progress_bar:
for i, timestep in enumerate(batch.timesteps):
t_expand = (
timestep.unsqueeze(0)
.repeat(video.shape[0])
.to(device=device, dtype=target_dtype)
)
if use_piflow:
video_input = video_state.to(dtype=target_dtype)
if video.shape[-1] > num_channels:
video_input = torch.cat(
[
video_input,
video[..., num_channels:].to(dtype=target_dtype),
],
dim=-1,
)
else:
video_input = video.to(dtype=target_dtype)
video_vel, audio_vel = self._predict_joint_velocity(
transformer=transformer,
cfg_policy=cfg_policy,
batch=batch,
server_args=server_args,
step_index=i,
video_input=video_input,
audio_input=audio.to(dtype=target_dtype),
t_expand=t_expand,
visual_rope_pos=visual_rope_pos,
scale_factor=scale_factor,
sparse_params=sparse_params,
visual_token_type_ids=visual_token_type_ids,
)
# scheduler.step() is called exactly once per iteration,
# against video only -- it owns the internal step index.
if use_piflow:
video_state = scheduler.step(
video_vel, timestep, video_state, return_dict=False
)[0]
video[..., :num_channels] = video_state.to(video.dtype)
else:
video[..., :num_channels] = scheduler.step(
video_vel,
timestep,
video[..., :num_channels],
return_dict=False,
)[0]
if tail_cond:
ref_frame = image_latent.to(
device=video.device, dtype=video.dtype
)
video[:, -1:, :, :, :num_channels] = ref_frame
if use_piflow:
video_state[:, -1:] = ref_frame.to(torch.float32)
# Manual Euler update for audio, using the SAME per-step
# sigma delta the scheduler.step() call above just
# consumed for video.
if use_piflow:
audio = audio_scheduler.step(
audio_vel, timestep, audio, return_dict=False
)[0]
else:
step_size = scheduler.sigmas[i + 1] - scheduler.sigmas[i]
audio = (
audio
+ step_size.to(device=audio.device, dtype=audio.dtype)
* audio_vel
)
if progress_bar is not None:
progress_bar.update()
video = video[..., :num_channels]
if tail_cond:
video = video[:, :-1]
batch.latents = video
batch.audio_latents = audio
return batch
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
result.add_check(
"audio_latents", batch.audio_latents, [V.is_tensor, V.with_dims(3)]
)
result.add_check("prompt_embeds", batch.prompt_embeds, V.list_not_empty)
result.add_check("timesteps", batch.timesteps, [V.is_tensor, V.with_dims(1)])
return result
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
result.add_check(
"audio_latents", batch.audio_latents, [V.is_tensor, V.with_dims(3)]
)
return result
@@ -0,0 +1,214 @@
# SPDX-License-Identifier: Apache-2.0
"""Optional image conditioning, adapted from FastVideo's Kandinsky6ImageEncodingStage.
Append a clean reference latent frame and mark its token type. Denoising re-pins
that frame every step and removes it before decoding."""
from __future__ import annotations
import PIL.Image
import torch
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
StageValidators as V,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
VerificationResult,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.condition_expansion import (
PromptToSampleBatchExpander,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
from sglang.multimodal_gen.runtime.utils.precision import (
autocast_context,
autocast_enabled,
resolve_precision,
temporary_module_dtype,
)
from sglang.multimodal_gen.runtime.utils.vision import (
normalize,
numpy_to_pt,
pil_to_numpy,
)
logger = init_logger(__name__)
# model-specific reference-frame state shared through Req.extra
TAIL_COND_ACTIVE_EXTRA_KEY = "kandinsky6_tail_cond_active"
VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY = "kandinsky6_visual_token_type_ids"
class Kandinsky6ImageEncodingStage(PipelineStage):
"""Optional IT2VA conditioning: no-op for a pure T2VA call.
See module docstring for the tail-cond scheme.
"""
def __init__(self, vae: ParallelTiledVAE) -> None:
super().__init__()
self.vae = vae
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
stage_name = self._component_stage_name(stage_name)
return [ComponentUse(stage_name, "vae", target_dtype=vae_dtype)]
@staticmethod
def _cover_resize_dims(
src_h: int, src_w: int, height: int, width: int
) -> tuple[int, int]:
"""Resize to cover (height, width), preserving aspect before the centre crop."""
scale = min(src_h / height, src_w / width)
# clamp float round-trip undershoot on the constraining axis before centre crop
new_h = max(height, int(src_h / scale))
new_w = max(width, int(src_w / scale))
return new_h, new_w
@classmethod
def _preprocess(cls, image, height: int, width: int) -> torch.Tensor:
if isinstance(image, PIL.Image.Image):
src_w, src_h = image.size
new_h, new_w = cls._cover_resize_dims(src_h, src_w, height, width)
image = image.resize((new_w, new_h), resample=PIL.Image.BILINEAR)
top, left = (new_h - height) // 2, (new_w - width) // 2
image = image.crop((left, top, left + width, top + height))
image = numpy_to_pt(pil_to_numpy(image))
return normalize(image)
if image.min() >= 0:
logger.warning(
"Kandinsky6 conditioning image tensor has no negative values; "
"assuming range [0, 1] and normalizing to [-1, 1]. Pass a "
"[-1, 1] tensor with negative values to skip normalization."
)
image = normalize(image)
if image.ndim == 3:
image = image.unsqueeze(0)
src_h, src_w = image.shape[-2], image.shape[-1]
if (src_h, src_w) != (height, width):
new_h, new_w = cls._cover_resize_dims(src_h, src_w, height, width)
image = torch.nn.functional.interpolate(
image.float(), size=(new_h, new_w), mode="bilinear", antialias=True
)
top, left = (new_h - height) // 2, (new_w - width) // 2
image = image[..., top : top + height, left : left + width]
return image
@staticmethod
def _encode_scale_and_shift(
latents: torch.Tensor, scaling_factor, shift_factor
) -> torch.Tensor:
if shift_factor is not None:
if isinstance(shift_factor, torch.Tensor):
shift_factor = shift_factor.to(latents.device, latents.dtype)
latents = latents - shift_factor
if isinstance(scaling_factor, torch.Tensor):
scaling_factor = scaling_factor.to(latents.device, latents.dtype)
return latents * scaling_factor
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
if batch.condition_image is None:
return batch
if batch.latents is None:
raise ValueError(
"Kandinsky6 video latents must be prepared before image conditioning."
)
image_source = batch.condition_image
if isinstance(image_source, list):
image_source = image_source[0]
device = get_local_torch_device()
height = int(batch.height)
width = int(batch.width)
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
vae_autocast_enabled = autocast_enabled(vae_dtype, server_args.disable_autocast)
image = self._preprocess(image_source, height, width)
image = image.to(device=device, dtype=torch.float32).unsqueeze(2) # [B,C,1,H,W]
generator = batch.generator
if isinstance(generator, list) and len(generator) != image.shape[0]:
generator = generator[0]
with self.use_declared_component(component_name="vae", module=self.vae) as vae:
assert vae is not None
self.vae = vae
prev_use_tiling = vae.use_tiling
vae.use_tiling = False
try:
with autocast_context(vae_dtype, server_args.disable_autocast):
should_cast_vae = not vae_autocast_enabled
if not vae_autocast_enabled:
image = image.to(vae_dtype)
with temporary_module_dtype(
vae, vae_dtype, enabled=should_cast_vae
) as vae_cast:
image_latent = vae_cast.encode(image).sample(
generator=generator
)
finally:
vae.use_tiling = prev_use_tiling
scaling_factor, shift_factor = (
server_args.pipeline_config.get_decode_scale_and_shift(
image_latent.device, image_latent.dtype, vae
)
)
image_latent = self._encode_scale_and_shift(
image_latent, scaling_factor, shift_factor
)
# [B,C,1,H,W] -> [B,1,H,W,C] channel-last, matching the video latent.
image_latent = image_latent.permute(0, 2, 3, 4, 1).contiguous()
batch.image_latent = image_latent
latents = batch.latents
if batch.image_latent.shape[0] != latents.shape[0]:
# encode one reference image, then expand it to the request's output batch
expander = PromptToSampleBatchExpander(
prompt_batch_size=batch.image_latent.shape[0],
sample_batch_size=latents.shape[0],
)
expander.expand_field(batch, "image_latent")
image_latent = batch.image_latent.to(device=latents.device, dtype=latents.dtype)
num_channels = image_latent.shape[-1]
ref_frame = image_latent
if latents.shape[-1] > num_channels:
# [real, cond, mask]: write the reference only into real; cond stays zero
cond_block = torch.zeros_like(image_latent)
mask_block = torch.ones_like(image_latent[..., :1])
ref_frame = torch.cat([image_latent, cond_block, mask_block], dim=-1)
batch.latents = torch.cat([latents, ref_frame], dim=1)
batch.extra[TAIL_COND_ACTIVE_EXTRA_KEY] = True
num_video_frames = latents.shape[1]
token_type_ids = torch.zeros(
(latents.shape[0], num_video_frames + 1), dtype=torch.long, device=device
)
token_type_ids[:, -1] = 1
batch.extra[VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY] = token_type_ids
return batch
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
return result
@@ -0,0 +1,151 @@
# SPDX-License-Identifier: Apache-2.0
"""Joint noise preparation, adapted from FastVideo's Kandinsky6LatentPreparationStage.
Draw video then audio from the same generator. Image conditioning is appended later."""
from __future__ import annotations
import math
import torch
from diffusers.utils.torch_utils import randn_tensor
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
StageValidators as V,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
VerificationResult,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYPE
def audio_latent_duration(
video_latent_frames: int,
*,
fps: float,
audio_sample_rate: int,
audio_downsample_factor: int,
) -> int:
"""Audio latent length from causal video duration at the request's fps."""
pixel_frames = (video_latent_frames - 1) * 4 + 1
return int(
math.ceil(pixel_frames / fps * audio_sample_rate / audio_downsample_factor)
)
class Kandinsky6LatentPreparationStage(PipelineStage):
"""Draw initial video *and* audio noise latents."""
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
if batch.height is None or batch.width is None:
raise ValueError("height and width must be provided for Kandinsky6.")
height = int(batch.height)
width = int(batch.width)
num_frames = int(batch.num_frames)
pipeline_config = server_args.pipeline_config
arch = pipeline_config.dit_config.arch_config
temporal_ratio = (
pipeline_config.vae_config.arch_config.temporal_compression_ratio
)
spatial_ratio = pipeline_config.vae_config.arch_config.spatial_compression_ratio
patch_size = arch.patch_size
if num_frames % temporal_ratio != 1:
num_frames = num_frames // temporal_ratio * temporal_ratio + 1
batch.num_frames = num_frames
required_divisor_h = spatial_ratio * patch_size[1]
required_divisor_w = spatial_ratio * patch_size[2]
if height % required_divisor_h != 0 or width % required_divisor_w != 0:
raise ValueError(
f"Kandinsky6 height must be divisible by {required_divisor_h} and width by "
f"{required_divisor_w}; got height={height}, width={width}."
)
batch_size = batch.batch_size
dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision]
device = get_local_torch_device()
num_latent_frames = (num_frames - 1) // temporal_ratio + 1
num_channels = arch.in_visual_dim
video_shape = (
batch_size,
num_latent_frames,
height // spatial_ratio,
width // spatial_ratio,
num_channels,
)
generator = batch.generator
if batch.latents is None:
video = randn_tensor(
video_shape, generator=generator, device=device, dtype=dtype
)
else:
video = batch.latents.to(device=device, dtype=dtype)
if tuple(video.shape) != video_shape:
raise ValueError(
f"Provided latents shape {list(video.shape)} does not match expected "
f"Kandinsky6 video latent shape {list(video_shape)}."
)
if bool(arch.visual_cond):
cond = torch.zeros_like(video)
mask = torch.zeros(
(*video.shape[:-1], 1), device=video.device, dtype=video.dtype
)
video = torch.cat([video, cond, mask], dim=-1)
num_audio_channels = arch.in_audio_dim
# align audio duration with request fps, not the pipeline default
num_audio_frames = audio_latent_duration(
num_latent_frames,
fps=float(batch.fps),
audio_sample_rate=pipeline_config.audio_sample_rate,
audio_downsample_factor=pipeline_config.audio_downsample_factor,
)
audio_shape = (batch_size, num_audio_frames, num_audio_channels)
if batch.audio_latents is None:
# Drawn from the same generator right after the video noise, so
# it is automatically an independent sample -- no manual seed
# offset needed.
audio = randn_tensor(
audio_shape, generator=generator, device=device, dtype=dtype
)
else:
audio = batch.audio_latents.to(device=device, dtype=dtype)
if tuple(audio.shape) != audio_shape:
raise ValueError(
f"Provided audio_latents shape {list(audio.shape)} does not match expected "
f"Kandinsky6 audio latent shape {list(audio_shape)}."
)
batch.latents = video
batch.audio_latents = audio
batch.raw_latent_shape = (
batch_size,
num_channels,
num_latent_frames,
height // spatial_ratio,
width // spatial_ratio,
)
return batch
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("num_frames", batch.num_frames, V.positive_int)
result.add_check("height", batch.height, V.positive_int)
result.add_check("width", batch.width, V.positive_int)
return result
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
result.add_check(
"audio_latents", batch.audio_latents, [V.is_tensor, V.with_dims(3)]
)
return result
@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky 6 video super-resolution (VSR) stages and framework-independent helpers.
Import the stage classes from their modules (``input_stage``, ``encode_stage``,
``latent_prep_stage``, ``denoising_stage``, ``decode_stage``, ``output_stage``); this package
``__init__`` stays empty so that configs can import the pure helpers (``video_utils``,
``tiling``, ...) cheaply.
"""
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
"""Decode stage of Kandinsky 6 video SR: denoised latent chunks -> uint8 tiles."""
import torch
from sglang.multimodal_gen.runtime.distributed import (
get_decode_parallel_group_coordinator,
get_local_torch_device,
)
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.models.vaes.common import (
can_install_spatial_shard_parallel_decode,
has_decode_parallel_world,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_DENOISED_KEY,
SR_TILES_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
decode_chunks,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
class Kandinsky6SRDecodeStage(PipelineStage):
"""VAE-decodes every denoised chunk into row-major uint8 tiles (``[3, T, Hb, Wb]``)."""
def __init__(self, vae) -> None:
super().__init__()
self.vae = vae
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
return [
ComponentUse(
self._component_stage_name(stage_name),
"vae",
phase="decode_tiles",
target_dtype=vae_dtype,
)
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
denoised = batch.extra.pop(SR_DENOISED_KEY)
device = get_local_torch_device()
group = None
if (
server_args.pipeline_config.vae_config.use_parallel_tiling
and has_decode_parallel_world()
and not can_install_spatial_shard_parallel_decode(
server_args.pipeline_config.vae_config
)
):
group = get_decode_parallel_group_coordinator()
with self.use_declared_component(
component_name="vae", module=self.vae, phase="decode_tiles"
) as vae:
assert vae is not None
self.vae = vae
tiles = decode_chunks(
denoised,
vae,
scaling_factor=vae.scaling_factor,
device=device,
group=group,
)
batch.extra[SR_TILES_KEY] = tiles
return batch
@@ -0,0 +1,137 @@
# SPDX-License-Identifier: Apache-2.0
"""Text-free SR tile denoising with shared residency, compile and cache-DiT hooks.
Each chunk resets its scheduler; the generic single-latent CFG loop and
prompt-padding BCG path do not apply."""
from __future__ import annotations
import torch
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_CHUNKS_KEY,
SR_DENOISED_KEY,
SR_DIT_SPEC_KEY,
SR_SAMPLING_SPEC_KEY,
effective_scheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
denoise_chunks,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
StageValidators as V,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
VerificationResult,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision import precision_to_dtype
class Kandinsky6SRDenoisingStage(DenoisingStage):
"""Runs the bundle's scheduler over every tile chunk prepared by the latent-prep stage."""
def __init__(self, transformer, scheduler, pipeline=None) -> None:
super().__init__(
transformer=transformer,
scheduler=scheduler,
pipeline=pipeline,
transformer_2=None,
vae=None,
)
def _owns_compile_warmup_lifecycle(self) -> bool:
# ``forward`` below does not go through the shared ``_denoise`` loop that this guard
# normally protects, so claim ownership explicitly (same as MiniMaxH3DenoisingStage):
# the offload/restore wrapper is still entered through ``_offload_for_torch_compile_warmup``.
return True
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
"""Validate tile-chunk state instead of the shared text/CFG loop inputs."""
result = VerificationResult()
result.add_check(
"extra[SR_SAMPLING_SPEC_KEY]",
batch.extra.get(SR_SAMPLING_SPEC_KEY),
V.not_none,
)
result.add_check(
"extra[SR_DIT_SPEC_KEY]", batch.extra.get(SR_DIT_SPEC_KEY), V.not_none
)
result.add_check(
"extra[SR_CHUNKS_KEY]", batch.extra.get(SR_CHUNKS_KEY), V.list_not_empty
)
return result
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
result.add_check(
"extra[SR_DENOISED_KEY]", batch.extra.get(SR_DENOISED_KEY), V.list_not_empty
)
return result
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
dit_dtype = precision_to_dtype(
server_args.pipeline_config.dit_precision, "dit_precision"
)
return [
ComponentUse(
self._component_stage_name(stage_name),
"transformer",
phase="denoise_tiles",
target_dtype=dit_dtype,
preferred_ready_after_request=True,
memory_intensive=True,
)
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
with self._offload_for_torch_compile_warmup(batch):
return self._denoise_tiles(batch, server_args)
def _denoise_tiles(self, batch: Req, server_args: ServerArgs) -> Req:
spec = batch.extra.pop(SR_SAMPLING_SPEC_KEY)
dit_spec = batch.extra.pop(SR_DIT_SPEC_KEY)
chunks = batch.extra.pop(SR_CHUNKS_KEY)
device = get_local_torch_device()
# Reuse the shared cache-dit / torch.compile wiring (both are no-ops unless the
# corresponding server arg actually requests them): ``spec.steps_per_chunk`` is this
# request's DiT-call count, the closest analogue of the global ``num_inference_steps``
# those hooks expect.
self._maybe_enable_cache_dit_and_torch_compile(spec.steps_per_chunk, batch)
scheduler = effective_scheduler(spec, self.scheduler)
with self.use_declared_component(
component_name="transformer", module=self.transformer, phase="denoise_tiles"
) as transformer:
assert transformer is not None
self.transformer = transformer
total = spec.steps_per_chunk * len(chunks)
with self.progress_bar(
total=total, batch=batch, desc="Kandinsky6 SR denoising"
) as progress:
denoised = denoise_chunks(
chunks,
transformer,
scheduler,
dit_spec=dit_spec,
spec=spec,
device=device,
step_context=lambda step: set_forward_context(
current_timestep=step, attn_metadata=None, forward_batch=batch
),
on_step=progress.update,
)
self._finish_active_component_use()
batch.extra[SR_DENOISED_KEY] = denoised
return batch
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
"""Whole-video KVAE encode of the latent-upscaler (LU) path of Kandinsky 6 video SR."""
import torch
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_LR_LATENT_KEY,
SR_TILING_SCALE_KEY,
SR_VIDEO_KEY,
uses_latent_path,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
encode_video_to_lr_latent,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
class Kandinsky6SREncodeStage(PipelineStage):
"""Encode the whole clip only for the LU path; pixel mode encodes tiles later."""
def __init__(self, vae, latent_upscaler=None) -> None:
super().__init__()
self.vae = vae
# Only consulted to pick the path (does the bank have this scale?).
self.latent_upscaler = latent_upscaler
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
return [
ComponentUse(
self._component_stage_name(stage_name),
"vae",
phase="encode_video",
target_dtype=vae_dtype,
start_at_stage_entry=False,
)
]
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
if not uses_latent_path(self.latent_upscaler, tiling_scale):
return batch
video = batch.extra.pop(SR_VIDEO_KEY)
with self.use_declared_component(
component_name="vae", module=self.vae, phase="encode_video"
) as vae:
assert vae is not None
self.vae = vae
lr_latent = encode_video_to_lr_latent(
video, vae, device=get_local_torch_device()
)
batch.extra[SR_LR_LATENT_KEY] = lr_latent.cpu()
return batch
@@ -0,0 +1,95 @@
# SPDX-License-Identifier: Apache-2.0
"""Input stage of Kandinsky 6 video SR: source video -> aligned clip (+ source audio)."""
import os
import torch
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_REQUESTED_HW_KEY,
SR_TILING_SCALE_KEY,
SR_VIDEO_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
VAE_SPATIAL_FACTOR,
pad_to_spatial_factor,
pre_upscale_video,
resolve_scale_request,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
SOURCE_AUDIO_SAMPLE_RATE,
DecodedClip,
decode_clip,
extract_source_audio,
synthetic_clip,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
# Warmup requests carry no (usable) video: run the real path on a small synthetic clip.
WARMUP_CLIP_FRAMES = 9
WARMUP_CLIP_HW = (256, 384)
class Kandinsky6SRInputStage(PipelineStage):
"""Decode [T, 3, H, W] uint8 video, align fps/frames/size and retain source audio.
Record requested dimensions before VAE padding so output can remove it."""
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
tiling_scale, pre_upscale = resolve_scale_request(batch.sr_resolution_scale)
clip = self._load_clip(batch)
video = clip.frames
if pre_upscale != 1.0:
video = pre_upscale_video(video, pre_upscale, VAE_SPATIAL_FACTOR)
frames = video.shape[0]
# Align to the VAE spatial factor for every scale, not only 2.25 (whose pixel
# pre-upscale happens to already land on an aligned size): without this, a source
# whose dims are not already a multiple of 16 (e.g. a plain 1080p source at x2/x4,
# which runs no pixel resize at all) fails partway through the whole-video KVAE
# encode's four halvings instead of at a clear boundary. Padding (not resizing) keeps
# every source pixel; batch.height / batch.width below, and the output stage's final
# crop, restore the originally requested dimensions.
video, (requested_h, requested_w) = pad_to_spatial_factor(
video, VAE_SPATIAL_FACTOR
)
batch.extra[SR_VIDEO_KEY] = video
batch.extra[SR_TILING_SCALE_KEY] = tiling_scale
batch.extra[SR_REQUESTED_HW_KEY] = (requested_h, requested_w)
batch.fps = clip.fps
batch.num_frames = frames
batch.height = requested_h * tiling_scale
batch.width = requested_w * tiling_scale
self._attach_source_audio(batch, frames=frames, fps=clip.fps)
return batch
def _load_clip(self, batch: Req) -> DecodedClip:
if batch.is_warmup:
height, width = WARMUP_CLIP_HW
frames = synthetic_clip(
height=height, width=width, frames=WARMUP_CLIP_FRAMES
)
return DecodedClip(frames=frames, fps=24, source_fps=24.0)
path = batch.sampling_params.source_video_path()
if path is None or not os.path.isfile(path):
raise ValueError(f"Kandinsky6 SR input video not found: {path!r}")
self.log_info("Decoding %s", path)
return decode_clip(path)
@staticmethod
def _attach_source_audio(batch: Req, *, frames: int, fps: int) -> None:
if batch.is_warmup:
return
path = batch.sampling_params.source_video_path()
audio = extract_source_audio(
path, sample_rate=SOURCE_AUDIO_SAMPLE_RATE, max_seconds=frames / fps
)
if audio is None:
batch.audio = None
batch.audio_sample_rate = None
return
batch.audio = torch.from_numpy(audio)
batch.audio_sample_rate = SOURCE_AUDIO_SAMPLE_RATE
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
"""Prepare noisy tile chunks through the latent upscaler or pixel/VAE fallback.
Validate sampler/head compatibility and step count before processing tiles."""
from functools import partial
import torch
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
ComponentUse,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_CHUNKS_KEY,
SR_DIT_SPEC_KEY,
SR_LR_LATENT_KEY,
SR_PLAN_KEY,
SR_SAMPLING_SPEC_KEY,
SR_TILING_SCALE_KEY,
SR_VIDEO_KEY,
build_dit_spec,
build_sampling_spec,
check_denoising_request,
uses_latent_path,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
DitSpec,
SamplingSpec,
module_dtype,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
TilePlan,
plan_tiles,
prepare_lu_tile_latents,
prepare_pixel_tile_latents,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
VAE_SPATIAL_FACTOR,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.precision import (
resolve_precision,
)
class Kandinsky6SRLatentPrepStage(PipelineStage):
"""LU-or-pixel tile encode + upscale -> initial noisy latent, one chunk at a time."""
def __init__(self, vae, transformer, latent_upscaler=None, scheduler=None) -> None:
super().__init__()
self.vae = vae
self.transformer = transformer # arch-config / dtype only: never called here
self.latent_upscaler = latent_upscaler
self.scheduler = (
scheduler # resolved sampler validation only: never stepped here
)
def component_uses(
self, server_args: ServerArgs, stage_name: str | None = None
) -> list[ComponentUse]:
stage_name = self._component_stage_name(stage_name)
vae_dtype = resolve_precision(
server_args, "vae", precision_attr="vae_precision"
)
uses: list[ComponentUse] = []
if self.latent_upscaler is not None:
uses.append(
ComponentUse(
stage_name,
"latent_upscaler",
phase="upscale_tiles",
target_dtype=torch.bfloat16,
)
)
uses.append(
ComponentUse(
stage_name, "vae", phase="encode_tiles", target_dtype=vae_dtype
)
)
return uses
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
arch = server_args.pipeline_config.dit_config.arch_config
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
seed = batch.seed if isinstance(batch.seed, int) else batch.seed[0]
spec = build_sampling_spec(
arch=arch,
tiling_scale=tiling_scale,
seed=seed,
num_steps=batch.num_inference_steps,
tiles_batch_size=batch.sr_tiles_batch_size,
tile_min_overlap=batch.sr_tile_min_overlap,
scheduler=self.scheduler,
)
check_denoising_request(spec, batch.num_inference_steps)
dit_spec = build_dit_spec(self.transformer)
device = get_local_torch_device()
plan, chunks = self._prepare_chunks(batch, spec, dit_spec, device)
batch.extra[SR_PLAN_KEY] = plan
batch.extra[SR_CHUNKS_KEY] = chunks
batch.extra[SR_SAMPLING_SPEC_KEY] = spec
batch.extra[SR_DIT_SPEC_KEY] = dit_spec
return batch
def _prepare_chunks(
self, batch: Req, spec: SamplingSpec, dit_spec: DitSpec, device: torch.device
) -> tuple[TilePlan, list[torch.Tensor]]:
common = dict(
visual_size=spec.visual_size,
tiling_scale=spec.tiling_scale,
tile_min_overlap=spec.tile_min_overlap,
)
if uses_latent_path(self.latent_upscaler, spec.tiling_scale):
lr_latent = batch.extra.pop(SR_LR_LATENT_KEY)
plan = plan_tiles(
frame_hw=(
lr_latent.shape[-2] * VAE_SPATIAL_FACTOR,
lr_latent.shape[-1] * VAE_SPATIAL_FACTOR,
),
**common,
)
with self.use_declared_component(
component_name="latent_upscaler",
module=self.latent_upscaler,
phase="upscale_tiles",
) as bank:
assert bank is not None
self.latent_upscaler = bank
chunks = prepare_lu_tile_latents(
lr_latent,
plan,
upscale_fn=partial(bank.upscale, scale=spec.tiling_scale),
lu_dtype=module_dtype(bank),
scaling_factor=self.vae.scaling_factor,
dit_spec=dit_spec,
spec=spec,
device=device,
)
return plan, chunks
video = batch.extra.pop(SR_VIDEO_KEY)
plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common)
with self.use_declared_component(
component_name="vae", module=self.vae, phase="encode_tiles"
) as vae:
assert vae is not None
self.vae = vae
chunks = prepare_pixel_tile_latents(
video,
plan,
vae=vae,
scaling_factor=vae.scaling_factor,
dit_spec=dit_spec,
spec=spec,
device=device,
)
return plan, chunks
@@ -0,0 +1,134 @@
# SPDX-License-Identifier: Apache-2.0
# Vendored from sr_core (parity-tested against the k6_video reference); adapted from
# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0).
"""SR latent construction with reference RNG ordering.
Initial noise uses a per-request generator; conditioning noise uses the global RNG."""
from __future__ import annotations
from typing import Literal
import torch
InstructType = Literal["noise", "channel", "hybrid", "hybrid_anchor"]
LqNoiseType = Literal["linear", "ddpm"]
def degrade_lq_latent(
lq_latent: torch.Tensor,
noise_scale: float = 0.7,
noise_type: LqNoiseType = "linear",
generator: torch.Generator | None = None,
) -> torch.Tensor:
"""Mix linear or variance-preserving (ddpm) noise; nonpositive scales are a no-op."""
if noise_scale <= 0:
return lq_latent
eps = torch.randn(
lq_latent.shape,
device=lq_latent.device,
dtype=lq_latent.dtype,
generator=generator,
)
if noise_type == "ddpm":
return (1 - noise_scale**2) ** 0.5 * lq_latent + noise_scale * eps
return (1 - noise_scale) * lq_latent + noise_scale * eps
def _cast_floating(value: torch.Tensor, dtype: torch.dtype | None) -> torch.Tensor:
"""Cast floating-point ``value`` to ``dtype`` (no-op for ``None`` / non-float / same dtype)."""
if dtype is not None and value.is_floating_point() and value.dtype != dtype:
return value.to(dtype=dtype)
return value
def build_initial_latent( # noqa: PLR0913
*,
instruct_type: InstructType,
visual_cond: bool,
in_visual_dim: int,
lq_latent: torch.Tensor,
batch_size: int,
duration: int,
height: int,
width: int,
device: str | int | torch.device,
seed: int,
lq_noise_scale: float,
lq_noise_type: LqNoiseType,
lq_channel_noise_scale: float,
dtype: torch.dtype | None = None,
) -> torch.Tensor:
"""Build [starting | conditioning | mask] for the selected instruction mode.
Inputs and output use [B*T, H, W, C]; the output has 2C+1 channels except
unconditioned noise mode. Cast LQ to the DiT dtype before drawing noise."""
lq_latent = lq_latent.to(device)
lq_latent = _cast_floating(lq_latent, dtype)
if instruct_type == "noise":
g_noise = torch.Generator(device=device)
g_noise.manual_seed(seed)
degraded_lq = degrade_lq_latent(
lq_latent, lq_noise_scale, lq_noise_type, generator=g_noise
)
if not visual_cond:
# No conditioning channels in the model input layer (e.g. the
# t2v-initialized KVAE SR DiT) — feed the bare C-channel latent.
return degraded_lq
zero_cond = torch.zeros_like(degraded_lq)
zero_mask = torch.zeros(
[*degraded_lq.shape[:-1], 1], dtype=degraded_lq.dtype, device=device
)
return torch.cat([degraded_lq, zero_cond, zero_mask], dim=-1)
if instruct_type in ("channel", "hybrid"):
if instruct_type == "hybrid":
g_noise = torch.Generator(device=device)
g_noise.manual_seed(seed)
starting = degrade_lq_latent(
lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise
)
else:
g = torch.Generator(device=device)
g.manual_seed(seed)
starting = torch.randn(
batch_size * duration,
height,
width,
in_visual_dim,
device=device,
generator=g,
)
channel_lq = degrade_lq_latent(
lq_latent, lq_channel_noise_scale, noise_type="linear"
)
mask = torch.ones_like(lq_latent[..., :1])
return torch.cat([starting, channel_lq, mask], dim=-1)
if instruct_type == "hybrid_anchor":
# Anchor-free inference (tiled real-world LQ has no HR ground truth):
# a zeroed anchor + zeroed mask runs the model with no anchor signal,
# equivalent to anchor_indices=[] for the packed-latent datasets.
anchor_latent = torch.zeros_like(lq_latent)
anchor_mask = torch.zeros_like(lq_latent[..., :1])
g_noise = torch.Generator(device=device)
g_noise.manual_seed(seed)
starting = degrade_lq_latent(
lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise
)
anchor = degrade_lq_latent(
anchor_latent, lq_channel_noise_scale, noise_type="linear"
)
return torch.cat([starting, anchor, anchor_mask], dim=-1)
msg = f"generate_sample_sr does not support instruct_type={instruct_type!r}"
raise ValueError(msg)
__all__ = [
"InstructType",
"LqNoiseType",
"build_initial_latent",
"degrade_lq_latent",
]
@@ -0,0 +1,72 @@
# SPDX-License-Identifier: Apache-2.0
"""Stitch / crop / resize / output stage of Kandinsky 6 video SR."""
import torch
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
resolve_target_hw,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_PLAN_KEY,
SR_REQUESTED_HW_KEY,
SR_TILES_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
stitch_tiles,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
crop_to_hw,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
to_output_video,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
resize_to_target,
)
from sglang.multimodal_gen.runtime.server_args import ServerArgs
class Kandinsky6SROutputStage(PipelineStage):
"""Hann-blends the decoded tiles, restores the requested output size, applies the
optional delivery resize and returns ``output = [1, 3, T, H, W]`` (fp16 in [0, 1])
plus the mono source audio.
CPU only: every model phase already ran. ``batch.height`` / ``batch.width`` are set
to the final size, ``batch.fps`` was set by the input stage (and now also carried on
the returned ``OutputBatch``, so a worker-resampled fps reaches the client-side save
path unchanged).
"""
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
tiles = batch.extra.pop(SR_TILES_KEY)
plan = batch.extra.pop(SR_PLAN_KEY)
video = stitch_tiles(tiles, plan)
del tiles
requested_hw = batch.extra.pop(SR_REQUESTED_HW_KEY, None)
if requested_hw is not None:
requested_h, requested_w = requested_hw
video = crop_to_hw(
video,
(requested_h * plan.tiling_scale, requested_w * plan.tiling_scale),
)
target_hw = resolve_target_hw(
batch.sr_target_resolution,
tuple(video.shape[-2:]),
batch.sr_target_resize_mode,
)
if target_hw is not None:
video = resize_to_target(video, target_hw)
batch.height, batch.width = video.shape[-2:]
return OutputBatch(
output=to_output_video(video),
audio=batch.audio,
audio_sample_rate=batch.audio_sample_rate,
fps=batch.fps,
metrics=batch.metrics,
usage=batch.usage,
)
@@ -0,0 +1,221 @@
# SPDX-License-Identifier: Apache-2.0
"""SR request state and sampler resolution.
batch.extra contains only CPU tensors and serializable data, never model objects."""
from typing import Any
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRArchConfig,
)
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
PiflowScheduler,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
DitSpec,
PiflowParams,
SamplingSpec,
module_dtype,
)
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
SR_VIDEO_KEY = (
"kandinsky6_sr_video" # [T, 3, H, W] uint8 clip (after pre-upscale + pad)
)
SR_TILING_SCALE_KEY = "kandinsky6_sr_tiling_scale" # int tile scale (2 or 4)
SR_LR_LATENT_KEY = "kandinsky6_sr_lr_latent" # [T', C, h, w] fp32 (LU path)
SR_PLAN_KEY = "kandinsky6_sr_tile_plan" # tiled.TilePlan
SR_TILES_KEY = "kandinsky6_sr_tiles" # list of uint8 [3, T, Hb, Wb]
# Handoff between the three model-phase stages (latent prep -> denoise -> decode), split out
# of the single combined stage so the denoising stage can subclass the shared DenoisingStage
# (see denoising_stage.py): SamplingSpec / DitSpec are immutable msgspec.Struct values (plain
# data, like TilePlan above), and the chunk lists are CPU tensors throughout.
SR_SAMPLING_SPEC_KEY = "kandinsky6_sr_sampling_spec" # sampling.SamplingSpec
SR_DIT_SPEC_KEY = "kandinsky6_sr_dit_spec" # sampling.DitSpec
SR_CHUNKS_KEY = (
"kandinsky6_sr_chunks" # list of fp32 [B, T', H', W', 2C+1] (pre-denoise)
)
SR_DENOISED_KEY = (
"kandinsky6_sr_denoised" # list of fp32 [B, T', H', W', C] (post-denoise)
)
# (H, W) of the clip actually requested, before any VAE-spatial-factor alignment padding the
# input stage added so whole-video KVAE encoding survives every supported sr_resolution_scale
# (not only 2.25, whose pixel pre-upscale happens to already land on an aligned size). The
# output stage crops the stitched result from (padded_H * scale, padded_W * scale) down to
# (H * scale, W * scale) so the delivered video matches what was actually asked for.
SR_REQUESTED_HW_KEY = "kandinsky6_sr_requested_hw"
def uses_latent_path(latent_upscaler: Any, tiling_scale: int) -> bool:
"""LU path iff the bank has an entry for the tiling scale (else the pixel path)."""
return (
latent_upscaler is not None
and latent_upscaler.for_scale(tiling_scale) is not None
)
def build_dit_spec(dit: Any) -> DitSpec:
"""Resolved DiT input and conditioning layout."""
return DitSpec(
instruct_type=dit.instruct_type,
visual_cond=bool(dit.visual_cond),
in_visual_dim=dit.in_visual_dim,
use_motion_score=bool(dit.use_motion_score),
patch_size=tuple(dit.patch_size),
dtype=module_dtype(dit),
)
def _rope_scale_factor(arch: Kandinsky6SRArchConfig) -> tuple[float, float, float]:
table = arch.sr_scale_factor
visual_size = arch.sr_visual_size[0]
if table is None:
raise ValueError(
"transformer/config.json carries no sr_params (visual_size / scale_factor); "
"an official Diffusers Kandinsky6SRPipeline repo is required"
)
values = table.get(str(visual_size), table.get(visual_size))
if values is None:
raise ValueError(
f"sr_scale_factor has no entry for visual size {visual_size}: {table}"
)
return tuple(float(value) for value in values)
def piflow_params_from_scheduler(scheduler: Any) -> PiflowParams | None:
"""pi-Flow sampler values of a ``PiflowScheduler`` component (``None``: flow-Euler)."""
if not isinstance(scheduler, PiflowScheduler):
return None
config = scheduler.config
if config["nfe"] is None:
return None
return PiflowParams(
nfe=int(config["nfe"]),
num_policy_substeps=int(config["num_policy_substeps"]),
final_step_size_scale=float(config["final_step_size_scale"]),
shift=float(config["shift"]),
n_grid=int(config["n_grid"]),
eps=float(config["eps"]),
)
def _describe_scheduler(scheduler: Any) -> str:
if scheduler is None:
return "missing"
if isinstance(scheduler, PiflowScheduler):
return "a PiflowScheduler without nfe"
return f"a {type(scheduler).__name__}"
def _check_sampler_fits_head(
*, arch: Kandinsky6SRArchConfig, piflow: PiflowParams | None, scheduler: Any
) -> None:
"""Reject incompatible scheduler and DiT head widths before encoding tiles."""
head, grid = arch.head_width, arch.base_out_visual_dim
if piflow is None:
if head != grid:
raise ValueError(
f"Kandinsky6SR: the transformer head is {head} channels wide (a DX / "
f"pi-Flow head: {grid} channels per latent), but the sampler resolved "
"to flow-Euler: the scheduler component is "
f"{_describe_scheduler(scheduler)}, and only a PiflowScheduler with "
"nfe selects pi-Flow. Use the scheduler of the same repo as the "
"transformer."
)
return
if piflow.nfe < 1:
raise ValueError(
"Kandinsky6SR: the pi-Flow sampler needs nfe >= 1 model calls per tile, "
f"got nfe={piflow.nfe}."
)
if head != piflow.n_grid * grid:
raise ValueError(
f"Kandinsky6SR: the PiflowScheduler (nfe {piflow.nfe}, n_grid "
f"{piflow.n_grid}) needs a {piflow.n_grid} x {grid} = "
f"{piflow.n_grid * grid} channel DX head, but the transformer head is "
f"{head} channels wide (out_visual_dim {arch.out_visual_dim}). The "
"scheduler and the transformer come from different repos."
)
def build_sampling_spec(
*,
arch: Kandinsky6SRArchConfig,
tiling_scale: int,
seed: int,
num_steps: int,
tiles_batch_size: int,
tile_min_overlap: float,
scheduler: Any = None,
) -> SamplingSpec:
"""Resolve sampling from the bundle scheduler, falling back to legacy DiT fields.
num_steps counts DiT calls, not Diffusers timestep grid points."""
piflow = piflow_params_from_scheduler(scheduler)
if piflow is None and arch.is_piflow:
piflow = PiflowParams(
nfe=arch.piflow_nfe,
num_policy_substeps=arch.piflow_num_policy_substeps,
final_step_size_scale=arch.piflow_final_step_size_scale,
shift=arch.piflow_shift,
n_grid=arch.n_grid,
eps=arch.piflow_eps,
)
_check_sampler_fits_head(arch=arch, piflow=piflow, scheduler=scheduler)
return SamplingSpec(
tiling_scale=tiling_scale,
tiles_batch_size=tiles_batch_size,
seed=seed,
num_steps=num_steps,
tile_min_overlap=tile_min_overlap,
visual_size=arch.sr_visual_size[0],
scale_factor=_rope_scale_factor(arch),
scheduler_scale=arch.sr_scheduler_scale,
lq_noise_scale=arch.sr_lq_noise_scale,
lq_noise_type=arch.sr_lq_noise_type,
lq_channel_noise_scale=arch.sr_lq_channel_noise_scale,
cap_noise_timestep=arch.sr_cap_noise_timestep,
piflow=piflow,
)
def effective_scheduler(spec: SamplingSpec, scheduler: Any) -> Any:
"""Use the loaded scheduler when compatible; synthesize one for legacy configs."""
if spec.piflow is not None:
if isinstance(scheduler, PiflowScheduler):
return scheduler
return PiflowScheduler(
nfe=spec.piflow.nfe,
n_grid=spec.piflow.n_grid,
shift=spec.piflow.shift,
eps=spec.piflow.eps,
final_step_size_scale=spec.piflow.final_step_size_scale,
num_policy_substeps=spec.piflow.num_policy_substeps,
)
if isinstance(scheduler, FlowMatchEulerDiscreteScheduler):
return scheduler
return FlowMatchEulerDiscreteScheduler(shift=spec.scheduler_scale)
def check_denoising_request(spec: SamplingSpec, num_inference_steps: int) -> None:
"""Validate the requested DiT-call count before denoising.
Diffusers counts timestep grid points instead: its default 5 is 4 calls here."""
if num_inference_steps < 1:
raise ValueError(
f"Kandinsky6 SR: num_inference_steps must be >= 1, got {num_inference_steps}"
)
if spec.piflow is not None and num_inference_steps != spec.piflow.nfe:
logger.warning(
"Kandinsky6 SR: num_inference_steps=%d was requested, but this is a pi-Flow "
"bundle distilled for nfe=%d; num_inference_steps is ignored for pi-Flow "
"bundles and every tile will run exactly nfe=%d DiT calls instead.",
num_inference_steps,
spec.piflow.nfe,
spec.piflow.nfe,
)
@@ -0,0 +1,191 @@
# SPDX-License-Identifier: Apache-2.0
"""Scheduler-driven denoising for Kandinsky 6 SR tile chunks."""
from collections.abc import Callable
from contextlib import AbstractContextManager, nullcontext
from typing import Any
import msgspec
import torch
# (latent [B, T, H, W, C], model time [B] (already * 1000), step index) -> DiT output
DitFn = Callable[[torch.Tensor, torch.Tensor, int], torch.Tensor]
StepCallback = Callable[[], None]
StepContext = Callable[[int], AbstractContextManager]
class PiflowParams(msgspec.Struct, frozen=True, kw_only=True):
"""pi-Flow parameters from the scheduler or legacy flat DiT config."""
nfe: int
num_policy_substeps: int
final_step_size_scale: float
shift: float
n_grid: int
eps: float
class DitSpec(msgspec.Struct, frozen=True, kw_only=True):
"""Post-override behaviour of the DiT that the sampling code needs."""
instruct_type: str | None
visual_cond: bool
in_visual_dim: int
use_motion_score: bool
patch_size: tuple[int, int, int]
# The DiT's loaded parameter dtype (``module_dtype(dit)``); the reference casts the
# LQ latent to this dtype before building the initial latent, so a bf16-loaded
# checkpoint's LQ-noise mix rounds in bf16, not fp32. ``None`` (parameter-less DiT,
# e.g. a mocked one in a test) keeps the LQ latent's own dtype.
dtype: torch.dtype | None = None
class SamplingSpec(msgspec.Struct, frozen=True, kw_only=True):
"""Everything besides the components that defines one tiled SR run."""
tiling_scale: int
tiles_batch_size: int
seed: int
num_steps: int
tile_min_overlap: float
visual_size: int
scale_factor: tuple[float, float, float]
scheduler_scale: float
lq_noise_scale: float
lq_noise_type: str
lq_channel_noise_scale: float
cap_noise_timestep: bool
piflow: PiflowParams | None
@property
def steps_per_chunk(self) -> int:
"""DiT calls per chunk of tiles: ``nfe`` (pi-Flow), else ``num_steps``."""
return self.piflow.nfe if self.piflow is not None else self.num_steps
def bf16_autocast(device: torch.device | str) -> AbstractContextManager:
"""CUDA bf16 autocast (a no-op on other devices), as the reference wraps its models."""
if torch.device(device).type != "cuda":
return nullcontext()
return torch.autocast(device_type="cuda", dtype=torch.bfloat16)
def module_dtype(module: torch.nn.Module) -> torch.dtype | None:
"""Dtype of the first parameter (``None`` for a parameter-less module)."""
return next((param.dtype for param in module.parameters()), None)
def cast_to_module_dtype(module: torch.nn.Module, value: torch.Tensor) -> torch.Tensor:
"""Cast a floating tensor to the module's parameter dtype (reference helper)."""
dtype = module_dtype(module)
if dtype is not None and value.is_floating_point() and value.dtype != dtype:
return value.to(dtype=dtype)
return value
def euler_start_timestep(
*, cap_noise_timestep: bool, lq_noise_scale: float, instruct_type: str | None
) -> float:
"""Euler start time: the LQ noise level when capped for noise-type conditioning."""
capped = cap_noise_timestep and instruct_type in (
"noise",
"hybrid",
"hybrid_anchor",
)
return lq_noise_scale if capped else 1.0
def check_sampler_options(
*, piflow: PiflowParams | None, cap_noise_timestep: bool, instruct_type: str | None
) -> None:
"""Reject option combinations the reference sampler rejects."""
if (
piflow is not None
and cap_noise_timestep
and instruct_type in ("noise", "hybrid")
):
raise NotImplementedError(
"piflow sampler does not support cap_noise_timestep for noise/hybrid "
"instruct"
)
def make_dit_fn(
dit: Callable[..., torch.Tensor],
*,
latent_frames_hw: tuple[int, int, int],
patch_size: tuple[int, int, int],
scale_factor: tuple[float, float, float],
use_motion_score: bool,
step_context: StepContext | None = None,
) -> DitFn:
"""Bind per-chunk RoPE/motion inputs and an optional per-step context."""
frames, height, width = latent_frames_hw
context = step_context or (lambda step: nullcontext())
def call(x: torch.Tensor, model_time: torch.Tensor, step: int) -> torch.Tensor:
rope_pos = [
torch.arange(frames, device=x.device),
torch.arange(height // patch_size[1], device=x.device),
torch.arange(width // patch_size[2], device=x.device),
]
motion = (
torch.full((1,), 900.0, device=x.device, dtype=x.dtype)
if use_motion_score
else None
)
with context(step):
return dit(
x,
model_time,
rope_pos,
scale_factor=scale_factor,
motion_score=motion,
)
return call
@torch.no_grad()
def denoise_with_scheduler(
x: torch.Tensor,
dit_fn: DitFn,
scheduler: Any,
*,
channels: int,
is_piflow: bool,
on_step: StepCallback | None = None,
) -> torch.Tensor:
"""Denoise one chunk after set_timesteps; keep conditioning channels fixed.
Promote predictions to fp32 so flow-Euler does not downcast its running
state to the bf16 prediction dtype. pi-Flow already preserves the state dtype."""
state = x[..., :channels].float()
cond = x[..., channels:]
for step, t in enumerate(scheduler.timesteps):
model_time = t.to(device=x.device, dtype=torch.float32).expand(state.shape[0])
hidden_states = torch.cat([state, cond], dim=-1) if cond.shape[-1] else state
prediction = dit_fn(hidden_states, model_time, step)
if not is_piflow:
prediction = prediction.float()
state = scheduler.step(prediction, t, state, return_dict=False)[0]
if on_step is not None:
on_step()
return state.float()
__all__ = [
"DitFn",
"DitSpec",
"PiflowParams",
"SamplingSpec",
"StepCallback",
"StepContext",
"bf16_autocast",
"cast_to_module_dtype",
"check_sampler_options",
"denoise_with_scheduler",
"euler_start_timestep",
"make_dit_fn",
"module_dtype",
]
@@ -0,0 +1,413 @@
# SPDX-License-Identifier: Apache-2.0
"""Tile preparation, denoising and blending for Kandinsky 6 SR.
Stages group work by component to avoid swapping weights per tile.
Each chunk uses seed + its first tile index, matching the reference.
"""
from collections.abc import Callable, Iterator, Mapping, Sequence
from typing import Any
import msgspec
import torch
from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latents import (
build_initial_latent,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
DitSpec,
SamplingSpec,
StepCallback,
StepContext,
bf16_autocast,
cast_to_module_dtype,
check_sampler_options,
denoise_with_scheduler,
euler_start_timestep,
make_dit_fn,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
RESOLUTIONS,
VAE_SPATIAL_FACTOR,
TileGrid,
closest_base_resolution,
compute_tile_grid_even,
extract_all_tiles,
latent_tile_grid_from_pixel_grid,
stitch_tiles_hanning,
upsample_tiles_to_base,
)
LatentUpscaleFn = Callable[[torch.Tensor], torch.Tensor]
STITCH_FRAME_CHUNK = 8
class TilePlan(msgspec.Struct, frozen=True):
"""Tile geometry of one run (all sizes in pixels of the frame that is tiled)."""
tiling_scale: int
frame_hw: tuple[int, int]
base_hw: tuple[int, int]
tile_hw: tuple[int, int]
pixel_grid: TileGrid
def plan_tiles(
*,
frame_hw: tuple[int, int],
visual_size: int,
tiling_scale: int,
tile_min_overlap: float,
spatial_factor: int = VAE_SPATIAL_FACTOR,
resolutions: Mapping[int, Sequence[tuple[int, int]]] = RESOLUTIONS,
) -> TilePlan:
"""Even tile grid for a frame; the frame must hold at least one full tile."""
height, width = frame_hw
base_hw = closest_base_resolution(height, width, visual_size, resolutions)
if any(size % tiling_scale for size in base_hw):
raise ValueError(
f"resolution_scale={tiling_scale} does not divide the base resolution "
f"{base_hw[0]}x{base_hw[1]} exactly"
)
tile_hw = (base_hw[0] // tiling_scale, base_hw[1] // tiling_scale)
grid = compute_tile_grid_even(
height, width, tile_hw, tile_min_overlap, spatial_factor
)
if height < tile_hw[0] or width < tile_hw[1]:
raise ValueError(
f"The {width}x{height} frame is smaller than one SR tile "
f"({tile_hw[1]}x{tile_hw[0]}) at x{tiling_scale}; use a larger input or a "
"higher scale."
)
return TilePlan(
tiling_scale=tiling_scale,
frame_hw=(height, width),
base_hw=base_hw,
tile_hw=tile_hw,
pixel_grid=grid,
)
def chunk_ranges(num_tiles: int, tiles_batch_size: int) -> Iterator[tuple[int, int]]:
"""``(start, stop)`` tile ranges; ``start`` also offsets the chunk's seed."""
for start in range(0, num_tiles, tiles_batch_size):
yield start, min(start + tiles_batch_size, num_tiles)
@torch.no_grad()
def encode_video_to_lr_latent(
video: torch.Tensor, vae: torch.nn.Module, *, device: torch.device
) -> torch.Tensor:
"""Encode a whole ``[T, C, H, W]`` uint8 video to the raw latent ``[T', C', h, w]``.
The latent is *not* multiplied by the VAE scaling factor (the LU does that itself).
"""
if video.ndim != 4:
raise ValueError(f"video must be [T, C, H, W], got {tuple(video.shape)}")
pixel = video.permute(1, 0, 2, 3).unsqueeze(0).to(device=device)
pixel = cast_to_module_dtype(vae, vae.normalize_data(pixel.float()))
latent = vae.encode(pixel)[0]
return latent.squeeze(0).permute(1, 0, 2, 3).float()
@torch.no_grad()
def upscale_lr_latent_tile(
tile: torch.Tensor,
upscale_fn: LatentUpscaleFn,
*,
lu_dtype: torch.dtype | None,
scaling_factor: float,
device: torch.device,
) -> torch.Tensor:
"""LU on one raw tile ``[T', C', h, w]`` -> scaled latent ``[T', Hb, Wb, C']``."""
z = tile.permute(1, 0, 2, 3).unsqueeze(0).to(device=device, dtype=torch.float32)
if lu_dtype is not None and z.dtype != lu_dtype:
z = z.to(dtype=lu_dtype)
with bf16_autocast(device):
upscaled = upscale_fn(z * scaling_factor)
return upscaled.squeeze(0).permute(1, 2, 3, 0).float()
@torch.no_grad()
def encode_pixel_tile(
tile: torch.Tensor,
vae: torch.nn.Module,
*,
scaling_factor: float,
device: torch.device,
) -> torch.Tensor:
"""Pixel path: ``[T, Hb, Wb, 3]`` floats in [0, 255] -> scaled ``[T', Hb', Wb', C']``.
The tile goes through bf16 first (also on CPU), like the reference.
"""
lq = tile.permute(3, 0, 1, 2).unsqueeze(0).to(device=device, dtype=torch.bfloat16)
lq = cast_to_module_dtype(vae, vae.normalize_data(lq))
latent = vae.encode(lq)[0]
return latent.squeeze(0).permute(1, 2, 3, 0).float() * scaling_factor
def build_chunk_latent(
lq_tiles: Sequence[torch.Tensor],
*,
seed: int,
dit_spec: DitSpec,
spec: SamplingSpec,
device: torch.device,
) -> torch.Tensor:
"""Initial noisy latent ``[B, T', H', W', C]`` of one chunk of scaled LQ tiles."""
lq = torch.cat(list(lq_tiles), dim=0)
batch = len(lq_tiles)
x = build_initial_latent(
instruct_type=dit_spec.instruct_type,
visual_cond=dit_spec.visual_cond,
in_visual_dim=dit_spec.in_visual_dim,
lq_latent=lq,
batch_size=batch,
duration=lq.shape[0] // batch,
height=lq.shape[1],
width=lq.shape[2],
device=device,
seed=seed,
lq_noise_scale=spec.lq_noise_scale,
lq_noise_type=spec.lq_noise_type,
lq_channel_noise_scale=spec.lq_channel_noise_scale,
# The DiT's own loaded dtype (reference: cast_to_module_dtype(dit, ...) before
# building the initial latent), not a hard-coded fp32 regardless of precision.
dtype=dit_spec.dtype,
)
return x.reshape(batch, -1, *x.shape[1:])
@torch.no_grad()
def prepare_lu_tile_latents(
lr_latent: torch.Tensor,
plan: TilePlan,
*,
upscale_fn: LatentUpscaleFn,
lu_dtype: torch.dtype | None,
scaling_factor: float,
dit_spec: DitSpec,
spec: SamplingSpec,
device: torch.device,
spatial_factor: int = VAE_SPATIAL_FACTOR,
) -> list[torch.Tensor]:
"""LU path: initial latents (CPU) of every chunk from the whole-video latent."""
latent_grid = latent_tile_grid_from_pixel_grid(plan.pixel_grid, spatial_factor)
tiles = extract_all_tiles(lr_latent, latent_grid)
chunks: list[torch.Tensor] = []
for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size):
lq_tiles = [
upscale_lr_latent_tile(
tile,
upscale_fn,
lu_dtype=lu_dtype,
scaling_factor=scaling_factor,
device=device,
)
for tile in tiles[start:stop]
]
chunk = build_chunk_latent(
lq_tiles,
seed=spec.seed + start,
dit_spec=dit_spec,
spec=spec,
device=device,
)
chunks.append(chunk.cpu())
return chunks
@torch.no_grad()
def prepare_pixel_tile_latents(
video: torch.Tensor,
plan: TilePlan,
*,
vae: torch.nn.Module,
scaling_factor: float,
dit_spec: DitSpec,
spec: SamplingSpec,
device: torch.device,
) -> list[torch.Tensor]:
"""Pixel path: bilinear tile -> KVAE encode -> initial latents (CPU) per chunk."""
tiles = extract_all_tiles(video, plan.pixel_grid)
chunks: list[torch.Tensor] = []
for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size):
enlarged = upsample_tiles_to_base(tiles[start:stop], *plan.base_hw)
lq_tiles = [
encode_pixel_tile(tile, vae, scaling_factor=scaling_factor, device=device)
for tile in enlarged
]
chunk = build_chunk_latent(
lq_tiles,
seed=spec.seed + start,
dit_spec=dit_spec,
spec=spec,
device=device,
)
chunks.append(chunk.cpu())
return chunks
def reset_scheduler_for_chunk(scheduler: Any) -> None:
"""Reset step and begin indices so each tile chunk starts at the same timestep."""
scheduler._step_index = None
scheduler.set_begin_index(0)
@torch.no_grad()
def denoise_chunk(
x: torch.Tensor,
dit: Callable[..., torch.Tensor],
scheduler: Any,
*,
dit_spec: DitSpec,
spec: SamplingSpec,
step_context: StepContext | None = None,
on_step: StepCallback | None = None,
) -> torch.Tensor:
"""Denoise one [B, T, H, W, C] chunk, resetting the configured scheduler."""
is_piflow = spec.piflow is not None
check_sampler_options(
piflow=spec.piflow,
cap_noise_timestep=spec.cap_noise_timestep,
instruct_type=dit_spec.instruct_type,
)
if not is_piflow:
# The Euler start timestep is only meaningful for the scheduler this chunk is about to
# run: re-derive it instead of baking a stale one into ``set_timesteps`` up front.
start = euler_start_timestep(
cap_noise_timestep=spec.cap_noise_timestep,
lq_noise_scale=spec.lq_noise_scale,
instruct_type=dit_spec.instruct_type,
)
sigmas = torch.linspace(start, 0.0, spec.num_steps + 1)[:-1].tolist()
scheduler.set_timesteps(sigmas=sigmas, device=x.device)
reset_scheduler_for_chunk(scheduler)
dit_fn = make_dit_fn(
dit,
latent_frames_hw=tuple(x.shape[1:4]),
patch_size=dit_spec.patch_size,
scale_factor=spec.scale_factor,
use_motion_score=dit_spec.use_motion_score and not is_piflow,
step_context=step_context,
)
with bf16_autocast(x.device):
return denoise_with_scheduler(
x,
dit_fn,
scheduler,
channels=dit_spec.in_visual_dim,
is_piflow=is_piflow,
on_step=on_step,
)
def denoise_chunks(
chunks: Sequence[torch.Tensor],
dit: Callable[..., torch.Tensor],
scheduler: Any,
*,
dit_spec: DitSpec,
spec: SamplingSpec,
device: torch.device,
step_context: StepContext | None = None,
on_step: StepCallback | None = None,
) -> list[torch.Tensor]:
"""Denoise chunks sequentially on device and return CPU latents."""
if spec.piflow is not None:
scheduler.set_timesteps(spec.piflow.nfe, device=device)
return [
denoise_chunk(
chunk.to(device),
dit,
scheduler,
dit_spec=dit_spec,
spec=spec,
step_context=step_context,
on_step=on_step,
).cpu()
for chunk in chunks
]
@torch.no_grad()
def _decode_tile(
latents: torch.Tensor,
vae: torch.nn.Module,
*,
scaling_factor: float,
device: torch.device,
) -> torch.Tensor:
"""Decode one ``[1, T', H', W', C]`` tile to uint8 ``[3, T, H, W]`` on device."""
with bf16_autocast(device):
z = (latents.to(device) / scaling_factor).permute(0, 4, 1, 2, 3)
latent = cast_to_module_dtype(vae, z)
# preserve the reference's FP32 rescale before uint8 quantization
decoded = vae.decode(latent).sample.float()
pixels = (vae.denormalize_data(decoded) / 255.0).clamp(0.0, 1.0)
return (pixels * 255.0).to(torch.uint8).squeeze(0)
def decode_chunks(
chunks: Sequence[torch.Tensor],
vae: torch.nn.Module,
*,
scaling_factor: float,
device: torch.device,
group: GroupCoordinator | None = None,
) -> list[torch.Tensor]:
"""Decode in tile order, optionally sharing independent tiles within one DP replica."""
latents = [tile for chunk in chunks for tile in chunk.split(1)]
if group is None or group.world_size == 1 or len(latents) == 1:
return [
_decode_tile(tile, vae, scaling_factor=scaling_factor, device=device).cpu()
for tile in latents
]
tiles = []
for start in range(0, len(latents), group.world_size):
index = start + group.rank_in_group
if index < len(latents):
decoded = _decode_tile(
latents[index], vae, scaling_factor=scaling_factor, device=device
)
else:
_, frames, height, width, _ = latents[start].shape
decoded = torch.zeros(
(
3,
1 + (frames - 1) * vae.temporal_factor,
height * vae.spatial_factor,
width * vae.spatial_factor,
),
device=device,
dtype=torch.uint8,
)
# bound device staging to one tile per rank, not the whole decoded video
gathered = group.all_gather(decoded.unsqueeze(0))[: len(latents) - start].cpu()
del decoded
tiles.extend(gathered.unbind(0))
return tiles
@torch.no_grad()
def stitch_tiles(tiles: Sequence[torch.Tensor], plan: TilePlan) -> torch.Tensor:
"""Hann-blend [C, T, H, W] tiles in frame chunks to bound memory."""
height, width = plan.frame_hw
scale = plan.tiling_scale
channels, frames = tiles[0].shape[:2]
out = torch.empty(
(channels, frames, height * scale, width * scale), dtype=torch.uint8
)
for start in range(0, frames, STITCH_FRAME_CHUNK):
stop = min(start + STITCH_FRAME_CHUNK, frames)
blended = stitch_tiles_hanning(
[tile[:, start:stop].float() for tile in tiles],
plan.pixel_grid,
height,
width,
scale=scale,
)
out[:, start:stop] = blended.clamp(0, 255).to(torch.uint8)
return out
@@ -0,0 +1,248 @@
# SPDX-License-Identifier: Apache-2.0
# Vendored from sr_core (parity-tested against the k6_video reference); adapted from
# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0).
"""VAE-aligned tile grids and Hann blending for Kandinsky 6 video SR."""
from __future__ import annotations
import math
from collections.abc import Mapping, Sequence
from typing import NamedTuple
import torch
from torch.nn import functional
VAE_SPATIAL_FACTOR: int = 16
VAE_TEMPORAL_FACTOR: int = 4
# trained base resolutions: visual_size -> [(H, W), ...]
RESOLUTIONS: dict[int, list[tuple[int, int]]] = {
512: [(512, 512), (512, 768), (768, 512)],
}
class TileGrid(NamedTuple):
"""Tile starts are authoritative; gaps can differ near the frame edges."""
tile_h: int
tile_w: int
tops: tuple[int, ...]
lefts: tuple[int, ...]
@property
def n_h(self) -> int:
"""Number of tile rows."""
return len(self.tops)
@property
def n_w(self) -> int:
"""Number of tile columns."""
return len(self.lefts)
@property
def total_tiles(self) -> int:
"""Total number of tiles (``n_h * n_w``)."""
return self.n_h * self.n_w
def extract_all_tiles(
video: torch.Tensor,
grid: TileGrid,
) -> list[torch.Tensor]:
"""Extract [T, C, tile_h, tile_w] views in row-major order."""
tiles: list[torch.Tensor] = []
for top in grid.tops:
for left in grid.lefts:
tile = video[:, :, top : top + grid.tile_h, left : left + grid.tile_w]
tiles.append(tile)
return tiles
def _normalize_blend(acc: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
"""Normalize Hann-weighted tiles without flooring positive weights.
Corner weights are tiny but valid; clamping them darkens tile boundaries.
Pixels outside all tiles have zero weight and remain zero."""
covered = weight > 0
safe_weight = torch.where(covered, weight, torch.ones_like(weight))
return torch.where(covered, acc / safe_weight, torch.zeros_like(acc))
def hanning_window_2d(h: int, w: int, device: torch.device) -> torch.Tensor:
"""Create a [h, w] Hann window with nonzero endpoints."""
wy = torch.hann_window(h + 2, device=device)[1:-1]
wx = torch.hann_window(w + 2, device=device)[1:-1]
return wy[:, None] * wx[None, :]
def stitch_tiles_hanning(
tiles: list[torch.Tensor],
grid: TileGrid,
original_h: int,
original_w: int,
scale: int = 1,
) -> torch.Tensor:
"""Blend row-major [C, T, H, W] tiles into a scale-enlarged frame."""
first = tiles[0]
c, t = first.shape[0], first.shape[1]
hr_tile_h, hr_tile_w = first.shape[2], first.shape[3]
device = first.device
window = hanning_window_2d(hr_tile_h, hr_tile_w, device)
window = window.unsqueeze(0).unsqueeze(0) # [1, 1, hr_tile_h, hr_tile_w]
out_h = original_h * scale
out_w = original_w * scale
pred_acc = torch.zeros(c, t, out_h, out_w, device=device)
weight_acc = torch.zeros(1, 1, out_h, out_w, device=device)
tile_idx = 0
for top in grid.tops:
for left in grid.lefts:
y = top * scale
x = left * scale
tile = tiles[tile_idx]
pred_acc[:, :, y : y + hr_tile_h, x : x + hr_tile_w] += tile * window
weight_acc[:, :, y : y + hr_tile_h, x : x + hr_tile_w] += window
tile_idx += 1
return _normalize_blend(pred_acc, weight_acc)
def axis_positions_even(
length: int, tile: int, min_overlap: float, snap: int
) -> tuple[int, ...]:
"""Cover the axis with the fewest evenly spaced, snap-aligned tiles.
Fall back to pixel alignment for unaligned sizes; latent-grid conversion
validates VAE alignment separately."""
if tile <= 0 or snap <= 0 or not 0 <= min_overlap < 1:
msg = f"invalid axis spec: tile={tile}, snap={snap}, min_overlap={min_overlap}"
raise ValueError(msg)
if tile >= length:
return (0,)
unit = snap if length % snap == 0 and tile % snap == 0 else 1
span_units = (length - tile) // unit
max_stride_units = max(1, math.floor(tile * (1.0 - min_overlap) / unit))
count = math.ceil(span_units / max_stride_units) + 1
return tuple(round(i * span_units / (count - 1)) * unit for i in range(count))
def compute_tile_grid_even(
h: int,
w: int,
tile_hw: tuple[int, int],
min_overlap: float,
snap: int,
) -> TileGrid:
"""Build a grid with evenly distributed, aligned tile starts."""
tile_h, tile_w = tile_hw
return TileGrid(
tile_h=tile_h,
tile_w=tile_w,
tops=axis_positions_even(h, tile_h, min_overlap, snap),
lefts=axis_positions_even(w, tile_w, min_overlap, snap),
)
def latent_tile_grid_from_pixel_grid(
pixel_grid: TileGrid, spatial_factor: int
) -> TileGrid:
"""Convert an aligned pixel grid to the corresponding latent grid."""
values = (pixel_grid.tile_h, pixel_grid.tile_w, *pixel_grid.tops, *pixel_grid.lefts)
if any(value % spatial_factor for value in values):
raise ValueError(
f"Pixel tile grid (tile {pixel_grid.tile_h}x{pixel_grid.tile_w}, "
f"tops={pixel_grid.tops}, lefts={pixel_grid.lefts}) not aligned to VAE spatial factor "
f"{spatial_factor}."
)
return TileGrid(
pixel_grid.tile_h // spatial_factor,
pixel_grid.tile_w // spatial_factor,
tuple(top // spatial_factor for top in pixel_grid.tops),
tuple(left // spatial_factor for left in pixel_grid.lefts),
)
def closest_base_resolution(
h: int,
w: int,
visual_size: int,
resolutions: Mapping[int, Sequence[tuple[int, int]]] = RESOLUTIONS,
) -> tuple[int, int]:
"""Choose the trained base resolution with the closest aspect ratio."""
if visual_size not in resolutions:
raise ValueError(
f"Unsupported SR visual_size={visual_size}; known sizes: {sorted(resolutions)}"
)
ratio = w / h if h else 1.0
return min(resolutions[visual_size], key=lambda hw: abs(hw[1] / hw[0] - ratio))
def upsample_tiles_to_base(
raw_tiles: list[torch.Tensor], base_h: int, base_w: int
) -> list[torch.Tensor]:
"""Bilinearly resize ``[T, C, h, w]`` tiles to ``base_h x base_w`` and return ``[T, H, W, C]``."""
return [
functional.interpolate(
tile.float(), size=(base_h, base_w), mode="bilinear", align_corners=False
).permute(0, 2, 3, 1)
for tile in raw_tiles
]
def resolve_scale_request(scale: float) -> tuple[int, float]:
"""Decompose total scale into (integer tiling scale, pixel pre-upscale)."""
requested = float(scale)
if requested == 2.25: # the one supported fractional total
return 2, 1.125
if requested in (2.0, 4.0):
return int(requested), 1.0
raise ValueError("SR supports total scales 2, 4, and 2.25")
def pre_upscale_video(
video: torch.Tensor, factor: float, spatial_multiple: int
) -> torch.Tensor:
"""Bilinearly enlarge [T, C, H, W] uint8 video, aligning target dimensions."""
if video.ndim != 4:
raise ValueError(f"video must have rank 4 [T,C,H,W], got {tuple(video.shape)}")
if factor <= 0 or spatial_multiple <= 0:
raise ValueError("factor and spatial_multiple must be positive")
height, width = video.shape[-2:]
target_h = max(
spatial_multiple, round(height * factor / spatial_multiple) * spatial_multiple
)
target_w = max(
spatial_multiple, round(width * factor / spatial_multiple) * spatial_multiple
)
resized = functional.interpolate(
video.float(),
size=(target_h, target_w),
mode="bilinear",
align_corners=False,
)
return resized.round_().clamp_(0, 255).to(torch.uint8)
def pad_to_spatial_factor(
video: torch.Tensor, spatial_factor: int
) -> tuple[torch.Tensor, tuple[int, int]]:
"""Pad bottom/right to the VAE spatial factor; return the original size.
Crop the scaled padding after SR to preserve the source framing."""
if video.ndim != 4:
raise ValueError(f"video must have rank 4 [T,C,H,W], got {tuple(video.shape)}")
height, width = video.shape[-2:]
pad_h = (-height) % spatial_factor
pad_w = (-width) % spatial_factor
if pad_h == 0 and pad_w == 0:
return video, (height, width)
padded = functional.pad(video, (0, pad_w, 0, pad_h), mode="replicate")
return padded, (height, width)
def crop_to_hw(video: torch.Tensor, hw: tuple[int, int]) -> torch.Tensor:
"""Crop [..., H, W] to hw from the top-left, undoing bottom/right padding."""
height, width = hw
return video[..., :height, :width]
@@ -0,0 +1,190 @@
# SPDX-License-Identifier: Apache-2.0
"""Streaming video / audio input and output helpers of Kandinsky 6 video SR.
Decoding follows the reference CLI (``read_video_tchw_uint8`` ->
``resample_to_target_fps`` -> ``clip_to_aligned_frames``) frame for frame, but streams:
only the frames that survive fps resampling and the 121-frame cap are converted to RGB
and kept, and decoding stops as soon as the outcome is known. PyAV is imported lazily
(it is part of the ``diffusion`` extra).
"""
import msgspec
import numpy as np
import torch
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
MAX_NUM_FRAMES,
RESAMPLE_FPS_TOLERANCE,
TARGET_FPS,
clip_to_aligned_frames,
)
SOURCE_AUDIO_SAMPLE_RATE = 44100
OUTPUT_FRAME_CHUNK = 8
class DecodedClip(msgspec.Struct, frozen=True):
"""Frames ready for SR plus the rate they play at."""
frames: torch.Tensor # [T, 3, H, W] uint8, T = 1 + 8k <= 121
fps: int # effective fps (24 when downsampled, else round(source fps))
source_fps: float
class FrameSelector:
"""Streaming form of ``resample_to_target_fps`` + the first-121-frames cap.
Feed source frames in order: :meth:`keep` says whether frame ``index`` is one of the
selected ones (call :meth:`mark_kept` for each), :meth:`enough` says when no further
frame can change the result, and :meth:`limit` trims the selection once the source
length is known. For a source faster than the target the selected indices are
``round(i * step)``, ``step = source_fps / target_fps``, of which there are
``int(total / step)`` (those below ``total``), exactly as in the reference;
otherwise every frame is selected.
"""
def __init__(
self,
source_fps: float,
*,
target_fps: int = TARGET_FPS,
max_frames: int = MAX_NUM_FRAMES,
) -> None:
self.max_frames = max_frames
self.downsample = (
abs(source_fps - target_fps) >= RESAMPLE_FPS_TOLERANCE
and source_fps > target_fps
)
self.step = source_fps / target_fps
self.effective_fps = target_fps if self.downsample else round(source_fps)
self._kept = 0
self._next_index = 0
def keep(self, index: int) -> bool:
if self._kept >= self.max_frames:
return False
return not self.downsample or index == self._next_index
def mark_kept(self) -> None:
self._kept += 1
if self.downsample:
self._next_index = round(self._kept * self.step)
def enough(self, frames_seen: int) -> bool:
"""True once ``max_frames`` are kept and the source is known to be long enough."""
if self._kept < self.max_frames:
return False
return not self.downsample or int(frames_seen / self.step) >= self.max_frames
def limit(self, total_frames: int, kept: int) -> int:
"""Number of kept frames the reference would return for a source of this length."""
if self.downsample:
kept = min(kept, int(total_frames / self.step))
return min(kept, self.max_frames)
def decode_clip(
path: str,
*,
target_fps: int = TARGET_FPS,
max_frames: int = MAX_NUM_FRAMES,
) -> DecodedClip:
"""Decode the first video stream into the aligned uint8 clip the SR model expects."""
import av
frames: list[torch.Tensor] = []
seen = 0
with av.open(str(path), mode="r") as container:
stream = container.streams.video[0]
source_fps = float(stream.average_rate or stream.base_rate or 0.0)
if source_fps <= 0:
raise ValueError(f"Video {path} reports no usable fps (got {source_fps}).")
selector = FrameSelector(
source_fps, target_fps=target_fps, max_frames=max_frames
)
if selector.effective_fps < 1:
raise ValueError(f"Video {path} has an unusable fps of {source_fps}.")
stream.thread_type = "AUTO"
for frame in container.decode(stream):
if selector.keep(seen):
array = frame.to_ndarray(format="rgb24")
frames.append(torch.from_numpy(array).permute(2, 0, 1))
selector.mark_kept()
seen += 1
if selector.enough(seen):
break
kept = selector.limit(seen, len(frames))
if kept == 0:
raise ValueError(f"Video {path} has no readable frames.")
video = torch.stack(frames[:kept]).contiguous()
return DecodedClip(
frames=clip_to_aligned_frames(video, max_frames),
fps=selector.effective_fps,
source_fps=source_fps,
)
def extract_source_audio(
path: str,
*,
sample_rate: int = SOURCE_AUDIO_SAMPLE_RATE,
max_seconds: float | None = None,
) -> np.ndarray | None:
"""Decode the first audio stream as mono float32 at ``sample_rate`` (None if absent).
``max_seconds`` stops decoding once that much audio has been produced; the result is
trimmed to exactly that length.
"""
import av
if sample_rate <= 0:
raise ValueError("sample_rate must be positive")
limit = None if max_seconds is None else round(max_seconds * sample_rate)
chunks: list[np.ndarray] = []
produced = 0
with av.open(str(path), mode="r") as container:
if not container.streams.audio:
return None
resampler = av.AudioResampler(format="fltp", layout="mono", rate=sample_rate)
for frame in container.decode(audio=0):
for resampled in resampler.resample(frame):
chunk = resampled.to_ndarray().reshape(-1)
chunks.append(chunk)
produced += chunk.shape[0]
if limit is not None and produced >= limit:
break
else:
for resampled in resampler.resample(None):
chunks.append(resampled.to_ndarray().reshape(-1))
if not chunks:
return None
audio = np.concatenate(chunks).astype(np.float32, copy=False)
return audio if limit is None else audio[:limit]
def synthetic_clip(*, height: int, width: int, frames: int) -> torch.Tensor:
"""Deterministic ``[T, 3, H, W]`` uint8 clip (warmup input; no file needed)."""
shape = (frames, 1, height, width)
ys = torch.linspace(0, 255, height).view(1, 1, height, 1).expand(shape)
xs = torch.linspace(0, 255, width).view(1, 1, 1, width).expand(shape)
ts = torch.linspace(0, 255, frames).view(frames, 1, 1, 1).expand(shape)
return (torch.cat([ys, xs, ts], dim=1) / 2 + 64).clamp(0, 255).to(torch.uint8)
def to_output_video(video: torch.Tensor) -> torch.Tensor:
"""uint8 ``[3, T, H, W]`` -> fp16 ``[1, 3, T, H, W]`` in [0, 1] that saves losslessly.
The framework writers truncate ``(x * 255)`` to uint8. fp16 halves the memory of a
large clip, but a plain ``k / 255`` stored in fp16 truncates to ``k - 1`` for over half
of the values when a consumer upcasts before multiplying. ``(k + 0.5) / 255`` stays
inside the bucket of ``k`` in every evaluation order. Converted in frame chunks to
bound the float32 temporaries.
"""
channels, frames, height, width = video.shape
out = torch.empty((1, channels, frames, height, width), dtype=torch.float16)
for start in range(0, frames, OUTPUT_FRAME_CHUNK):
chunk = video[:, start : start + OUTPUT_FRAME_CHUNK]
out[0, :, start : start + OUTPUT_FRAME_CHUNK] = (
(chunk.float() + 0.5) / 255.0
).to(torch.float16)
return out
@@ -0,0 +1,136 @@
# SPDX-License-Identifier: Apache-2.0
# Vendored from sr_core (parity-tested against the k6_video reference); adapted from
# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0).
"""Frame selection and bounded-memory output resize for Kandinsky 6 SR."""
from __future__ import annotations
import logging
import torch
from torch.nn import functional as f # noqa: N812
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
ASPECT_MISMATCH_TOLERANCE,
)
logger = logging.getLogger(__name__)
# --- Model / pipeline contract (from the reference ``constants.py``; NOT configurable) ---
# The SR model was trained on 5 s clips at 24 fps -> 121 pixel frames
# (= 1 + 8*15); feeding more frames or another rate is out of contract.
TARGET_FPS: int = 24
MAX_NUM_FRAMES: int = 121
# Treat ``|src_fps - TARGET_FPS| < tolerance`` as already-at-target (no
# resample): covers 23.98 (NTSC) and 25.00 sources.
RESAMPLE_FPS_TOLERANCE: float = 1.5
# Frames per interpolate call when downscaling the output (a 2K clip is
# ~2.4 GB as uint8, so the resize walks the clip in chunks).
RESIZE_FRAME_CHUNK: int = 8
def select_frame_indices(
total_frames: int, src_fps: float, target_fps: float
) -> list[int]:
"""Select rounded frame indices to downsample src_fps to target_fps."""
step = src_fps / target_fps
indices = [round(i * step) for i in range(int(total_frames / step))]
return [i for i in indices if i < total_frames]
def resample_to_target_fps(
video: torch.Tensor,
src_fps: float,
target_fps: int = TARGET_FPS,
) -> tuple[torch.Tensor, int]:
"""Downsample high-fps inputs; retain near-target and lower-fps inputs.
Return [T, C, H, W] video and its effective playback rate."""
if abs(src_fps - target_fps) < RESAMPLE_FPS_TOLERANCE:
return video, round(src_fps)
if src_fps > target_fps:
indices = select_frame_indices(video.shape[0], src_fps, target_fps)
logger.warning(
"Source fps %.2f > target %sfps: downsampling %s frames -> %s (fixed-stride); "
"source temporal detail beyond %sfps is discarded.",
src_fps,
target_fps,
video.shape[0],
len(indices),
target_fps,
)
return video[indices], target_fps
logger.warning(
"Source fps %.2f < target %sfps: keeping native frames (no minterpolate upsample); "
"output is mildly out of distribution.",
src_fps,
target_fps,
)
return video, round(src_fps)
def clip_to_aligned_frames(
video: torch.Tensor, max_num_frames: int = MAX_NUM_FRAMES
) -> torch.Tensor:
"""Cap the frame count, then floor-align to 1+8k; reject empty video."""
if max_num_frames > 0:
video = video[:max_num_frames]
aligned = 1 + 8 * ((video.shape[0] - 1) // 8) if video.shape[0] > 0 else 0
if aligned == 0:
msg = "Video has no readable frames."
raise ValueError(msg)
return video[:aligned]
def resize_to_target(video: torch.Tensor, target_hw: tuple[int, int]) -> torch.Tensor:
"""Downscale [C, T, H, W] uint8 video in bounded-memory chunks, without cropping."""
_c, _t, height, width = video.shape
target_h, target_w = target_hw
if (height, width) == (target_h, target_w):
return video
if target_h > height or target_w > width:
msg = (
f"target {target_w}x{target_h} exceeds the SR result {width}x{height} — that would "
f"upscale. Pick a lower tier or a higher --resolution-scale for this source."
)
raise ValueError(msg)
source_ratio, target_ratio = width / height, target_w / target_h
if abs(source_ratio - target_ratio) / target_ratio > ASPECT_MISMATCH_TOLERANCE:
logger.warning(
"Aspect mismatch: SR %sx%s is %.3f, target %sx%s is %.3f — resizing anisotropically "
"(no crop), the picture will be squeezed by %.1f%%.",
width,
height,
source_ratio,
target_w,
target_h,
target_ratio,
abs(source_ratio / target_ratio - 1) * 100,
)
channels, frames = video.shape[0], video.shape[1]
out = torch.empty((channels, frames, target_h, target_w), dtype=torch.uint8)
for start in range(0, frames, RESIZE_FRAME_CHUNK):
chunk = video[:, start : start + RESIZE_FRAME_CHUNK].permute(1, 0, 2, 3).float()
resized = f.interpolate(
chunk,
size=(target_h, target_w),
mode="bilinear",
antialias=True,
align_corners=False,
)
out[:, start : start + RESIZE_FRAME_CHUNK] = (
resized.clamp(0, 255).round().to(torch.uint8).permute(1, 0, 2, 3)
)
logger.info(
"Final output: %sx%s -> %sx%s (downscale x%.2f / x%.2f)",
width,
height,
target_w,
target_h,
width / target_w,
height / target_h,
)
return out
@@ -12,7 +12,13 @@ import inspect
import torch
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
from sglang.multimodal_gen.configs.pipeline_configs.base import TextConditioningOutput
from sglang.multimodal_gen.configs.pipeline_configs.base import (
PipelineConfig,
TextConditioningOutput,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
Kandinsky6TI2VAPipelineConfig,
)
from sglang.multimodal_gen.runtime.cache.conditioning import (
cached_encoder_call,
prefer_conditioning_cache,
@@ -106,6 +112,26 @@ def _data_parallel_text_encode(forward_fn, forward_kwargs: dict, group):
)
def _text_encoder_max_length_is_fixed(
pipeline_config: PipelineConfig, encoder_index: int
) -> bool:
"""True when ``encoder_index``'s tokenizer ``max_length`` is
architecturally fixed and must never be overridden by a request's
``max_sequence_length``.
Flux v1's encoder 0 and Kandinsky6 TI2VA's encoder 1 are both CLIP with
a fixed 77-token pooled-embedding context; overriding it corrupts CLIP
tokenization. Every other encoder (Flux v1's encoder 1 T5, Kandinsky6's
encoder 0 Reason1/Qwen, and every other pipeline's encoders) is meant to
take the request's override, so this returns False for them.
"""
if pipeline_config.is_flux_v1():
return encoder_index == 0
if isinstance(pipeline_config, Kandinsky6TI2VAPipelineConfig):
return encoder_index == 1
return False
def stack_tensors(name: str, tensors: list[torch.Tensor]) -> torch.Tensor:
base_shape = list(tensors[0].shape)
for tensor in tensors[1:]:
@@ -593,10 +619,12 @@ class TextEncodingStage(ConditionEncodingStage):
encoder_config.tokenizer_kwargs,
**text_encoder_extra_arg,
)
# Pass max_length to tokenizer if specified in the request. Flux v1 encoder 0
# is CLIP with a fixed 77-token context; overriding breaks tokenization.
is_flux_v1 = server_args.pipeline_config.is_flux_v1()
if max_length is not None and not (is_flux_v1 and i == 0):
# Pass max_length to tokenizer if specified in the request, except
# for a text encoder whose context is architecturally fixed (see
# _text_encoder_max_length_is_fixed).
if max_length is not None and not _text_encoder_max_length_is_fixed(
server_args.pipeline_config, i
):
tok_kwargs["max_length"] = max_length
text_inputs: dict = server_args.pipeline_config.tokenize_prompt(
@@ -386,7 +386,8 @@ def _verify_diffusers_model_complete(path: str) -> bool:
component_keys = [
key
for key, value in model_index.items()
if _is_diffusers_component_entry(value)
if not key.startswith("_")
and _is_diffusers_component_entry(value)
and any(item is not None for item in value)
]
if component_keys:
@@ -561,6 +562,35 @@ def get_diffusers_component_config(
lambda acc, path: acc | load_dict(path), config_file_paths, {}
)
if "_class_name" not in combined_config:
# Some checkpoints (e.g. real Kandinsky6 exports, which declare
# "uses_patched_diffusers": true) omit "_class_name" from a
# component's own config.json, relying on the parent
# model_index.json's [library, class_name] entry for that component
# subfolder instead. Fall back to that so component loaders (which
# unconditionally pop "_class_name") still resolve the right class.
model_index_path = os.path.join(
os.path.dirname(os.path.normpath(component_path)), "model_index.json"
)
component_key = os.path.basename(os.path.normpath(component_path))
if os.path.exists(model_index_path):
try:
with open(model_index_path) as f:
model_index = json.load(f)
entry = model_index.get(component_key)
class_name = (
entry[-1] if isinstance(entry, (list, tuple)) and entry else entry
)
if class_name:
combined_config["_class_name"] = str(class_name)
except Exception as exc:
logger.warning(
"Failed to backfill _class_name for %s from %s: %s",
component_path,
model_index_path,
exc,
)
quant_config = combined_config.get("quantization_config")
if quant_config is not None:
combined_config["quantization_config"] = normalize_flat_modelopt_quant_config(
@@ -736,7 +766,8 @@ def verify_model_config_and_directory(model_path: str) -> dict[str, Any]:
component_keys = [
key
for key, value in config.items()
if isinstance(value, (list, tuple))
if not key.startswith("_")
and isinstance(value, (list, tuple))
and len(value) == 2
and all(isinstance(item, str) for item in value)
]
@@ -0,0 +1,250 @@
# SPDX-License-Identifier: Apache-2.0
"""Opt-in checkpoint test; SGLANG_KANDINSKY6_TEST_SERVER_ARGS selects topology."""
import io
import json
import os
import sys
import time
from pathlib import Path
import av
import numpy as np
import pytest
import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image, ImageDraw
from safetensors.torch import save_file
from sglang.multimodal_gen.test.server.test_server_utils import ServerManager
from sglang.multimodal_gen.test.test_utils import (
extract_audio_pcm_from_video_bytes,
get_dynamic_server_port,
)
pytestmark = pytest.mark.skipif(
not os.environ.get("SGLANG_KANDINSKY6_TEST_MODEL"),
reason="set SGLANG_KANDINSKY6_TEST_MODEL for full-checkpoint HTTP tests",
)
@pytest.fixture(scope="module")
def server():
manager = ServerManager(
model=os.environ["SGLANG_KANDINSKY6_TEST_MODEL"],
port=get_dynamic_server_port(),
extra_args=(
"--performance-mode speed --attention-backend fa --warmup-mode off "
+ os.environ.get("SGLANG_KANDINSKY6_TEST_SERVER_ARGS", "--num-gpus 1")
),
)
context = manager.start()
try:
yield context
finally:
context.cleanup()
def _generate_video(server, *, image=None, size="384x256", prompt=None, **overrides):
url = f"http://127.0.0.1:{server.port}/v1/videos"
payload = {
"prompt": prompt or "A red ceramic teapot on a wooden table. Quiet jazz music.",
"size": size,
"num_frames": 17,
"fps": 24,
"num_inference_steps": 2,
"seed": 42,
} | overrides
with requests.Session() as client:
if image is None:
response = client.post(url, json=payload, timeout=120)
else:
response = client.post(
url,
data=payload,
files={"input_reference": ("reference.png", image, "image/png")},
timeout=120,
)
assert response.ok, response.text
job_url = f"{url}/{response.json()['id']}"
try:
deadline = time.monotonic() + 300
while True:
response = client.get(job_url, timeout=30)
assert response.ok, response.text
job = response.json()
assert job["status"] not in {"failed", "cancelled", "deleted"}, job
if job["status"] == "completed":
break
assert time.monotonic() < deadline, (job, server.log_tail())
time.sleep(0.5)
response = client.get(f"{job_url}/content", timeout=60)
assert response.ok, response.text
content = response.content
finally:
client.delete(job_url, timeout=30)
with av.open(io.BytesIO(content)) as video:
frames = np.stack(
[frame.to_ndarray(format="rgb24") for frame in video.decode(video=0)]
)
audio = extract_audio_pcm_from_video_bytes(content)
width, height = map(int, size.split("x"))
assert frames.shape == (17, height, width, 3)
assert frames.std() > 1, "degenerate video"
assert audio.size > 0 and np.isfinite(audio).all()
assert np.sqrt(np.mean(np.square(audio, dtype=np.float64))) > 1e-5, "silent audio"
return frames, audio
def test_repeated_requests_and_conditioning_isolation(server):
reference = Image.new("RGB", (384, 256), "white")
ImageDraw.Draw(reference).rectangle((96, 64, 288, 192), fill="red")
buffer = io.BytesIO()
reference.save(buffer, format="PNG")
first = _generate_video(server)
repeated = _generate_video(server)
for actual, expected in zip(repeated, first, strict=True):
np.testing.assert_array_equal(actual, expected)
conditioned = _generate_video(server, image=buffer.getvalue())
conditioned_repeat = _generate_video(server, image=buffer.getvalue())
for actual, expected in zip(conditioned_repeat, conditioned, strict=True):
np.testing.assert_array_equal(actual, expected)
assert not np.array_equal(conditioned[0], first[0]), (
"image conditioning was ignored"
)
_generate_video(server, size="256x384", prompt="A blue glass vase. Rain sounds.")
restored = _generate_video(server)
for actual, expected in zip(restored, first, strict=True):
np.testing.assert_array_equal(actual, expected)
def test_request_cache_dit_refresh_and_opt_out(server):
baseline = _generate_video(server, num_inference_steps=6, enable_cache_dit=False)
no_skip = _generate_video(
server,
num_inference_steps=6,
enable_cache_dit=True,
cache_dit_params={"max_warmup_steps": 6},
)
for actual, expected in zip(no_skip, baseline, strict=True):
np.testing.assert_array_equal(actual, expected)
cached_params = dict(
num_inference_steps=6,
enable_cache_dit=True,
cache_dit_params={
"max_warmup_steps": 1,
"scm_compute_bins": [1, 1, 1],
"scm_cache_bins": [1, 1, 1],
"scm_policy": "static",
},
)
first = _generate_video(server, **cached_params)
assert any(not np.array_equal(a, b) for a, b in zip(first, baseline, strict=True))
_generate_video(
server,
size="256x384",
prompt="A blue glass vase. Rain sounds.",
**cached_params,
)
repeated = _generate_video(server, **cached_params)
for actual, expected in zip(repeated, first, strict=True):
np.testing.assert_array_equal(actual, expected)
restored = _generate_video(server, num_inference_steps=6, enable_cache_dit=False)
for actual, expected in zip(restored, baseline, strict=True):
np.testing.assert_array_equal(actual, expected)
@pytest.fixture(scope="module")
def lora_adapter(tmp_path_factory):
model = os.environ["SGLANG_KANDINSKY6_TEST_MODEL"]
config_path = Path(model) / "transformer" / "config.json"
if not config_path.is_file():
config_path = Path(hf_hub_download(model, "transformer/config.json"))
config = json.loads(config_path.read_text())
layers = {
"visual_transformer_blocks.0.video_dec_block.feed_forward.net.0.proj": (
config["ff_dim"],
config["model_dim"],
),
"visual_transformer_blocks.0.audio_dec_block.feed_forward.net.2": (
config["model_dim_a"],
config["ff_dim_a"],
),
"video_time_embeddings.timestep_embedder.linear_1": (
config["time_dim"],
config["model_dim"],
),
}
generator = torch.Generator().manual_seed(827)
weights = {}
for name, (out_dim, in_dim) in layers.items():
for suffix, shape in (("A", (2, in_dim)), ("B", (out_dim, 2))):
weights[f"transformer.{name}.lora_{suffix}.weight"] = (
torch.randn(shape, generator=generator) * 0.05
)
path = tmp_path_factory.mktemp("kandinsky6_lora") / "adapter.safetensors"
save_file(weights, path)
return str(path)
@pytest.mark.parametrize("merge_mode", ["dynamic", "merge"])
def test_lora_scaling_repeatability_and_restore(server, lora_adapter, merge_mode):
base_url = f"http://127.0.0.1:{server.port}/v1"
baseline = _generate_video(server, enable_cache_dit=False)
def set_strength(strength):
response = requests.post(
f"{base_url}/set_lora",
json={
"lora_nickname": "kandinsky6_synthetic",
"lora_path": lora_adapter,
"target": "transformer",
"strength": strength,
"merge_mode": merge_mode,
},
timeout=180,
)
assert response.ok, response.text
try:
set_strength(1.0)
adapted = _generate_video(server, enable_cache_dit=False)
for actual, expected in zip(adapted, baseline, strict=True):
assert not np.array_equal(actual, expected), (
"LoRA did not change the output"
)
repeated = _generate_video(server, enable_cache_dit=False)
for actual, expected in zip(repeated, adapted, strict=True):
np.testing.assert_array_equal(actual, expected)
set_strength(0.0)
zero = _generate_video(server, enable_cache_dit=False)
for actual, expected in zip(zero, baseline, strict=True):
np.testing.assert_array_equal(actual, expected)
set_strength(0.5)
scaled = _generate_video(server, enable_cache_dit=False)
assert any(
not np.array_equal(a, b) for a, b in zip(scaled, adapted, strict=True)
)
set_strength(1.0)
restored_adapter = _generate_video(server, enable_cache_dit=False)
for actual, expected in zip(restored_adapter, adapted, strict=True):
np.testing.assert_array_equal(actual, expected)
finally:
response = requests.post(
f"{base_url}/unmerge_lora_weights",
json={"target": "transformer"},
timeout=180,
)
assert response.ok, response.text
restored_base = _generate_video(server, enable_cache_dit=False)
for actual, expected in zip(restored_base, baseline, strict=True):
np.testing.assert_array_equal(actual, expected)
if __name__ == "__main__":
sys.exit(pytest.main([__file__]))
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
"""Opt-in SR checkpoint test covering real JSON and multipart video requests."""
import io
import os
import shlex
import subprocess
import time
import av
import numpy as np
import pytest
import requests
from sglang.multimodal_gen.test.server.test_server_utils import ServerManager
from sglang.multimodal_gen.test.test_utils import (
extract_audio_pcm_from_video_bytes,
get_dynamic_server_port,
)
pytestmark = pytest.mark.skipif(
not os.environ.get("SGLANG_KANDINSKY6_SR_TEST_MODEL"),
reason="set SGLANG_KANDINSKY6_SR_TEST_MODEL for full-checkpoint HTTP tests",
)
@pytest.fixture(scope="module")
def source_videos(tmp_path_factory):
directory = tmp_path_factory.mktemp("kandinsky6_sr_source")
source = directory / "source.mp4"
silent = directory / "silent.mp4"
subprocess.run(
[
"ffmpeg",
"-hide_banner",
"-loglevel",
"error",
"-f",
"lavfi",
"-i",
"testsrc2=size=384x256:rate=24",
"-f",
"lavfi",
"-i",
"sine=frequency=440:sample_rate=44100",
"-frames:v",
"33",
"-t",
str(33 / 24),
"-c:v",
"libx264",
"-crf",
"18",
"-pix_fmt",
"yuv420p",
"-c:a",
"aac",
str(source),
],
check=True,
timeout=60,
)
subprocess.run(
[
"ffmpeg",
"-hide_banner",
"-loglevel",
"error",
"-i",
str(source),
"-c:v",
"copy",
"-an",
str(silent),
],
check=True,
timeout=60,
)
return source, silent
@pytest.fixture(scope="module")
def server(tmp_path_factory):
directory = tmp_path_factory.mktemp("kandinsky6_sr_server")
manager = ServerManager(
model=os.environ["SGLANG_KANDINSKY6_SR_TEST_MODEL"],
port=get_dynamic_server_port(),
extra_args=(
"--performance-mode speed --attention-backend fa --warmup-mode off "
f"--input-save-path {shlex.quote(str(directory / 'uploads'))} "
f"--output-path {shlex.quote(str(directory / 'outputs'))} "
+ os.environ.get("SGLANG_KANDINSKY6_SR_TEST_SERVER_ARGS", "--num-gpus 1")
),
)
context = manager.start()
try:
yield context
finally:
context.cleanup()
def _generate(
server, source, *, scale=2, tiles_batch_size=1, upload=False, has_audio=True
):
url = f"http://127.0.0.1:{server.port}/v1/videos"
payload = {
"sr_resolution_scale": scale,
"sr_tiles_batch_size": tiles_batch_size,
"num_inference_steps": 2,
"seed": 42,
}
with requests.Session() as client:
if upload:
with source.open("rb") as stream:
response = client.post(
url,
data=payload,
files={"video_reference": (source.name, stream, "video/mp4")},
timeout=120,
)
else:
response = client.post(
url, json=payload | {"video_path": str(source)}, timeout=120
)
assert response.ok, response.text
job_url = f"{url}/{response.json()['id']}"
try:
deadline = time.monotonic() + 300
while True:
response = client.get(job_url, timeout=30)
assert response.ok, response.text
job = response.json()
assert job["status"] not in {"failed", "cancelled", "deleted"}, job
if job["status"] == "completed":
break
assert time.monotonic() < deadline, (job, server.log_tail())
time.sleep(0.5)
response = client.get(f"{job_url}/content", timeout=60)
assert response.ok, response.text
content = response.content
finally:
deleted = client.delete(job_url, timeout=30)
assert deleted.ok, deleted.text
with av.open(io.BytesIO(content)) as video:
assert bool(video.streams.audio) == has_audio
assert float(video.streams.video[0].average_rate) == 24
frames = np.stack(
[frame.to_ndarray(format="rgb24") for frame in video.decode(video=0)]
)
assert frames.shape == (33, int(256 * scale), int(384 * scale), 3)
assert frames.std() > 1, "degenerate video"
audio = extract_audio_pcm_from_video_bytes(content) if has_audio else np.empty(0)
assert np.isfinite(audio).all()
if has_audio:
assert audio.size > 0
assert np.sqrt(np.mean(np.square(audio, dtype=np.float64))) > 1e-5
return frames, audio
def test_sr_requests_preserve_video_and_audio(server, source_videos):
source, silent = source_videos
first = _generate(server, source)
uploaded = _generate(server, source, upload=True)
for actual, expected in zip(uploaded, first, strict=True):
np.testing.assert_array_equal(actual, expected)
_generate(server, source, scale=4, upload=True)
_generate(server, source, scale=2.25, tiles_batch_size=2)
silent_output = _generate(server, silent, upload=True, has_audio=False)
np.testing.assert_array_equal(silent_output[0], first[0])
restored = _generate(server, source)
for actual, expected in zip(restored, first, strict=True):
np.testing.assert_array_equal(actual, expected)
@@ -0,0 +1,46 @@
# SPDX-License-Identifier: Apache-2.0
"""The real component configs (no weights) of the two official Kandinsky 6 VSR Diffusers repos, verbatim from the Hub.
* flow-matching ``kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers``: ``FlowMatchEulerDiscreteScheduler``, a 64-wide DiT head;
* 2-step pi-Flow ``kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers``: ``PiflowScheduler`` (``nfe`` 2,
``n_grid`` 10), a 640-wide DiT head.
Both repos have the same DiT tensor names (only ``out_layer`` differs in width) and byte-identical ``vae/`` and
``latent_upscaler/`` files.
"""
import json
# --- kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers (flow-matching) ---
# The dict-valued ``_kandinsky6_sr`` entry of its model_index.json and the root ``sr_config.json`` exist only in this repo.
FLOW_MODEL_INDEX = json.loads(
r"""{"_class_name":"Kandinsky6SRPipeline","_diffusers_version":"0.39.0","_kandinsky6_sr":{"format_version":1,"latent_upscaler":"latent_upscaler","portable_components":true,"source_checkpoint":"9454578e88f86dd9c40df6a8a79529a7e5a5ecc9","uses_patched_diffusers":true,"vae_name":"video-kvae"},"latent_upscaler":["diffusers","Kandinsky6SRLatentUpscalerBank"],"scheduler":["diffusers","FlowMatchEulerDiscreteScheduler"],"transformer":["diffusers","Kandinsky6SRTransformer3DModel"],"vae":["diffusers","Kandinsky6SRVAE"]}"""
)
FLOW_SR_CONFIG = json.loads(
r"""{"component_loading":"Diffusers ModelMixin wrappers","default_resolution_scale":2.25,"description":"Kandinsky 6 SR Diffusers bundle"}"""
)
FLOW_TRANSFORMER_CONFIG = json.loads(
r"""{"attention_params":{"1024":{"causal":false,"chunk":false,"glob":false,"local":false,"type":"flash","window":3},"512":{"P":0.8,"P_warmup_start":0.95,"P_warmup_steps":200,"add_sta":true,"causal":false,"chunk":false,"glob":false,"local":false,"method":"topcdf","type":"nabla","wH":7,"wT":11,"wW":7,"window":3}},"attribute_overrides":{},"axes_dims":[16,24,24],"ff_dim":7168,"in_text_dim":3584,"in_text_dim2":768,"in_visual_dim":64,"instruct_type":"hybrid_anchor","model_dim":1792,"num_text_blocks":2,"num_visual_blocks":32,"out_visual_dim":64,"patch_size":[1,1,1],"sr_params":{"cap_noise_timestep":false,"fps":24,"lq_channel_noise_scale":0.0,"lq_noise_scale":0.7,"lq_noise_type":"ddpm","scale_factor":{"512":[1.0,2.0,2.0]},"scheduler_scale":5.0,"visual_size":[512]},"time_dim":512,"use_text":false,"visual_cond":true}"""
)
FLOW_SCHEDULER_CONFIG = json.loads(
r"""{"_class_name":"FlowMatchEulerDiscreteScheduler","_diffusers_version":"0.39.0","base_image_seq_len":256,"base_shift":0.5,"invert_sigmas":false,"max_image_seq_len":4096,"max_shift":1.15,"num_train_timesteps":1000,"shift":5.0,"shift_terminal":null,"stochastic_sampling":false,"time_shift_type":"exponential","use_beta_sigmas":false,"use_dynamic_shifting":false,"use_exponential_sigmas":false,"use_karras_sigmas":false}"""
)
# --- kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers (2-step pi-Flow) ---
DISTILLED_MODEL_INDEX = json.loads(
r"""{"_class_name":"Kandinsky6SRPipeline","_diffusers_version":"0.41.0.dev0","latent_upscaler":["diffusers","Kandinsky6SRLatentUpscalerBank"],"scheduler":["diffusers","PiflowScheduler"],"transformer":["diffusers","Kandinsky6SRTransformer3DModel"],"vae":["diffusers","Kandinsky6SRVAE"]}"""
)
DISTILLED_TRANSFORMER_CONFIG = json.loads(
r"""{"attention_params":{"1024":{"causal":false,"chunk":false,"glob":false,"local":false,"type":"flash","window":3},"512":{"P":0.8,"P_warmup_start":0.95,"P_warmup_steps":0,"add_sta":true,"causal":false,"chunk":false,"glob":false,"local":false,"method":"topcdf","type":"nabla","wH":7,"wT":11,"wW":7,"window":3}},"attribute_overrides":{},"axes_dims":[16,24,24],"ff_dim":7168,"in_text_dim":3584,"in_text_dim2":768,"in_visual_dim":64,"instruct_type":"hybrid_anchor","model_dim":1792,"num_text_blocks":2,"num_visual_blocks":32,"out_visual_dim":640,"patch_size":[1,1,1],"sr_params":{"cap_noise_timestep":false,"fps":24,"lq_channel_noise_scale":0.0,"lq_noise_scale":0.7,"lq_noise_type":"ddpm","scale_factor":{"512":[1.0,2.0,2.0]},"scheduler_scale":3.5,"visual_size":[512]},"time_dim":512,"use_text":false,"visual_cond":true}"""
)
DISTILLED_SCHEDULER_CONFIG = json.loads(
r"""{"_class_name":"PiflowScheduler","_diffusers_version":"0.41.0.dev0","eps":1e-06,"final_step_size_scale":0.5,"n_grid":10,"nfe":2,"num_policy_substeps":128,"num_train_timesteps":1000,"shift":3.5}"""
)
# --- identical in both repos (the VAE config stores the *string* "None" for unused channel fields) ---
VAE_CONFIG = json.loads(
r"""{"decoder_config":{"ch":256,"ch_mult":[0.0625,1,2,4,8],"checkpoint_list":[true,true,true,true,true,true],"in_channels":"None","norm_type":"rms_norm","num_res_blocks":2,"out_ch":3,"padding_mode":"zeros","resolution":0,"temporal_compress_start_level":1,"temporal_compress_times":4,"z_channels":64},"encoder_config":{"ch":128,"ch_mult":[0.125,1,2,4,8],"checkpoint_list":[true,true,true,true,true,true],"double_z":true,"downsample_version":2,"fix_pxs":true,"in_channels":3,"norm_type":"rms_norm","num_res_blocks":2,"out_ch":"None","padding_mode":"zeros","resolution":0,"temporal_compress_start_level":1,"temporal_compress_times":4,"z_channels":64},"scaling_factor":0.910344004631042,"spatial_factor":16,"temporal_factor":4,"vae_type":"video-kvae"}"""
)
LATENT_UPSCALER_CONFIG = json.loads(
r"""{"models":[{"model":{"architecture":"multi_scale","bare_stem":true,"bottleneck_channels":null,"depthwise":false,"dims":3,"enable_x2_entry":false,"expand_ratio":1,"gradient_checkpointing":false,"grn":false,"hidden_channels":2048,"in_channels":64,"input_skip":false,"kernel_size":3,"layer_scale_init":null,"loss_weight_2x":0.0,"loss_weight_4x":1.0,"modulated_norm":true,"modulated_output_proj":true,"num_mid_blocks":3,"num_post_blocks":3,"num_pre_blocks":5,"stage_channels":[2048,1024,512],"stochastic_depth_rate":0.0,"temporal_mix":false,"temporal_padding":"replicate","upsample_mode":"pxs_v2","upsample_padding_mode":"zeros","upscale_factor":4},"target_scale":"4x"},{"model":{"architecture":"multi_scale","bare_stem":true,"bottleneck_channels":null,"depthwise":false,"dims":3,"enable_x2_entry":true,"expand_ratio":1,"gradient_checkpointing":false,"grn":false,"hidden_channels":2048,"in_channels":64,"input_skip":false,"kernel_size":3,"layer_scale_init":null,"loss_weight_2x":0.0,"loss_weight_4x":1.0,"modulated_norm":true,"modulated_output_proj":true,"num_mid_blocks":3,"num_post_blocks":3,"num_pre_blocks":5,"stage_channels":[2048,1024,512],"stochastic_depth_rate":0.0,"temporal_mix":false,"temporal_padding":"replicate","upsample_mode":"pxs_v2","upsample_padding_mode":"zeros","upscale_factor":4,"x2_adapter_blocks":2,"x2_adapter_sources":[0,4],"x2_finisher":"pxs_residual","x2_tail_mode":"private_full"},"target_scale":"2x"}],"scaling_factor":0.910344004631042}"""
)
@@ -0,0 +1,259 @@
# SPDX-License-Identifier: Apache-2.0
"""Tiny random Kandinsky 6 SR components and requests for the stage-level tests."""
import copy
from collections.abc import Callable, Mapping, Sequence
from types import SimpleNamespace
from typing import Any
import torch
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
Kandinsky6SRVAEConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
Kandinsky6SRTransformer3DModel,
)
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
Kandinsky6SRLatentUpscalerBank,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_REQUESTED_HW_KEY,
SR_TILING_SCALE_KEY,
SR_VIDEO_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
DitSpec,
SamplingSpec,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
LatentUpscaleFn,
decode_chunks,
denoise_chunks,
encode_video_to_lr_latent,
plan_tiles,
prepare_lu_tile_latents,
prepare_pixel_tile_latents,
stitch_tiles,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
RESOLUTIONS,
VAE_SPATIAL_FACTOR,
)
# Flat (legacy) transformer config with a pi-Flow DX head: 3 grids of 4 channels.
TINY_DIT = dict(
in_visual_dim=4,
in_text_dim=8,
in_text_dim2=8,
time_dim=16,
out_visual_dim=4,
patch_size=[1, 2, 2],
model_dim=32,
ff_dim=64,
num_text_blocks=0,
num_visual_blocks=2,
axes_dims=[8, 4, 4],
visual_cond=False,
instruct_type="noise",
use_text=False,
n_grid=3,
piflow_nfe=2,
piflow_num_policy_substeps=8,
piflow_final_step_size_scale=0.5,
piflow_shift=5.0,
piflow_eps=1e-6,
attribute_overrides={"instruct_type": "noise"},
sr_visual_size=[512],
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
)
TINY_KVAE = dict(
ch=8,
ch_mult=(1, 1, 2, 2, 2),
num_res_blocks=2,
z_channels=4,
temporal_compress_times=4,
norm_type="rms_norm",
)
TINY_LU_MODEL = {
"architecture": "multi_scale",
"in_channels": 4,
"hidden_channels": 8,
"num_pre_blocks": 1,
"num_mid_blocks": 2,
"num_post_blocks": 2,
"expand_ratio": 2,
"dims": 3,
"bare_stem": True,
"input_skip": False,
"modulated_norm": True,
"modulated_output_proj": True,
"temporal_padding": "replicate",
"upsample_mode": "pxs_v2",
"upsample_padding_mode": "zeros",
"enable_x2_entry": True,
"x2_adapter_blocks": 1,
"x2_tail_mode": "private_full",
"x2_finisher": "pxs_residual",
}
def _randomize(module, seed):
generator = torch.Generator().manual_seed(seed)
with torch.no_grad():
for param in module.parameters():
param.copy_(torch.randn(param.shape, generator=generator) * 0.05)
def build_components(bank_scales, dit_arch=TINY_DIT):
"""``(vae, dit, latent-upscaler bank or None, server args)`` with random weights."""
vae_config = Kandinsky6SRVAEConfig()
vae_config.update_model_arch(
dict(
vae_type="video-kvae",
encoder_config=dict(TINY_KVAE, in_channels=3),
decoder_config=dict(TINY_KVAE, out_ch=3),
scaling_factor=0.5,
)
)
vae = Kandinsky6SRVAE(vae_config).eval()
_randomize(vae, 1)
dit_config = Kandinsky6SRDitConfig()
dit_config.update_model_arch(dict(dit_arch))
dit = Kandinsky6SRTransformer3DModel(dit_config, dict(dit_arch)).eval()
_randomize(dit, 2)
bank = None
if bank_scales:
models = [
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
for scale in bank_scales
]
bank = Kandinsky6SRLatentUpscalerBank(
models,
0.5,
scales=tuple(int(scale.removesuffix("x")) for scale in bank_scales),
).eval()
_randomize(bank, 3)
# keep initialization reproducible on CPU, then exercise the native device kernels
device = get_local_torch_device()
vae.to(device)
dit.to(device)
if bank is not None:
bank.to(device)
pipeline_config = Kandinsky6SRPipelineConfig()
pipeline_config.dit_config = dit_config
server_args = SimpleNamespace(
pipeline_config=pipeline_config,
component_precisions={},
# Kandinsky6SRDenoisingStage reuses the shared DenoisingStage's cache-dit /
# torch.compile wiring (_maybe_enable_cache_dit_and_torch_compile), which reads
# these; both stay off for this fixture.
enable_breakable_cuda_graph=False,
enable_torch_compile=False,
)
return vae, dit, bank, server_args
def make_stage(stage_cls, server_args, *args, **kwargs):
stage = stage_cls(*args, **kwargs)
stage.server_args = server_args # component_uses() reads the bf16 precisions
return stage
def make_request(video, *, tiles_batch_size=1, num_steps=5):
params = Kandinsky6SRSamplingParams(
video_path="unused.mp4",
sr_resolution_scale=2,
sr_tiles_batch_size=tiles_batch_size,
num_inference_steps=num_steps,
seed=42,
)
batch = Req(sampling_params=params)
batch.extra[SR_VIDEO_KEY] = video.clone()
batch.extra[SR_TILING_SCALE_KEY] = 2
batch.extra[SR_REQUESTED_HW_KEY] = (video.shape[-2], video.shape[-1])
return batch
def random_video(frames, height, width, seed=0):
generator = torch.Generator().manual_seed(seed)
shape = (frames, 3, height, width)
return torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
@torch.no_grad()
def super_resolve(
video: torch.Tensor,
*,
vae: torch.nn.Module,
scaling_factor: float,
dit: Callable[..., torch.Tensor],
dit_spec: DitSpec,
spec: SamplingSpec,
scheduler: Any,
device: torch.device | str,
upscale_fn: LatentUpscaleFn | None = None,
lu_dtype: torch.dtype | None = None,
spatial_factor: int = VAE_SPATIAL_FACTOR,
resolutions: Mapping[int, Sequence[tuple[int, int]]] = RESOLUTIONS,
) -> torch.Tensor:
"""Tiled SR of a ``[T, C, H, W]`` uint8 video -> uint8 ``[3, T, H*s, W*s]``.
``upscale_fn`` selects the LU path; without it the pixel path runs. ``scheduler`` is the
already-resolved effective scheduler (``run_spec.effective_scheduler``) -- a real loaded
component, or one synthesized from ``spec.piflow`` / ``spec.scheduler_scale`` when none fits.
"""
device = torch.device(device)
common = dict(
visual_size=spec.visual_size,
tiling_scale=spec.tiling_scale,
tile_min_overlap=spec.tile_min_overlap,
spatial_factor=spatial_factor,
resolutions=resolutions,
)
if upscale_fn is not None:
lr_latent = encode_video_to_lr_latent(video, vae, device=device).cpu()
frame_hw = (
lr_latent.shape[-2] * spatial_factor,
lr_latent.shape[-1] * spatial_factor,
)
plan = plan_tiles(frame_hw=frame_hw, **common)
chunks = prepare_lu_tile_latents(
lr_latent,
plan,
upscale_fn=upscale_fn,
lu_dtype=lu_dtype,
scaling_factor=scaling_factor,
dit_spec=dit_spec,
spec=spec,
device=device,
spatial_factor=spatial_factor,
)
else:
plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common)
chunks = prepare_pixel_tile_latents(
video,
plan,
vae=vae,
scaling_factor=scaling_factor,
dit_spec=dit_spec,
spec=spec,
device=device,
)
denoised = denoise_chunks(
chunks, dit, scheduler, dit_spec=dit_spec, spec=spec, device=device
)
tiles = decode_chunks(denoised, vae, scaling_factor=scaling_factor, device=device)
return stitch_tiles(tiles, plan)
@@ -5,7 +5,7 @@ import sys
import types
import unittest
from pathlib import Path
from unittest.mock import patch
from unittest.mock import Mock, patch
class _FakeDBCacheConfig:
@@ -19,6 +19,7 @@ class _FakeDBCacheConfig:
class _FakeForwardPattern:
# A class (not a SimpleNamespace instance) so it is a valid type in
# annotations like List[ForwardPattern], matching the real Enum.
Pattern_0 = "Pattern_0"
Pattern_2 = "Pattern_2"
Pattern_3 = "Pattern_3"
@@ -155,8 +156,10 @@ def _install_torch_stub():
torch_nn.Module = _FakeModule
torch_dist.ProcessGroup = _FakeProcessGroup
torch_dist.ReduceOp = _FakeReduceOp
torch_dist.is_initialized = lambda: False
torch.distributed = torch_dist
torch.nn = torch_nn
torch.Tensor = object
return {
"torch": torch,
@@ -218,7 +221,50 @@ def _import_module_with_legacy_stub():
return module, stub_modules
class TestCacheDitSimilarityGroups(unittest.TestCase):
def test_independent_cfg_or_dp_peers_do_not_reduce_over_world(self):
for group in (None, "singleton-sp"):
with self.subTest(group=group):
module = _import_module_with_stub()
original = Mock(return_value=True)
manager_type = type(
"CachedContextManager", (), {"similarity": original}
)
contexts = types.ModuleType("cache_dit.caching.cache_contexts")
contexts.cache_manager = types.SimpleNamespace(
CachedContextManager=manager_type
)
module.dist.get_world_size = Mock(return_value=1)
with patch.dict(sys.modules, {contexts.__name__: contexts}):
module._patch_cache_dit_similarity()
manager = manager_type()
manager._sglang_sp_group = group
manager.similarity(
"previous", "current", threshold=0.1, parallelized=True
)
self.assertFalse(original.call_args.kwargs["parallelized"])
class TestCacheDitRefreshContext(unittest.TestCase):
def test_refresh_preserves_explicit_scm_mask_and_policy(self):
module = _import_module_with_stub()
module.refresh_context_on_transformer(
"transformer",
4,
scm_preset="fast",
steps_computation_mask=[1, 0, 1, 0],
steps_computation_policy="static",
)
self.assertEqual(module.cache_dit.steps_mask_calls, [])
self.assertEqual(
module.cache_dit.refresh_calls[0]["cache_config"],
{
"num_inference_steps": 4,
"steps_computation_mask": [1, 0, 1, 0],
"steps_computation_policy": "static",
},
)
def test_refresh_context_without_scm_preset_skips_steps_mask(self):
module = _import_module_with_stub()
module.refresh_context_on_transformer(
@@ -298,6 +344,51 @@ def _make_transformer(class_name, layers=None):
class TestBuildCustomBlockAdapter(unittest.TestCase):
def test_fsdp_subclass_preserves_native_adapter(self):
module = _import_module_with_stub()
for name, attr, multimodal, pattern in (
(
"Kandinsky6Transformer3DModel",
"visual_transformer_blocks",
True,
"Pattern_0",
),
(
"Kandinsky6Transformer3DModel",
"visual_transformer_blocks",
False,
"Pattern_2",
),
("AnimaTransformer3DModel", "transformer_blocks", False, "Pattern_3"),
):
with self.subTest(name=name, multimodal=multimodal):
blocks = ["block_0", "block_1"]
native = type(name, (), {attr: blocks})
fsdp_mixin = type("FSDPModule", (), {})
transformer = type(f"FSDP{name}", (fsdp_mixin, native), {})()
transformer.config = types.SimpleNamespace(is_multimodal=multimodal)
adapter = module._build_custom_block_adapter(
transformer, has_separate_cfg=True
)
self.assertIs(adapter.transformer, transformer)
self.assertIs(adapter.blocks, blocks)
self.assertEqual(adapter.forward_pattern, pattern)
self.assertTrue(adapter.has_separate_cfg)
def test_kandinsky6_caches_both_joint_streams(self):
module = _import_module_with_stub()
transformer = _make_transformer("Kandinsky6Transformer3DModel")
transformer.visual_transformer_blocks = ["joint_block"]
for multimodal, pattern in ((True, "Pattern_0"), (False, "Pattern_2")):
with self.subTest(multimodal=multimodal):
transformer.config = types.SimpleNamespace(is_multimodal=multimodal)
adapter = module._build_custom_block_adapter(
transformer, has_separate_cfg=True
)
self.assertIs(adapter.blocks, transformer.visual_transformer_blocks)
self.assertEqual(adapter.forward_pattern, pattern)
self.assertTrue(adapter.has_separate_cfg)
def test_builds_adapter_for_registered_class(self):
module = _import_module_with_stub()
blocks = ["block_0", "block_1"]
@@ -99,10 +99,11 @@ def _batch(
enable_cache_dit=None,
cache_dit_params=None,
is_warmup=False,
do_classifier_free_guidance=False,
) -> SimpleNamespace:
return SimpleNamespace(
is_warmup=is_warmup,
do_classifier_free_guidance=False,
do_classifier_free_guidance=do_classifier_free_guidance,
sampling_params=SimpleNamespace(
enable_cache_dit=enable_cache_dit,
cache_dit_params=cache_dit_params,
@@ -116,6 +117,7 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase):
self.stage.server_args = SimpleNamespace(
enable_breakable_cuda_graph=False,
enable_torch_compile=False,
enable_cfg_parallel=False,
)
self.stage.transformer = object()
self.stage.transformer_2 = None
@@ -127,9 +129,12 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase):
self.enable_calls = []
self.disable_calls = []
self.refresh_calls = []
self.cfg_modes = []
self.refresh_options = []
def fake_enable(transformer, config, **kwargs):
self.enable_calls.append(config)
self.cfg_modes.append(kwargs["has_separate_cfg"])
return transformer
def fake_disable(transformer):
@@ -138,6 +143,7 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase):
def fake_refresh(transformer, num_inference_steps, scm_preset=None, **kwargs):
self.refresh_calls.append(num_inference_steps)
self.refresh_options.append(kwargs)
patchers = [
patch.object(denoising_module, "enable_cache_on_transformer", fake_enable),
@@ -157,7 +163,42 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase):
self.stage._maybe_enable_cache_dit(8, _batch(enable_cache_dit=True))
self.assertEqual(len(self.enable_calls), 1)
self.assertTrue(self.stage._cache_dit_enabled)
self.assertEqual(self.stage._cache_dit_active_key, cache_dit_overrides_key({}))
self.assertEqual(
self.stage._cache_dit_active_key, (cache_dit_overrides_key({}), False)
)
def test_cfg_change_remounts_and_parallel_cfg_has_one_branch_per_rank(self):
self.stage._maybe_enable_cache_dit(8, _batch(enable_cache_dit=True))
cfg_batch = _batch(enable_cache_dit=True, do_classifier_free_guidance=True)
self.stage._maybe_enable_cache_dit(8, cfg_batch)
self.stage.server_args.enable_cfg_parallel = True
self.stage._maybe_enable_cache_dit(8, cfg_batch)
self.assertEqual(self.cfg_modes, [False, True, False])
self.assertEqual(len(self.disable_calls), 2)
def test_repeated_custom_scm_request_keeps_mask_and_policy(self):
batch = _batch(
enable_cache_dit=True,
cache_dit_params={
"scm_compute_bins": [2, 2],
"scm_cache_bins": [2, 2],
"scm_policy": "static",
},
)
self.stage._maybe_enable_cache_dit(8, batch)
self.stage._maybe_enable_cache_dit(8, batch)
self.assertEqual(len(self.enable_calls), 1)
self.assertEqual(
self.refresh_options,
[
{
"steps_computation_mask": self.enable_calls[
0
].steps_computation_mask,
"steps_computation_policy": "static",
}
],
)
def test_server_default_off_unmounts_after_request_enable(self):
self.stage._maybe_enable_cache_dit(8, _batch(enable_cache_dit=True))
@@ -0,0 +1,151 @@
# SPDX-License-Identifier: Apache-2.0
import weakref
from unittest.mock import patch
import pytest
import torch
from torch.nn.attention import SDPBackend, sdpa_kernel
from sglang.multimodal_gen.runtime.layers.parallel_conv import (
disable_spatial_parallel_decode,
)
from sglang.multimodal_gen.runtime.models.vaes.hunyuanvae import (
HunyuanVAEAttention,
HunyuanVideoMidBlock3D,
HunyuanVideoUpsampleCausal3D,
prepare_causal_attention_mask,
)
@pytest.mark.parametrize("frames", [1, 3, 7])
@pytest.mark.parametrize("batch_size,heads", [(1, 1), (2, 2)])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16, torch.float16])
def test_frame_attention_matches_dense_causal_mask(frames, batch_size, heads, dtype):
if not torch.cuda.is_available():
pytest.skip("CUDA is required for the production SDPA path")
torch.manual_seed(0)
attention = HunyuanVAEAttention(32, heads, 16, 1e-6, 4, True).cuda().to(dtype)
hidden = torch.randn(batch_size, frames * 15, 32, device="cuda", dtype=dtype)
mask = prepare_causal_attention_mask(frames, 15, dtype, hidden.device)
with torch.inference_mode():
expected = attention(hidden, attention_mask=mask)
actual = attention(hidden, num_frames=frames)
# unmasked prefixes may select a different fused SDPA reduction kernel
tolerance = max(1e-6, torch.finfo(dtype).eps)
torch.testing.assert_close(actual, expected, rtol=tolerance, atol=tolerance)
@pytest.mark.parametrize("batch_size,heads", [(1, 1), (2, 2)])
def test_bf16_frame_attention_matches_dense_with_same_math_backend(batch_size, heads):
if not torch.cuda.is_available():
pytest.skip("CUDA is required for the production SDPA path")
torch.manual_seed(0)
attention = HunyuanVAEAttention(32, heads, 16, 1e-6, 4, True).cuda().bfloat16()
hidden = torch.randn(batch_size, 105, 32, device="cuda", dtype=torch.bfloat16)
mask = prepare_causal_attention_mask(7, 15, hidden.dtype, hidden.device)
with torch.inference_mode(), sdpa_kernel(SDPBackend.MATH):
expected = attention(hidden, attention_mask=mask)
actual = attention(hidden, num_frames=7)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
def test_spatial_attention_avoids_dense_mask_and_preserves_height_shards():
torch.manual_seed(0)
attention = HunyuanVAEAttention(8, 2, 4, 1e-6, 2, True)
stage = HunyuanVideoMidBlock3D.__new__(HunyuanVideoMidBlock3D)
torch.nn.Module.__init__(stage)
stage.spatial_parallel = True
hidden = torch.randn(1, 8, 3, 5, 3)
local = hidden[:, :, :, :2]
mask = prepare_causal_attention_mask(3, 15, hidden.dtype, hidden.device)
with torch.inference_mode():
expected = attention(hidden.permute(0, 2, 3, 4, 1).flatten(1, 3), mask)
expected = expected.unflatten(1, (3, 5, 3)).permute(0, 4, 1, 2, 3)
with (
patch(
"sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.gather_variable_height",
return_value=(hidden, [2, 3]),
) as gather,
patch(
"sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.chunk_height_by_sizes",
side_effect=lambda value, sizes: value[:, :, :, : sizes[0]],
) as chunk,
patch(
"sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.prepare_causal_attention_mask",
side_effect=AssertionError(
"spatial decode must not build a dense mask"
),
),
):
actual = stage._run_attention(attention, local)
gather.assert_called_once_with(local)
assert chunk.call_args.args[1] == [2, 3]
torch.testing.assert_close(actual, expected[:, :, :, :2])
def test_frame_attention_preserves_gradient_flow():
torch.manual_seed(0)
attention = HunyuanVAEAttention(8, 2, 4, 1e-6, 2, True).double()
hidden = torch.randn(1, 12, 8, dtype=torch.float64, requires_grad=True)
mask = prepare_causal_attention_mask(3, 4, hidden.dtype, hidden.device)
expected = attention(hidden, attention_mask=mask)
expected_grad = torch.autograd.grad(expected.square().sum(), hidden)[0]
actual = attention(hidden, num_frames=3)
actual_grad = torch.autograd.grad(actual.square().sum(), hidden)[0]
torch.testing.assert_close(actual, expected)
torch.testing.assert_close(actual_grad, expected_grad)
def test_disabled_spatial_decode_preserves_dense_attention():
torch.manual_seed(0)
attention = HunyuanVAEAttention(8, 2, 4, 1e-6, 2, True)
stage = HunyuanVideoMidBlock3D.__new__(HunyuanVideoMidBlock3D)
torch.nn.Module.__init__(stage)
hidden = torch.randn(1, 8, 3, 5, 3)
with torch.inference_mode():
stage.spatial_parallel = False
expected = stage._run_attention(attention, hidden)
stage.spatial_parallel = True
with (
disable_spatial_parallel_decode(),
patch.object(attention, "forward", wraps=attention.forward) as forward,
):
actual = stage._run_attention(attention, hidden)
assert "num_frames" not in forward.call_args.kwargs
assert forward.call_args.kwargs["attention_mask"].shape == (1, 45, 45)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
def test_upsample_releases_intermediates_before_convolution():
upsample = HunyuanVideoUpsampleCausal3D(4)
hidden = torch.randn(1, 4, 3, 5, 7)
interpolate = torch.nn.functional.interpolate
buffers = []
def track_interpolate(*args, **kwargs):
result = interpolate(*args, **kwargs)
buffers.append(weakref.ref(result))
return result
def check_input(value):
assert all(buffer() is None for buffer in buffers)
return value
with (
torch.inference_mode(),
patch(
"sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.F.interpolate",
side_effect=track_interpolate,
),
patch.object(upsample.conv, "forward", side_effect=check_input),
):
actual = upsample(hidden)
expected = torch.cat(
(
interpolate(hidden[:, :, 0], scale_factor=2, mode="nearest").unsqueeze(2),
interpolate(hidden[:, :, 1:].contiguous(), scale_factor=2, mode="nearest"),
),
dim=2,
)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@@ -1,14 +1,51 @@
from unittest.mock import patch
import pytest
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import (
IpcA2AState,
_peer_cuda_device,
_Unsupported,
ipc_a2a_ready,
)
_IPC = "sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a"
@pytest.mark.parametrize("rank", [0, 1])
@pytest.mark.parametrize("nvlink", [False, True])
def test_ipc_requires_nvlink_before_mapping_peer_memory(monkeypatch, rank, nvlink):
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3,1")
state = IpcA2AState()
with (
patch(f"{_IPC}.dist.get_rank", return_value=rank),
patch(f"{_IPC}.torch.cuda.current_device", return_value=rank),
patch(f"{_IPC}._peer_cuda_device", return_value=1 - rank),
patch(
"sglang.multimodal_gen.runtime.platforms.current_platform.is_full_nvlink",
return_value=nvlink,
) as topology,
patch(f"{_IPC}.torch.cuda.can_device_access_peer", return_value=True) as peer,
patch("ctypes.CDLL") as cudart,
patch(f"{_IPC}.load_ipc_a2a_sync") as kernels,
patch(f"{_IPC}.torch.zeros"),
patch.object(state, "_share"),
):
if nvlink:
state.init(object())
assert state.inited
peer.assert_called_once_with(rank, 1 - rank)
kernels.assert_called_once()
else:
with pytest.raises(_Unsupported, match="NVLink"):
state.init(object())
assert not state.inited
peer.assert_not_called()
cudart.assert_not_called()
kernels.assert_not_called()
topology.assert_called_once_with([1, 3])
def test_reinitializes_ipc_transport_for_replaced_process_group():
state = IpcA2AState()
old_group = object()
@@ -0,0 +1,114 @@
# SPDX-License-Identifier: Apache-2.0
"""Checkpoint architecture validation and native parameter-name mapping."""
import pytest
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
Kandinsky6ArchConfig,
Kandinsky6VideoAudioConfig,
)
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
Kandinsky6AudioVAEConfig,
)
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
@pytest.mark.parametrize(
"overrides,expected",
[
({}, (48, 32, 128, (8, 8, 8))),
({"ff_dim_a": 96, "axes_dims_a": (4, 4, 4)}, (48, 32, 96, (4, 4, 4))),
({"time_dim_a": 16, "fix_modulation": True}, (48, 16, 128, (8, 8, 8))),
({"time_dim_a": 16, "is_multimodal": False}, (48, 16, 128, (8, 8, 8))),
],
)
def test_audio_dimension_inheritance_and_overrides(overrides, expected):
arch = Kandinsky6ArchConfig(
model_dim=48, time_dim=32, ff_dim=128, axes_dims=(8, 8, 8), **overrides
)
assert (
arch.model_dim_a,
arch.time_dim_a,
arch.ff_dim_a,
arch.axes_dims_a,
) == expected
assert (arch.hidden_size, arch.num_attention_heads) == (48, 2)
@pytest.mark.parametrize(
"overrides,error",
[
({"model_dim_a": 50, "axes_dims_a": (4, 4, 4)}, "model_dim_a"),
({"time_dim_a": 16}, "time_dim_a"),
],
)
def test_incompatible_audio_dimensions_are_rejected(overrides, error):
with pytest.raises(ValueError, match=error):
Kandinsky6ArchConfig(
model_dim=48, time_dim=32, axes_dims=(8, 8, 8), **overrides
)
@pytest.mark.parametrize(
"source,target",
[
(
"video_text_transformer_blocks.0.feed_forward.net.0.proj.weight",
"video_text_transformer_blocks.0.feed_forward.mlp.fc_in.weight",
),
(
"video_text_transformer_blocks.3.attn.to_q.weight",
"video_text_transformer_blocks.3.self_attention.to_q.weight",
),
(
"video_time_embeddings.timestep_embedder.linear_1.weight",
"video_time_embeddings.in_layer.weight",
),
(
"visual_transformer_blocks.0.video_dec_block.feed_forward.net.2.bias",
"visual_transformer_blocks.0.videoT.feed_forward.mlp.fc_out.bias",
),
],
)
def test_checkpoint_name_mapping(source, target):
map_name = get_param_names_mapping(Kandinsky6ArchConfig().param_names_mapping)
assert map_name(source)[0] == target
@pytest.mark.parametrize("n_grid", [1, 10], ids=["sft", "distilled"])
def test_checkpoint_head_width_does_not_change_latent_width(n_grid):
config = Kandinsky6VideoAudioConfig()
config.update_model_arch(
{
"in_visual_dim": 16,
"out_visual_dim": 16 * n_grid,
"in_audio_dim": 40,
"out_audio_dim": 40 * n_grid,
"model_dim_a": 2048,
"ff_dim_a": 7168,
"fix_modulation": True,
"scale_factor": [1.0, 2.0, 2.0],
}
)
arch = config.arch_config
assert (arch.in_channels, arch.out_channels, arch.num_channels_latents) == (
16,
16 * n_grid,
16,
)
assert (arch.in_audio_dim, arch.out_audio_dim) == (40, 40 * n_grid)
assert tuple(arch.scale_factor) == (1.0, 2.0, 2.0)
@pytest.mark.parametrize("scaling_factor", [0.5302, 0.417])
def test_audio_vae_scaling_factor_comes_from_the_checkpoint(scaling_factor):
config = Kandinsky6AudioVAEConfig()
config.update_model_arch(
{
"mode": "44k",
"sample_rate": 44100,
"downsample_factor": 1024,
"scaling_factor": scaling_factor,
}
)
assert config.arch_config.scaling_factor == scaling_factor
@@ -0,0 +1,96 @@
# SPDX-License-Identifier: Apache-2.0
"""GAP 2 regression: ``Kandinsky6AudioVAE`` must not re-renormalize an
already-loaded (final) checkpoint's audio-VAE Conv1d weights.
The diffusers reference calls ``MMAudioVAE.remove_weight_norm()`` exactly
once, inside ``MMAudioVAE.__init__`` -- BEFORE ``from_pretrained`` overwrites
those freshly-constructed (random) weights with the checkpoint's real,
already-final values, so a real checkpoint's own weights are never
renormalized. The pre-fix SGLang port instead triggered the same
RMS-renormalization lazily, on the first ``decode()`` call -- i.e. AFTER a
real checkpoint's weights were loaded -- corrupting them (measured by the
audit at ~9.7% decoded-waveform rel-L2 error against the diffusers
reference).
"""
from __future__ import annotations
import torch
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
Kandinsky6AudioVAEConfig,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio import (
Kandinsky6AudioVAE,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.mmaudio_vae import (
MMAudioVAE,
)
# A small, self-contained BigVGANV2 config (not a real checkpoint's) just so
# Kandinsky6AudioVAE.__init__ can build the whole module tree on CPU quickly;
# none of these tests call vocode()/wrapped_decode(), only the mel-level
# decode() where the audited bug lives.
_TINY_VOCODER_CONFIG = {
"resblock": "1",
"num_mels": 80, # matches mode="16k"'s data_dim
"upsample_rates": [2],
"upsample_kernel_sizes": [4],
"upsample_initial_channel": 8,
"resblock_kernel_sizes": [3],
"resblock_dilation_sizes": [[1, 1, 1]],
"activation": "snake",
"snake_logscale": False,
}
def _tiny_audio_vae_config() -> Kandinsky6AudioVAEConfig:
config = Kandinsky6AudioVAEConfig()
config.arch_config.mode = "16k"
config.arch_config.vocoder_config = dict(_TINY_VOCODER_CONFIG)
return config
def test_mmaudio_vae_remove_weight_norm_is_idempotent():
torch.manual_seed(0)
vae = MMAudioVAE(mode="16k", need_encoder=False)
vae.remove_weight_norm()
latents = torch.randn(1, vae.embed_dim, 4)
mel_once = vae.decode(latents.clone())
# A second, redundant call must be a true no-op: it must not re-derive
# (and thus change) already-normalized weights.
vae.remove_weight_norm()
mel_twice = vae.decode(latents.clone())
assert torch.equal(mel_once, mel_twice)
def test_audio_vae_weights_are_not_renormalized_after_checkpoint_load():
"""Simulates VAELoader's real flow: construct, then load_state_dict with
a checkpoint's own (already-final) weights -- decode() must use those
weights as-is, matching the diffusers reference's "already normalized,
no-op at inference" contract, not silently re-derive a different set of
weights from them.
"""
torch.manual_seed(0)
config = _tiny_audio_vae_config()
vae = Kandinsky6AudioVAE(config)
assert vae.vae._weights_normalized is True
# `load_state_dict(vae.state_dict())` mirrors VAELoader's real call
# (`vae.load_state_dict(loaded, strict=strict_load)`) with the module's
# own current weights standing in for "a checkpoint's own weights" --
# exactly what happens when a real checkpoint's stored tensors are
# loaded: no code path scales or otherwise disturbs them afterward.
vae.load_state_dict(vae.state_dict())
latents = torch.randn(1, vae.vae.embed_dim, 4)
mel_a = vae.decode(latents.clone())
# An explicit extra call (this wrapper's own public API) must also be a
# no-op now, not a second renormalization of the loaded weights.
vae.remove_weight_norm()
mel_b = vae.decode(latents.clone())
assert torch.equal(mel_a, mel_b)
@@ -0,0 +1,502 @@
# SPDX-License-Identifier: Apache-2.0
"""BF16 forward, attention dispatch, CUDA graph and cache lifetime regressions."""
from __future__ import annotations
import importlib.util
import os
from types import SimpleNamespace
import pytest
import torch
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
Kandinsky6VideoAudioConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
Kandinsky6TI2VAPipelineConfig,
)
from sglang.multimodal_gen.runtime.cache.cache_dit_integration import (
CacheDitConfig,
disable_cache_on_transformer,
enable_cache_on_transformer,
refresh_context_on_transformer,
)
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGBranch, CFGPolicy
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.layers.attention.selector import (
global_force_attn_backend_context_manager,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
Kandinsky6Attention,
Kandinsky6Modulation,
Kandinsky6QKNorm,
Kandinsky6TimeEmbeddings,
Kandinsky6Transformer3DModel,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.denoising import (
Kandinsky6DenoisingStage,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.server_args import get_global_server_args
from sglang.multimodal_gen.runtime.utils import precision
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
# Kandinsky6's feed-forward uses ColumnParallelLinear/RowParallelLinear, which
# need a (size-1) TP group to construct -- matches test_kandinsky6_sr_stages.py's
# fixture of the same name.
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29513",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
def _randomize(module: torch.nn.Module, seed: int) -> None:
generator = torch.Generator().manual_seed(seed)
with torch.no_grad():
for param in module.parameters():
param.copy_(torch.randn(param.shape, generator=generator) * 0.05)
def test_time_embeddings_forward_under_bf16_linear_weights():
# Minimal, pinpoint reproduction of the review comment: a bf16-weight
# Kandinsky6TimeEmbeddings fed the (always-fp32) sinusoidal embedding must not
# raise, and must return bf16 (the Linear weights' dtype).
module = Kandinsky6TimeEmbeddings(model_dim=16, time_dim=8)
_randomize(module, seed=0)
module = module.to(torch.bfloat16)
out = module(torch.tensor([0.0, 500.0, 999.0]))
assert out.dtype == torch.bfloat16
assert torch.isfinite(out.float()).all()
def test_modulation_forward_under_bf16_linear_weights():
# Kandinsky6Modulation (the AdaLN projection every block/out-layer uses) has the
# identical autocast(dtype=float32)-doesn't-cast-weights bug; fed an fp32 time
# embedding with bf16 weights, it must not raise either.
module = Kandinsky6Modulation(time_dim=8, model_dim=16, num_params=9)
module = module.to(torch.bfloat16)
out = module(torch.randn(2, 8)) # fp32 input, as Kandinsky6TimeEmbeddings used to
assert out.dtype == torch.bfloat16
assert torch.isfinite(out.float()).all()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA attention")
@pytest.mark.parametrize("backend", ["FA", "TORCH_SDPA", "SAGE_ATTN_3"])
@pytest.mark.parametrize("lengths", [(7, None), (129, None), (193, 65), (17, 129)])
@torch.no_grad()
def test_attention_backend_dispatch_and_repeated_forward(monkeypatch, backend, lengths):
if backend == "FA" and torch.cuda.get_device_capability()[0] == 12:
pytest.skip("the platform currently resolves FA to SDPA on SM12.x")
if backend == "SAGE_ATTN_3":
if importlib.util.find_spec("sageattn3") is None:
pytest.skip("requires the optional SageAttention3 extension")
if torch.cuda.get_device_capability() not in {(12, 0), (12, 1)}:
pytest.skip("upstream SageAttention3 requires SM120 or SM121")
monkeypatch.setattr(
precision._mixed_precision_state,
"state",
precision.MixedPrecisionState(param_dtype=torch.bfloat16),
)
query_len, key_len = lengths
selected = AttentionBackendEnum[backend]
with global_force_attn_backend_context_manager(selected):
layer = Kandinsky6Attention(
256,
128,
Kandinsky6Transformer3DModel._supported_attention_backends,
kv_dim=128 if key_len is not None else None,
is_cross_attention=key_len is not None,
).eval()
assert layer.attention.backend == selected, "must not silently fall back"
_randomize(layer, seed=419)
layer.query_norm.weight.fill_(1)
layer.key_norm.weight.fill_(1)
layer.cuda().bfloat16()
torch.manual_seed(420)
query = torch.randn(1, query_len, 256, device="cuda", dtype=torch.bfloat16)
key = (
torch.randn(1, key_len, 128, device="cuda", dtype=torch.bfloat16)
if key_len is not None
else None
)
original_query = query.clone()
original_key = key.clone() if key is not None else None
implementation = layer.attention.attn_impl
forward = implementation.forward
calls = 0
def counted_forward(*args, **kwargs):
nonlocal calls
calls += 1
return forward(*args, **kwargs)
monkeypatch.setattr(implementation, "forward", counted_forward)
with set_forward_context(current_timestep=0, attn_metadata=None):
first = layer(query, key)
repeated = layer(query, key)
assert calls == 2, "must execute the requested backend, not a masked bypass"
assert first.shape == query.shape and first.dtype == query.dtype
assert torch.isfinite(first).all()
torch.testing.assert_close(repeated, first, rtol=0, atol=0)
torch.testing.assert_close(query, original_query, rtol=0, atol=0)
if key is not None:
torch.testing.assert_close(key, original_key, rtol=0, atol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32])
@pytest.mark.parametrize("width", [16, 64, 128, 256])
@pytest.mark.parametrize("eps", [None, 1e-5])
@torch.no_grad()
def test_qk_norm_exact_output_and_parameter_storage(dtype, width, eps):
torch.manual_seed(943)
norm = Kandinsky6QKNorm(width, eps=eps, device="cuda", dtype=dtype)
norm.weight.normal_()
reference = torch.nn.RMSNorm(width, eps=eps, device="cuda", dtype=dtype)
reference.load_state_dict(norm.state_dict())
parameter = norm.weight
pointer = parameter.data_ptr()
for scale in (0.0, 0.001, 1.0, 100.0):
x = (torch.randn(4096, width * 2, device="cuda", dtype=dtype) * scale)[:, ::2]
expected = reference(x.float()).to(dtype)
actual = norm(x.float()).to(dtype)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
assert norm.weight is parameter
assert norm.weight.data_ptr() == pointer
assert norm.weight.dtype == dtype
norm.cpu().cuda()
torch.testing.assert_close(norm(x.float()).to(dtype), expected, rtol=0, atol=0)
torch.testing.assert_close(norm.weight, reference.weight, rtol=0, atol=0)
@pytest.mark.parametrize("device", ["cpu", "cuda"])
def test_qk_norm_autograd_preserves_native_math(device):
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("requires CUDA")
torch.manual_seed(944)
norm = Kandinsky6QKNorm(128, device=device, dtype=torch.bfloat16)
reference = torch.nn.RMSNorm(128, device=device, dtype=torch.bfloat16)
with torch.no_grad():
norm.weight.normal_()
reference.load_state_dict(norm.state_dict())
x = torch.randn(7, 128, device=device, requires_grad=True)
ref_x = x.detach().clone().requires_grad_()
expected = reference(ref_x)
actual = norm(x)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
actual.sum().backward()
expected.sum().backward()
torch.testing.assert_close(norm.weight.grad, reference.weight.grad, rtol=0, atol=0)
torch.testing.assert_close(x.grad, ref_x.grad, rtol=0, atol=0)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA graphs")
@torch.no_grad()
def test_qk_norm_graph_replay_observes_weight_updates():
norm = Kandinsky6QKNorm(128, device="cuda", dtype=torch.bfloat16)
x = torch.randn(17, 128, device="cuda", dtype=torch.bfloat16).float()
stream = torch.cuda.Stream()
stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(stream):
for _ in range(3):
norm(x)
torch.cuda.current_stream().wait_stream(stream)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = norm(x).to(torch.bfloat16)
for _ in range(2):
norm.weight.add_(0.125)
graph.replay()
expected = torch.nn.functional.rms_norm(
x, norm.normalized_shape, norm.weight, norm.eps
).to(torch.bfloat16)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
def _tiny_multimodal_arch_kwargs() -> dict:
"""Small but structurally real T2VA config: is_multimodal=True (every released
Kandinsky6 checkpoint), audio tower dims intentionally different from the video
tower's to exercise both ``Kandinsky6TimeEmbeddings``/``Kandinsky6Modulation``
instances the fused block owns."""
return dict(
model_dim=32,
time_dim=16,
ff_dim=64,
axes_dims=(8, 4, 4),
in_visual_dim=4,
out_visual_dim=4,
in_text_dim=8,
in_text_dim2=8,
in_audio_dim=4,
num_text_blocks=1,
num_visual_blocks=1,
patch_size=(1, 1, 1),
visual_cond=False,
visual_token_type_num_embeddings=0,
is_multimodal=True,
model_dim_a=16,
time_dim_a=16,
ff_dim_a=32,
axes_dims_a=(4, 2, 2),
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA graphs")
@pytest.mark.parametrize("use_cfg", [False, True])
@torch.no_grad()
def test_bcg_joint_cfg_output_lifetime_and_text_shape_fallback(monkeypatch, use_cfg):
args = get_global_server_args()
args.attention_backend = "fa"
args.pipeline_config = Kandinsky6TI2VAPipelineConfig()
config = Kandinsky6VideoAudioConfig()
config.update_model_arch(_tiny_multimodal_arch_kwargs())
with torch.device("meta"):
dit = Kandinsky6Transformer3DModel(config, {}).eval()
torch.manual_seed(7)
dit.load_state_dict(
{
name: (torch.randn(param.shape, device="cuda") * 0.05).to(torch.bfloat16)
for name, param in dit.named_parameters()
},
assign=True,
)
dit.post_load_weights()
stage = Kandinsky6DenoisingStage(dit, scheduler=None)
batch = SimpleNamespace(
is_warmup=False,
do_classifier_free_guidance=use_cfg,
guidance_scale=5.0,
cfg_normalization=0.0,
guidance_rescale=0.0,
)
inputs = dict(
transformer=dit,
batch=batch,
server_args=args,
step_index=0,
video_input=torch.randn(1, 2, 2, 2, 4, device="cuda", dtype=torch.bfloat16),
audio_input=torch.randn(1, 5, 4, device="cuda", dtype=torch.bfloat16),
t_expand=torch.tensor([500.0], device="cuda"),
visual_rope_pos=[torch.arange(2, device="cuda") for _ in range(3)],
scale_factor=(1.0, 1.0, 1.0),
sparse_params=None,
visual_token_type_ids=None,
)
def policy(text_len):
return CFGPolicy(
branches=[
CFGBranch(
name,
conditional,
{
"encoder_hidden_states": torch.randn(
1, text_len, 8, device="cuda", dtype=torch.bfloat16
),
"pooled_projections": torch.randn(
1, 8, device="cuda", dtype=torch.bfloat16
),
"text_rope_pos": torch.arange(text_len, device="cuda"),
},
)
for name, conditional in (
("conditional", True),
("unconditional", False),
)
if conditional or use_cfg
]
)
policies = [policy(3), policy(3), policy(5)]
references = [
stage._predict_joint_velocity(cfg_policy=p, **inputs) for p in policies
]
args.enable_breakable_cuda_graph = True
batch.is_warmup = True
stage._predict_joint_velocity(cfg_policy=policies[0], **inputs)
batch.is_warmup = False
runner = stage._bcg_runners[id(dit)]
replay = runner.replay
replay_calls = 0
def record_replay(entry, kwargs):
nonlocal replay_calls
replay_calls += 1
return replay(entry, kwargs)
monkeypatch.setattr(runner, "replay", record_replay)
try:
assert len(runner.entries) == 1, "eager fallback is not a capture test"
assert all(entry.num_segments > 0 for entry in runner.entries.values())
outputs = [
stage._predict_joint_velocity(cfg_policy=p, **inputs) for p in policies
]
assert replay_calls == (4 if use_cfg else 2)
# validate after every replay: neither CFG nor subsequent requests may
# overwrite a previously returned video/audio tensor
for actual, expected in zip(outputs, references, strict=True):
for tensor, reference in zip(actual, expected, strict=True):
torch.testing.assert_close(tensor, reference, rtol=0, atol=0)
assert len(runner.entries) == 1, "serving must not capture unseen text lengths"
finally:
runner.reset()
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason=(
"the DiT's fused LayerNormScaleShift/RMSNormScaleShift requires a real "
"CUDA/XPU tensor on any CUDA-capable host (see module docstring); bf16 "
"numerics are only meaningful on real hardware anyway"
),
)
def test_kandinsky6_dit_real_bf16_forward_does_not_crash():
"""Real (non-mocked) forward pass of a tiny Kandinsky6Transformer3DModel whose
weights are bf16 -- the exact configuration the SGLang PR review comment flagged
as broken (a fp32 sinusoidal embedding feeding a bf16 Linear)."""
device = torch.device("cuda")
config = Kandinsky6VideoAudioConfig()
arch_kwargs = _tiny_multimodal_arch_kwargs()
config.update_model_arch(dict(arch_kwargs))
dit = Kandinsky6Transformer3DModel(config, dict(arch_kwargs)).eval()
_randomize(dit, seed=1)
dit = dit.to(device=device, dtype=torch.bfloat16)
batch, duration, height, width = 1, 2, 2, 2
text_len, audio_len = 3, 5
hidden_states = torch.randn(
batch, duration, height, width, 4, dtype=torch.bfloat16, device=device
)
hidden_states_audio = torch.randn(
batch, audio_len, 4, dtype=torch.bfloat16, device=device
)
encoder_hidden_states = torch.randn(
batch, text_len, 8, dtype=torch.bfloat16, device=device
)
pooled_projections = torch.randn(batch, 8, dtype=torch.bfloat16, device=device)
# Scheduler timesteps are plain fp32 (not cast to the model's weight dtype by the
# caller), same as every real sampling loop -- exactly the dtype that used to
# mismatch the bf16 Linear.
timestep = torch.tensor([500.0], device=device)
visual_rope_pos = (
torch.arange(duration, device=device),
torch.arange(height, device=device),
torch.arange(width, device=device),
)
text_rope_pos = torch.arange(text_len, device=device)
video_out, audio_out = dit(
hidden_states=hidden_states,
hidden_states_audio=hidden_states_audio,
encoder_hidden_states=encoder_hidden_states,
timestep=timestep,
pooled_projections=pooled_projections,
visual_rope_pos=visual_rope_pos,
text_rope_pos=text_rope_pos,
)
assert video_out.shape == (batch, duration, height, width, 4)
assert audio_out.shape == (batch, audio_len, 4)
assert video_out.dtype == torch.bfloat16
assert audio_out.dtype == torch.bfloat16
assert torch.isfinite(video_out.float()).all()
assert torch.isfinite(audio_out.float()).all()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("separate_cfg", [False, True])
@torch.no_grad()
def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg):
config = Kandinsky6VideoAudioConfig()
arch = _tiny_multimodal_arch_kwargs() | {"num_visual_blocks": 4}
config.update_model_arch(arch)
dit = Kandinsky6Transformer3DModel(config, {}).eval()
_randomize(dit, seed=11)
dit = dit.to(device="cuda", dtype=torch.bfloat16)
torch.manual_seed(13)
inputs = dict(
hidden_states=torch.randn(1, 2, 2, 2, 4, device="cuda", dtype=torch.bfloat16),
hidden_states_audio=torch.randn(1, 5, 4, device="cuda", dtype=torch.bfloat16),
encoder_hidden_states=torch.randn(1, 3, 8, device="cuda", dtype=torch.bfloat16),
pooled_projections=torch.randn(1, 8, device="cuda", dtype=torch.bfloat16),
visual_rope_pos=[torch.arange(2, device="cuda") for _ in range(3)],
text_rope_pos=torch.arange(3, device="cuda"),
)
def run_steps():
outputs = []
for step in range(3):
for branch in range(2 if separate_cfg else 1):
outputs.append(
dit(
**(
inputs
| {
"pooled_projections": inputs["pooled_projections"]
+ branch
}
),
timestep=torch.tensor([500.0 - step * 100], device="cuda"),
)
)
return outputs
def assert_exact(actual, expected):
for pair, reference in zip(actual, expected, strict=True):
for tensor, target in zip(pair, reference, strict=True):
torch.testing.assert_close(tensor, target, rtol=0, atol=0)
reference = run_steps()
enable_cache_on_transformer(
dit,
CacheDitConfig(
enabled=True,
num_inference_steps=3,
Fn_compute_blocks=1,
Bn_compute_blocks=1,
max_warmup_steps=1,
steps_computation_mask=[1, 1, 1],
steps_computation_policy="static",
),
has_separate_cfg=separate_cfg,
)
try:
assert_exact(run_steps(), reference)
cached_requests = []
for _ in range(2):
refresh_context_on_transformer(
dit,
3,
steps_computation_mask=[1, 0, 1],
steps_computation_policy="static",
)
outputs = run_steps()
for video, audio in outputs:
assert torch.isfinite(video).all()
assert torch.isfinite(audio).all()
assert dit._context_manager.get_cached_steps() == [1]
if separate_cfg:
assert dit._context_manager.get_cfg_cached_steps() == [1]
cached_requests.append(outputs)
assert_exact(*cached_requests)
finally:
disable_cache_on_transformer(dit)
assert_exact(run_steps(), reference)
@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
import pytest
import torch
from safetensors.torch import load_file, save_file
from transformers import CLIPTextConfig, CLIPTextModel
from sglang.multimodal_gen.configs.models.encoders import (
CLIPTextConfig as NativeCLIPTextConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.flux import FluxPipelineConfig
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
Kandinsky6TI2VAPipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.krea2 import Krea2PipelineConfig
from sglang.multimodal_gen.configs.pipeline_configs.stablediffusion3 import (
StableDiffusion3PipelineConfig,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
ComponentLoader,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.text_encoder_loader import (
TextEncoderLoader,
)
from sglang.multimodal_gen.runtime.models.encoders.clip import (
CLIPTextModel as NativeCLIPTextModel,
)
from sglang.multimodal_gen.runtime.models.encoders.clip import (
CLIPTextModelWithProjection,
)
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import (
Kandinsky6TI2VAPipeline,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
_text_encoder_max_length_is_fixed,
)
from sglang.srt.runtime_context import get_parallel
@pytest.mark.parametrize(
"config_cls,fixed",
[
(FluxPipelineConfig, [True, False]),
(Kandinsky6TI2VAPipelineConfig, [False, True]),
(Krea2PipelineConfig, [False, False]),
(StableDiffusion3PipelineConfig, [False, False, False]),
],
)
def test_request_max_length_preserves_fixed_encoder_context(config_cls, fixed):
config = config_cls()
assert [
_text_encoder_max_length_is_fixed(config, index) for index in range(len(fixed))
] == fixed
def test_clip_uses_shared_native_loader():
assert "text_encoder_2" not in Kandinsky6TI2VAPipeline.component_loaders
assert "text_encoder_2" in Kandinsky6TI2VAPipelineConfig().native_only_components
for architecture in ("CLIPTextModel", "Qwen2_5_VLForConditionalGeneration"):
loader = ComponentLoader.for_component_type(
"text_encoder", "transformers", architecture
)
assert type(loader) is TextEncoderLoader
@pytest.mark.parametrize("eos_token_id", [2, 31])
@pytest.mark.parametrize("masked", [False, True])
def test_clip_bare_and_prefixed_checkpoints_match(tmp_path, eos_token_id, masked):
torch.manual_seed(42)
device = "cuda" if torch.cuda.is_available() else "cpu"
reference = (
CLIPTextModel(
CLIPTextConfig(
vocab_size=32,
hidden_size=16,
intermediate_size=32,
num_hidden_layers=2,
num_attention_heads=2,
max_position_embeddings=8,
bos_token_id=1,
eos_token_id=eos_token_id,
)
)
.to(device)
.eval()
)
config = NativeCLIPTextConfig()
config.update_model_arch(reference.config.to_dict())
config.post_diffusers_config_update()
weights = {name: value.cpu() for name, value in reference.state_dict().items()}
save_file(weights, str(tmp_path / "prefixed.safetensors"))
save_file(
{name.removeprefix("text_model."): value for name, value in weights.items()},
str(tmp_path / "bare.safetensors"),
)
ids = torch.tensor([[1, 5, 31, 31, 0], [1, 6, 7, 31, 0]], device=device)
mask = (
torch.tensor([[1, 1, 1, 0, 0], [1, 1, 1, 1, 0]], device=device)
if masked
else None
)
with get_parallel().override(
tp_size=1, tp_rank=0, tp_group=None, attn_tp_size=1, attn_tp_rank=0
):
models = []
for checkpoint in ("prefixed", "bare"):
model = NativeCLIPTextModel(config).to(device).eval()
loaded = model.load_weights(
load_file(str(tmp_path / f"{checkpoint}.safetensors")).items()
)
assert loaded == set(dict(model.named_parameters()))
models.append(model)
for name, value in models[0].state_dict().items():
torch.testing.assert_close(
models[1].state_dict()[name], value, rtol=0, atol=0
)
with torch.no_grad():
expected = reference(ids, attention_mask=mask, output_hidden_states=True)
outputs = [
model(ids, attention_mask=mask, output_hidden_states=True)
for model in models
]
for prefixed, bare, reference_output in (
(
outputs[0].last_hidden_state,
outputs[1].last_hidden_state,
expected.last_hidden_state,
),
(outputs[0].pooler_output, outputs[1].pooler_output, expected.pooler_output),
(
tuple(outputs[0].hidden_states),
tuple(outputs[1].hidden_states),
expected.hidden_states,
),
):
torch.testing.assert_close(prefixed, bare, rtol=0, atol=0)
torch.testing.assert_close(prefixed, reference_output, rtol=2e-5, atol=2e-5)
def test_bare_text_checkpoint_preserves_projection_weight():
config = NativeCLIPTextConfig()
config.update_model_arch(
{
"hidden_size": 16,
"intermediate_size": 32,
"num_hidden_layers": 1,
"num_attention_heads": 2,
"vocab_size": 32,
"projection_dim": 8,
}
)
with get_parallel().override(tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0):
model = CLIPTextModelWithProjection(config)
weight = torch.randn_like(model.text_projection.weight)
assert model.load_weights([("text_projection.weight", weight)]) == {
"text_projection.weight"
}
torch.testing.assert_close(model.text_projection.weight, weight, rtol=0, atol=0)
@@ -0,0 +1,127 @@
# SPDX-License-Identifier: Apache-2.0
"""GAP 6 regression: the Kandinsky6 IT2VA conditioning-image preprocessor
must aspect-preserving-resize-then-centre-crop, matching the diffusers
reference's ``encode_i2va_first_frame``
(``pipeline_kandinsky6_ti2va.py``, ``scale = min(src_h/height,
src_w/width)``; resize to ``(src_h/scale, src_w/scale)``; centre-crop to
``(height, width)``), not stretch-resize the whole source image to the
target box (which distorts its aspect ratio whenever the source doesn't
already match the target aspect ratio).
"""
from __future__ import annotations
import numpy as np
import PIL.Image
import pytest
import torch
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import (
Kandinsky6ImageEncodingStage,
)
def _gradient_image(width: int, height: int) -> PIL.Image.Image:
"""A deterministic, per-pixel-distinguishable RGB image (not a flat
color), so resize/crop geometry is actually verifiable pixel-for-pixel.
"""
xs = np.linspace(0, 255, width, dtype=np.uint8)
ys = np.linspace(0, 255, height, dtype=np.uint8)
grid = np.zeros((height, width, 3), dtype=np.uint8)
grid[..., 0] = xs[None, :]
grid[..., 1] = ys[:, None]
grid[..., 2] = 128
return PIL.Image.fromarray(grid, mode="RGB")
@pytest.mark.parametrize(
"src_h,src_w,height,width",
[
(1080, 1920, 512, 768), # 16:9 source, 3:2 target (audit's own case)
(1920, 1080, 512, 768), # 9:16 source, 3:2 target (audit's own case)
(1024, 1536, 512, 768), # already 3:2 -- matching aspect
],
)
def test_cover_resize_dims_matches_diffusers_formula(src_h, src_w, height, width):
# encode_i2va_first_frame: scale = min(src_h/height, src_w/width);
# resize to (int(src_h/scale), int(src_w/scale)), then centre-crop.
scale = min(src_h / height, src_w / width)
expected = (int(src_h / scale), int(src_w / scale))
assert (
Kandinsky6ImageEncodingStage._cover_resize_dims(src_h, src_w, height, width)
== expected
)
# The resize always covers the target box on both axes (never smaller).
assert expected[0] >= height
assert expected[1] >= width
@pytest.mark.parametrize(
"src_w,src_h,min_mean_abs_diff",
[
(1920, 1080, 0.015), # 16:9 landscape source (audit's own case)
(1080, 1920, 0.05), # 9:16 portrait source (audit's own case)
],
)
def test_mismatched_aspect_pil_image_is_cropped_not_stretched(
src_w, src_h, min_mean_abs_diff
):
# The model's documented default target resolution (512x768, 2:3)
# against realistic phone/webcam source aspect ratios.
height, width = 512, 768
image = _gradient_image(src_w, src_h)
produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width)
assert produced.shape == (1, 3, height, width)
# Independent oracle: the diffusers reference's own aspect-preserving
# resize + centre-crop, computed directly via PIL, without calling any
# of the code under test.
scale = min(src_h / height, src_w / width)
new_h, new_w = int(src_h / scale), int(src_w / scale)
resized = image.resize((new_w, new_h), resample=PIL.Image.BILINEAR)
top, left = (new_h - height) // 2, (new_w - width) // 2
cropped = resized.crop((left, top, left + width, top + height))
expected_np = np.array(cropped).astype(np.float32) / 255.0
expected = torch.from_numpy(expected_np).permute(2, 0, 1) * 2.0 - 1.0
assert torch.allclose(produced[0], expected, atol=2.0 / 255.0)
# The deleted stretch-resize procedure (plain `.resize((width, height))`,
# no crop) produces a materially different image whenever the source
# aspect ratio doesn't already match the target -- proving this is a
# real behavior change, not a no-op refactor.
stretched = image.resize((width, height), resample=PIL.Image.BILINEAR)
stretched_np = np.array(stretched).astype(np.float32) / 255.0
stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0
mean_abs_diff = (produced[0] - stretched_t).abs().mean().item()
assert mean_abs_diff > min_mean_abs_diff
def test_matching_aspect_pil_image_resize_only_agrees_with_stretch():
# A sanity check on the oracle itself: when the source already matches
# the target aspect ratio, cover-resize-then-crop and plain stretch
# resize agree closely (no crop actually removes anything) -- so the
# large the divergence asserted above is specifically an aspect-ratio
# effect, not a general property of the two code paths.
height, width = 64, 96
src_w, src_h = 288, 192 # exactly 3:2, matching height:width == 64:96 == 2:3...
# (use width:height == 96:64 == 3:2 to match src_w:src_h == 288:192 == 3:2)
image = _gradient_image(src_w, src_h)
produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width)
stretched = image.resize((width, height), resample=PIL.Image.BILINEAR)
stretched_np = np.array(stretched).astype(np.float32) / 255.0
stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0
mean_abs_diff = (produced[0] - stretched_t).abs().mean().item()
assert mean_abs_diff < 0.02
def test_tensor_input_uses_cover_crop_geometry_too():
height, width = 64, 96
src_h, src_w = 108, 192
tensor = torch.rand(1, 3, src_h, src_w) * 2.0 - 1.0 # already in [-1, 1]
produced = Kandinsky6ImageEncodingStage._preprocess(tensor, height, width)
assert produced.shape[-2:] == (height, width)
@@ -0,0 +1,271 @@
# SPDX-License-Identifier: Apache-2.0
"""Exercise the declared offload groups, not just component selection."""
import sys
from types import SimpleNamespace
import pytest
import torch
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
Kandinsky6VideoAudioConfig,
)
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
Kandinsky6AudioVAEConfig,
)
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
Kandinsky6SRVAEConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
Kandinsky6TI2VAPipelineConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.layers.attention.selector import (
global_force_attn_backend_context_manager,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
LayerwiseOffloadableModuleMixin,
)
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
Kandinsky6Transformer3DModel,
)
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
Kandinsky6SRLatentUpscalerBank,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio import (
Kandinsky6AudioVAE,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.server_args import get_global_server_args
from sglang.multimodal_gen.runtime.utils import precision
from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import (
TINY_KVAE,
TINY_LU_MODEL,
)
from sglang.srt.utils.network import get_free_port_below_ephemeral
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.fixture(autouse=True)
def single_gpu(monkeypatch, default_global_server_args):
if not model_parallel_is_initialized():
port = get_free_port_below_ephemeral()
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT=str(port),
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
monkeypatch.setenv(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
monkeypatch.setattr(
precision._mixed_precision_state,
"state",
precision.MixedPrecisionState(
param_dtype=torch.bfloat16, reduce_dtype=torch.float32
),
)
args = get_global_server_args()
args.attention_backend = "fa"
args.pipeline_config = Kandinsky6TI2VAPipelineConfig()
with global_force_attn_backend_context_manager(AttentionBackendEnum.FA):
yield
def _dit(multimodal):
config = Kandinsky6VideoAudioConfig()
config.update_model_arch(
dict(
model_dim=128,
time_dim=32,
ff_dim=256,
axes_dims=(32, 48, 48),
in_visual_dim=4,
out_visual_dim=4,
in_text_dim=16,
in_text_dim2=8,
in_audio_dim=4,
out_audio_dim=4,
num_text_blocks=2,
num_visual_blocks=2,
patch_size=(1, 1, 1),
visual_cond=False,
visual_token_type_num_embeddings=0,
is_multimodal=multimodal,
model_dim_a=128,
time_dim_a=32,
ff_dim_a=256,
axes_dims_a=(32, 48, 48),
ca_rope=True,
cross_gates=True,
fix_modulation=True,
)
)
model = Kandinsky6Transformer3DModel(config, {})
# TP-aware linears can use uninitialized storage before checkpoint loading
with torch.no_grad():
for parameter in model.parameters():
parameter.normal_(std=0.02)
model.to(device="cuda", dtype=torch.bfloat16).eval()
inputs = dict(
hidden_states=torch.randn(1, 2, 3, 3, 4, device="cuda", dtype=torch.bfloat16),
encoder_hidden_states=torch.randn(
1, 7, 16, device="cuda", dtype=torch.bfloat16
),
pooled_projections=torch.randn(1, 8, device="cuda", dtype=torch.bfloat16),
timestep=torch.tensor([500.0], device="cuda"),
visual_rope_pos=[torch.arange(n, device="cuda") for n in (2, 3, 3)],
text_rope_pos=torch.arange(7, device="cuda"),
)
if multimodal:
inputs["hidden_states_audio"] = torch.randn(
1, 5, 4, device="cuda", dtype=torch.bfloat16
)
groups = (
["video_text_transformer_blocks", "audio_text_transformer_blocks"]
if multimodal
else ["text_transformer_blocks"]
)
return model, lambda: model(**inputs), groups + ["visual_transformer_blocks"]
def _audio_vae():
config = Kandinsky6AudioVAEConfig()
config.arch_config.mode = "16k"
config.arch_config.vocoder_config = dict(
resblock="1",
num_mels=80,
upsample_rates=[2],
upsample_kernel_sizes=[4],
upsample_initial_channel=8,
resblock_kernel_sizes=[3],
resblock_dilation_sizes=[[1, 1, 1]],
activation="snake",
snake_logscale=False,
)
model = Kandinsky6AudioVAE(config).to("cuda").eval()
mel = torch.randn(1, 80, 8, device="cuda")
def forward():
latent = model.vae.encode(mel).mean
decoded = model.decode(latent)
return latent, decoded, model.vocode(decoded)
groups = [f"vae.encoder.down.{i}.block" for i in range(3)]
groups += [f"vae.decoder.up.{i}.block" for i in range(3)]
return model, forward, groups + ["vocoder.ups.0", "vocoder.resblocks"]
def _sr_vae():
config = Kandinsky6SRVAEConfig()
config.update_model_arch(
dict(
vae_type="video-kvae",
encoder_config=dict(TINY_KVAE, in_channels=3),
decoder_config=dict(TINY_KVAE, out_ch=3),
)
)
model = Kandinsky6SRVAE(config).to("cuda").eval()
pixels = torch.randn(1, 3, 33, 32, 32, device="cuda")
def forward():
latent, segments = model.encode(pixels)
assert segments == [17, 16]
return latent, model.decode(latent).sample
groups = [f"encoder.down.{i}.block" for i in range(5)]
groups += [f"decoder.up.{i}.block" for i in range(5)]
return model, forward, groups
def _upscaler():
model = (
Kandinsky6SRLatentUpscalerBank(
[{"target_scale": scale, "model": dict(TINY_LU_MODEL)} for scale in (2, 4)]
)
.to("cuda")
.eval()
)
latent = torch.randn(1, 4, 3, 3, 3, device="cuda")
def forward():
# exercise both branches of every loaded entry, including width changes
return tuple(
entry(latent, scale) for entry in model._models for scale in (2, 4)
)
groups = [
f"_models.{index}.{group}"
for index in range(2)
for group in (
"pre_blocks",
"mid_blocks",
"post_blocks",
"x2_branch.adapter",
"x2_branch.mid_blocks",
"x2_branch.blocks",
)
]
return model, forward, groups
@pytest.mark.parametrize(
"component", ["video-dit", "joint-dit", "audio-vae", "sr-vae", "upscaler"]
)
@pytest.mark.parametrize("resident_layers", [0, 1])
@torch.no_grad()
def test_layerwise_groups_preserve_repeated_forwards(component, resident_layers):
torch.manual_seed(13)
if component.endswith("dit"):
model, forward, groups = _dit(component == "joint-dit")
elif component == "audio-vae":
model, forward, groups = _audio_vae()
elif component == "sr-vae":
model, forward, groups = _sr_vae()
else:
model, forward, groups = _upscaler()
assert isinstance(model, LayerwiseOffloadableModuleMixin)
assert set(groups) <= set(model.layer_names)
assert model.layerwise_offload_dit_group_enabled == component.endswith("dit")
state = {name: value.cpu().clone() for name, value in model.state_dict().items()}
with (
torch.inference_mode(),
set_forward_context(current_timestep=0, attn_metadata=None),
):
expected = forward()
model.configure_layerwise_offload(
SimpleNamespace(
performance_mode="speed",
pin_cpu_memory=True,
layerwise_tuning_for=lambda *args, **kwargs: (
1,
resident_layers,
"leading",
"forward",
),
)
)
assert {
manager.layers_attr_str for manager in model.layerwise_offload_managers
} == set(groups)
for _ in range(2):
model.prepare_for_next_req()
torch.testing.assert_close(forward(), expected, rtol=0, atol=0)
for manager in model.layerwise_offload_managers:
assert manager._last_forwarded_layer == manager.num_layers - 1
manager.release_all()
model.disable_offload()
torch.testing.assert_close(forward(), expected, rtol=0, atol=0)
for name, value in model.state_dict().items():
torch.testing.assert_close(value.cpu(), state[name], rtol=0, atol=0)
if __name__ == "__main__":
sys.exit(pytest.main([__file__, *sys.argv[1:]]))
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
"""Load real PEFT safetensors through Kandinsky6's native adapter mapping."""
from collections import defaultdict
from types import SimpleNamespace
import pytest
import torch
from safetensors.torch import save_file
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
Kandinsky6TI2VAPipelineConfig,
)
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import (
Kandinsky6TI2VAPipeline,
)
@pytest.mark.parametrize(
"prefix", ["", "transformer.", "base_model.model.transformer."]
)
@pytest.mark.parametrize(
"source,target",
[
(
"visual_transformer_blocks.0.video_dec_block.feed_forward.net.0.proj",
"visual_transformer_blocks.0.videoT.feed_forward.mlp.fc_in",
),
(
"visual_transformer_blocks.0.audio_dec_block.feed_forward.net.2",
"visual_transformer_blocks.0.audioT.feed_forward.mlp.fc_out",
),
(
"video_text_transformer_blocks.1.feed_forward.net.0.proj",
"video_text_transformer_blocks.1.feed_forward.mlp.fc_in",
),
(
"audio_text_transformer_blocks.1.attn.to_query",
"audio_text_transformer_blocks.1.self_attention.to_query",
),
(
"video_time_embeddings.timestep_embedder.linear_1",
"video_time_embeddings.in_layer",
),
(
"audio_time_embeddings.timestep_embedder.linear_2",
"audio_time_embeddings.out_layer",
),
(
"visual_transformer_blocks.0.videoT.self_attention.to_key",
"visual_transformer_blocks.0.videoT.self_attention.to_key",
),
],
)
def test_peft_adapter_maps_to_native_layers(tmp_path, prefix, source, target):
pipeline = object.__new__(Kandinsky6TI2VAPipeline)
pipeline.server_args = SimpleNamespace(
pipeline_config=Kandinsky6TI2VAPipelineConfig(), lora_path=None
)
pipeline.modules = {
"transformer": SimpleNamespace(
param_names_mapping={}, lora_param_names_mapping={}
)
}
pipeline.device = torch.device("cpu")
pipeline.lora_adapters = defaultdict(dict)
pipeline.loaded_adapter_paths = {}
pipeline.loaded_adapter_alphas = {}
values = {
"lora_A": torch.arange(16, dtype=torch.float32).reshape(2, 8),
"lora_B": torch.arange(24, dtype=torch.float32).reshape(12, 2),
"alpha": torch.tensor(4.0),
}
path = tmp_path / "adapter.safetensors"
save_file(
{
f"{prefix}{source}.{key}.default.weight"
if key != "alpha"
else f"{prefix}{source}.alpha": value
for key, value in values.items()
},
path,
)
pipeline.load_lora_adapter(str(path), "probe", rank=0)
actual = pipeline.lora_adapters["probe"]
assert set(actual) == {f"{target}.{key}" for key in values}
for key, expected in values.items():
torch.testing.assert_close(actual[f"{target}.{key}"], expected, rtol=0, atol=0)
@@ -0,0 +1,346 @@
# SPDX-License-Identifier: Apache-2.0
"""Pipeline wiring, residency precision and joint video/audio denoising."""
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
Kandinsky6TI2VAPipelineConfig,
)
from sglang.multimodal_gen.configs.sample.kandinsky6 import (
Kandinsky6TI2VASamplingParams,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy
from sglang.multimodal_gen.runtime.managers.memory_managers import (
component_residency_strategies,
)
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
Kandinsky6RoPE1D,
Kandinsky6RoPE3D,
)
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
PiflowScheduler,
)
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import (
Kandinsky6TI2VAPipeline,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages import InputValidationStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6 import (
Kandinsky6AudioDecodingStage,
Kandinsky6DecodingStage,
Kandinsky6DenoisingStage,
Kandinsky6ImageEncodingStage,
Kandinsky6LatentPreparationStage,
denoising,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
TextEncodingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation import (
TimestepPreparationStage,
)
from sglang.multimodal_gen.runtime.server_args import get_global_server_args
EXPECTED_REQUIRED_MODULES = [
"scheduler",
"text_encoder",
"text_encoder_2",
"tokenizer",
"tokenizer_2",
"transformer",
"vae",
"audio_vae",
]
EXPECTED_STAGE_ORDER = [
("input_validation_stage", InputValidationStage),
("text_encoding_stage", TextEncodingStage),
("timestep_preparation_stage", TimestepPreparationStage),
("latent_preparation_stage", Kandinsky6LatentPreparationStage),
("image_encoding_stage", Kandinsky6ImageEncodingStage),
("denoising_stage", Kandinsky6DenoisingStage),
# Audio decodes before video, matching the diffusers reference -- see
# Kandinsky6TI2VAPipeline.create_pipeline_stages's comment.
("audio_decoding_stage", Kandinsky6AudioDecodingStage),
("decoding_stage", Kandinsky6DecodingStage),
]
def _make_pipeline() -> Kandinsky6TI2VAPipeline:
pipeline = object.__new__(Kandinsky6TI2VAPipeline)
pipeline.modules = {
name: MagicMock(name=name)
for name in Kandinsky6TI2VAPipeline._required_config_modules
}
pipeline._stages = []
pipeline._stage_name_mapping = {}
pipeline._disagg_role = RoleType.MONOLITHIC
return pipeline
def test_pipeline_identity_and_required_modules():
assert Kandinsky6TI2VAPipeline.pipeline_name == "Kandinsky6TI2VAPipeline"
assert Kandinsky6TI2VAPipeline.is_video_pipeline is True
assert Kandinsky6TI2VAPipeline._required_config_modules == EXPECTED_REQUIRED_MODULES
# No separate "vocoder" module -- audio_vae bundles the mel-VAE decoder
# and the BigVGAN vocoder in one checkpoint component.
assert "vocoder" not in Kandinsky6TI2VAPipeline._required_config_modules
def test_disagg_role_rejects_non_monolithic_deployment():
pipeline = object.__new__(Kandinsky6TI2VAPipeline)
with pytest.raises(ValueError, match="monolithic"):
pipeline.validate_disagg_role(RoleType.ENCODER)
with pytest.raises(ValueError, match="monolithic"):
pipeline.validate_disagg_role(RoleType.DENOISER)
# MONOLITHIC is the only accepted role -- no exception raised.
pipeline.validate_disagg_role(RoleType.MONOLITHIC)
def test_create_pipeline_stages_produces_documented_order():
pipeline = _make_pipeline()
server_args = MagicMock()
Kandinsky6TI2VAPipeline.create_pipeline_stages(pipeline, server_args)
actual_types = [type(stage) for stage in pipeline.stages]
expected_types = [stage_cls for _, stage_cls in EXPECTED_STAGE_ORDER]
assert actual_types == expected_types
actual_names = list(pipeline._stage_name_mapping.keys())
expected_names = [name for name, _ in EXPECTED_STAGE_ORDER]
assert actual_names == expected_names
# get_module wiring: each stage received the mocked module it asked for.
denoising_stage = pipeline._stage_name_mapping["denoising_stage"]
assert denoising_stage.transformer is pipeline.modules["transformer"]
audio_decoding_stage = pipeline._stage_name_mapping["audio_decoding_stage"]
assert audio_decoding_stage.audio_vae is pipeline.modules["audio_vae"]
decoding_stage = pipeline._stage_name_mapping["decoding_stage"]
assert decoding_stage.vae is pipeline.modules["vae"]
image_encoding_stage = pipeline._stage_name_mapping["image_encoding_stage"]
assert image_encoding_stage.vae is pipeline.modules["vae"]
def _pro_distill_scheduler() -> PiflowScheduler:
"""The scheduler of kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers."""
scheduler = PiflowScheduler(
nfe=None,
n_grid=10,
shift=5.0,
eps=1e-6,
final_step_size_scale=0.5,
num_policy_substeps=128,
)
scheduler.set_timesteps(10)
return scheduler
@pytest.mark.parametrize("stage_name", ["denoising_stage", "audio_decoding_stage"])
@pytest.mark.parametrize("precision", ["bf16", "fp32"])
def test_component_residency_preserves_loaded_precision(stage_name, precision):
pipeline = _make_pipeline()
args = MagicMock()
args.pipeline_config.dit_precision = precision
args.pipeline_config.audio_vae_precision = precision
args.component_precisions = {"transformer": precision, "audio_vae": precision}
pipeline.create_pipeline_stages(args)
use = pipeline._stage_name_mapping[stage_name].component_uses(args)[0]
# loaders own precision; movement must not downcast RoPE or audio buffers
assert use.target_dtype is None
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA transfers")
def test_denoising_offload_roundtrip_preserves_fp32_rope(monkeypatch):
device = torch.device("cuda", torch.cuda.current_device())
monkeypatch.setattr(
component_residency_strategies, "get_local_torch_device", lambda: device
)
transformer = torch.nn.ModuleDict(
{
"projection": torch.nn.Linear(8, 8, dtype=torch.bfloat16),
"rope_1d": Kandinsky6RoPE1D(8),
"rope_3d": Kandinsky6RoPE3D((4, 4, 4)),
}
).to(device)
buffers = {name: value.clone() for name, value in transformer.named_buffers()}
stage = Kandinsky6DenoisingStage(transformer, scheduler=MagicMock())
use = stage.component_uses(get_global_server_args())[0]
for _ in range(2):
transformer.cpu()
component_residency_strategies._module_to_local_device(
transformer, dtype=use.target_dtype
)
assert next(transformer.parameters()).dtype == torch.bfloat16
for name, value in transformer.named_buffers():
assert value.device == device
torch.testing.assert_close(value, buffers[name], rtol=0, atol=0)
def test_parallel_cfg_uses_serial_arithmetic_for_video_and_audio(monkeypatch):
config = Kandinsky6TI2VAPipelineConfig()
batch = SimpleNamespace(
do_classifier_free_guidance=True,
guidance_scale=5.0,
cfg_normalization=0.0,
guidance_rescale=0.0,
)
policy = config.cfg_policy.build(batch, {}, {}, {})
positive = torch.linspace(-1, 1, 256, dtype=torch.bfloat16)
negative = torch.linspace(-0.8, 0.9, 256, dtype=torch.bfloat16)
predictions = [(positive, negative), (negative, positive)]
expected = policy.combine(predictions, batch, 5.0, config)
legacy = CFGPolicy().combine(predictions, batch, 5.0, config, cfg_parallel=True)
assert any(not torch.equal(a, b) for a, b in zip(expected, legacy, strict=True))
monkeypatch.setattr(denoising, "get_classifier_free_guidance_world_size", lambda: 2)
monkeypatch.setattr(denoising, "run_cfg_parallel", lambda *_args: predictions)
monkeypatch.setattr(
denoising,
"run_two_branch_cfg_parallel",
lambda *_args: pytest.fail("legacy all-reduce changes BF16 rounding"),
)
stage = Kandinsky6DenoisingStage(MagicMock(), scheduler=MagicMock())
actual = stage._predict_joint_velocity(
transformer=stage.transformer,
cfg_policy=policy,
batch=batch,
server_args=SimpleNamespace(pipeline_config=config, enable_cfg_parallel=True),
step_index=0,
video_input=positive,
audio_input=negative,
t_expand=torch.zeros(1),
visual_rope_pos=[],
scale_factor=(1.0, 1.0, 1.0),
sparse_params=None,
visual_token_type_ids=None,
)
for result, reference in zip(actual, expected, strict=True):
torch.testing.assert_close(result, reference, rtol=0, atol=0)
@pytest.mark.parametrize(
"guidance_scale", [5.0, 0.5, 1.001, float("nan"), float("inf")]
)
def test_denoising_rejects_guidance_other_than_one_with_a_piflow_scheduler(
guidance_scale,
):
"""The distilled checkpoint's ``PiflowScheduler`` has no classifier-free guidance,
so the run must not silently ignore a guidance the caller asked for; the
Diffusers pipeline raises the same way ("PiflowScheduler requires
guidance_weight=1.0")."""
scheduler = _pro_distill_scheduler()
batch = SimpleNamespace(
timesteps=scheduler.timesteps,
latents=torch.zeros(1, 1, 1, 1, 1),
audio_latents=torch.zeros(1, 1, 1),
scheduler=scheduler,
guidance_scale=guidance_scale,
)
stage = Kandinsky6DenoisingStage(transformer=MagicMock(), scheduler=scheduler)
with pytest.raises(ValueError, match=r"requires guidance_scale=1\.0"):
stage.forward(batch, MagicMock())
@pytest.mark.parametrize("height,width", [(32, 32), (512, 768), (768, 512)])
def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance(
monkeypatch, height, width
):
"""Pro-distill: 10 steps at guidance 1.0. The DiT head is ``n_grid`` (= 10) times as
wide as the latents, the pi-Flow step folds it back, and the returned latents keep
the latent width. The negative prompt is never fed to the DiT."""
monkeypatch.setattr(
denoising, "get_local_torch_device", lambda: torch.device("cpu")
)
monkeypatch.setattr(
get_global_server_args().pipeline_config, "dit_precision", "fp32"
)
video_channels, audio_channels, n_grid, text_len = 4, 6, 10, 7
text_lens = []
class _FakeDiT(torch.nn.Module):
"""A real ``nn.Module`` (not a bare function): ``Kandinsky6DenoisingStage``
now subclasses the shared ``DenoisingStage``, whose ``__init__`` walks
``self.transformer.modules()`` to infer the attention backend -- a plain
callable has no ``.modules()``."""
def forward(self, **kwargs):
assert tuple(kwargs["scale_factor"]) == (1.0, 2.0, 2.0)
text_lens.append(kwargs["encoder_hidden_states"].shape[1])
video, audio = kwargs["hidden_states"], kwargs["hidden_states_audio"]
video_velocity = torch.randn(*video.shape[:-1], n_grid * video_channels)
audio_velocity = torch.randn(*audio.shape[:-1], n_grid * audio_channels)
assert kwargs["return_dict"] is False
return video_velocity, audio_velocity
fake_dit = _FakeDiT()
scheduler = _pro_distill_scheduler()
# visual_cond: the DiT input carries the latents, a condition copy and a mask
video = torch.randn(1, 2, 4, 4, 2 * video_channels + 1)
batch = SimpleNamespace(
timesteps=scheduler.timesteps,
latents=video,
audio_latents=torch.randn(1, 5, audio_channels),
scheduler=scheduler,
guidance_scale=1.0,
do_classifier_free_guidance=False,
extra={},
prompt_embeds=[torch.randn(1, text_len, 8)],
negative_prompt_embeds=[torch.randn(1, text_len + 2, 8)],
pooled_embeds=[torch.randn(1, 3)],
prompt_attention_mask=[torch.ones(1, text_len)],
height=height,
width=width,
image_latent=None,
is_warmup=True,
sampling_params=Kandinsky6TI2VASamplingParams(quality="lossless"),
)
arch = SimpleNamespace(
attention_engine="auto",
in_visual_dim=video_channels,
patch_size=(1, 2, 2),
scale_factor=(1.0, 2.0, 2.0),
)
pipeline_config = SimpleNamespace(
dit_config=SimpleNamespace(arch_config=arch),
dit_precision="fp32",
vae_config=SimpleNamespace(
arch_config=SimpleNamespace(spatial_compression_ratio=8)
),
get_pos_prompt_embeds=lambda batch: batch.prompt_embeds[0],
# CFG is disabled in this test (guidance_scale=1.0,
# do_classifier_free_guidance=False), so the policy never needs a
# negative branch, but Kandinsky6DenoisingStage still reads
# `cfg_policy` / `get_classifier_free_guidance_scale` off the config
# unconditionally to build the (single-branch) CFGPolicy.
cfg_policy=CFGPolicy(),
get_classifier_free_guidance_scale=lambda _batch, guidance_scale: (
guidance_scale
),
)
server_args = SimpleNamespace(
pipeline_config=pipeline_config, enable_cfg_parallel=False
)
stage = Kandinsky6DenoisingStage(transformer=fake_dit, scheduler=scheduler)
result = stage.forward(batch, server_args)
assert text_lens == [text_len] * 10
assert tuple(result.latents.shape) == (1, 2, 4, 4, video_channels)
assert tuple(result.audio_latents.shape) == (1, 5, audio_channels)
assert torch.isfinite(result.latents).all()
assert torch.isfinite(result.audio_latents).all()
@@ -0,0 +1,207 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky 6 model routing and request defaults, without Hub access."""
import json
from types import SimpleNamespace
import pytest
from sglang.multimodal_gen import registry
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
Kandinsky6TI2VAPipelineConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.configs.sample.kandinsky6 import (
Kandinsky6TI2VADistilledSamplingParams,
Kandinsky6TI2VASamplingParams,
)
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import (
Kandinsky6TI2VAPipeline,
)
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import (
Kandinsky6SRPipeline,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
SFT = "kandinskylab/Kandinsky-6.0-Pro-sft-5s-Diffusers"
DISTILL = "kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers"
TI2VA = (
Kandinsky6TI2VAPipeline,
Kandinsky6TI2VAPipelineConfig,
Kandinsky6TI2VASamplingParams,
)
DISTILLED = (
Kandinsky6TI2VAPipeline,
Kandinsky6TI2VAPipelineConfig,
Kandinsky6TI2VADistilledSamplingParams,
)
SR = (Kandinsky6SRPipeline, Kandinsky6SRPipelineConfig, Kandinsky6SRSamplingParams)
ROUTES = [
("kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers", TI2VA),
(SFT, TI2VA),
(DISTILL, DISTILLED),
("kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers", SR),
("kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", SR),
("/models/Kandinsky-6.0-Pro-5s-Diffusers", TI2VA),
("/models/Kandinsky-6.0-Pro-sft-5s-Diffusers", TI2VA),
("/models/Kandinsky-6.0-Pro-distill-5s-Diffusers/", DISTILLED),
("/models/Kandinsky-6.0-VSR-5s-Diffusers", SR),
("/models/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", SR),
("/distill_runs/Kandinsky-6.0-Pro-sft-5s-Diffusers", TI2VA),
(
"/hub/models--kandinskylab--Kandinsky-6.0-Pro-sft-5s-Diffusers/snapshots/4f2a",
TI2VA,
),
(
"/hub/models--kandinskylab--Kandinsky-6.0-Pro-distill-5s-Diffusers/snapshots/4f2a",
DISTILLED,
),
]
@pytest.fixture(autouse=True)
def clear_registry_cache():
registry.get_model_info.cache_clear()
registry._get_config_info.cache_clear()
yield
registry.get_model_info.cache_clear()
registry._get_config_info.cache_clear()
@pytest.mark.parametrize("model_path,expected", ROUTES)
def test_model_routing(monkeypatch, model_path, expected):
pipeline, _, _ = expected
monkeypatch.setattr(
registry,
"maybe_download_model_index",
lambda _: {"_class_name": pipeline.pipeline_name},
)
assert registry.is_registered_diffusion_model_path(model_path)
info = registry.get_model_info(model_path, backend="sglang")
assert (
info.pipeline_cls,
info.pipeline_config_cls,
info.sampling_param_cls,
) == expected
@pytest.mark.parametrize(
"name,expected",
[
(SFT, TI2VA),
(DISTILL, DISTILLED),
("/models/kandinsky6-sr-native", SR),
("/models/Kandinsky_6.0_SR_x2", SR),
("Kandinsky6TI2VAPipeline", TI2VA),
("Kandinsky6T2VAPipeline", TI2VA),
("Kandinsky6SRPipeline", SR),
("/distill_runs/Kandinsky-6.0-Pro-sft-5s-Diffusers", TI2VA),
("kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", SR),
],
)
def test_detectors_do_not_overlap(name, expected):
claimed = [
registry._CONFIG_REGISTRY[model_id]
for model_id, detector in registry._MODEL_NAME_DETECTORS
if detector(name.lower())
]
assert len(claimed) == 1
assert (claimed[0].pipeline_config_cls, claimed[0].sampling_param_cls) == expected[
1:
]
@pytest.mark.parametrize("expected", [TI2VA, SR])
def test_local_bundle_routing(tmp_path, expected):
pipeline, _, _ = expected
for component in ("transformer", "vae"):
(tmp_path / component).mkdir()
(tmp_path / "model_index.json").write_text(
json.dumps(
{
"_class_name": pipeline.pipeline_name,
"_diffusers_version": "0.37.0",
"transformer": ["diffusers", "X"],
"vae": ["diffusers", "Y"],
}
)
)
info = registry.get_model_info(str(tmp_path), backend="sglang")
assert (
info.pipeline_cls,
info.pipeline_config_cls,
info.sampling_param_cls,
) == expected
if expected == TI2VA:
info = registry.get_model_info(
str(tmp_path),
backend="sglang",
model_id="Kandinsky-6.0-Pro-distill-5s-Diffusers",
)
assert info.sampling_param_cls is Kandinsky6TI2VADistilledSamplingParams
@pytest.mark.parametrize(
"model_path,overrides,steps,guidance",
[
(SFT, {}, 50, 5.0),
(DISTILL, {}, 10, 1.0),
("/models/Kandinsky-6.0-Pro-distill-5s-Diffusers", {}, 10, 1.0),
(DISTILL, {"num_inference_steps": 8}, 8, 1.0),
(DISTILL, {"num_inference_steps": 50}, 50, 1.0),
(SFT, {"num_inference_steps": 25, "guidance_scale": 3.0}, 25, 3.0),
],
)
def test_request_defaults_and_overrides(
monkeypatch, model_path, overrides, steps, guidance
):
monkeypatch.setattr(
registry,
"maybe_download_model_index",
lambda _: {"_class_name": "Kandinsky6TI2VAPipeline"},
)
args = SimpleNamespace(
backend="sglang",
model_id=None,
pipeline_class_name=None,
pipeline_config=Kandinsky6TI2VAPipelineConfig(),
output_path=None,
comfyui_mode=True,
num_gpus=1,
)
params = SamplingParams.from_user_sampling_params_args(
model_path, args, prompt="a cat", **overrides
)
assert (params.num_inference_steps, params.guidance_scale) == (steps, guidance)
assert (params.height, params.width, params.num_frames, params.fps) == (
512,
768,
121,
24,
)
assert Req(sampling_params=params).do_classifier_free_guidance == (guidance > 1)
@pytest.mark.parametrize(
"params_cls",
[Kandinsky6TI2VASamplingParams, Kandinsky6TI2VADistilledSamplingParams],
)
@pytest.mark.parametrize("num_gpus", [1, 2, 3, 4, 8])
def test_request_frame_count_is_independent_of_gpu_count(params_cls, num_gpus):
params = params_cls()
params._adjust(
SimpleNamespace(
pipeline_config=Kandinsky6TI2VAPipelineConfig(),
output_path=None,
comfyui_mode=True,
num_gpus=num_gpus,
)
)
assert params.num_frames == 121
@@ -0,0 +1,123 @@
# SPDX-License-Identifier: Apache-2.0
import pytest
import torch
from sglang.multimodal_gen.runtime.models.dits import kandinsky6
@pytest.mark.parametrize("device", ["cpu", "cuda"])
@pytest.mark.parametrize("shape", [(1, 1, 1, 1), (2, 3, 5, 7), (1, 31, 32, 48)])
def test_3d_rope_broadcast_matches_materialized_axes(device, shape, monkeypatch):
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("requires CUDA")
rope = kandinsky6.Kandinsky6RoPE3D((64, 32, 32)).to(device)
batch, frames, height, width = shape
pos = [torch.arange(n * 2, device=device)[::2] for n in shape[1:]]
args = torch.cat(
[
rope.args_0[pos[0]]
.view(1, frames, 1, 1, -1)
.repeat(batch, 1, height, width, 1),
(rope.args_1[pos[1]] / 2.0)
.view(1, 1, height, 1, -1)
.repeat(batch, frames, 1, width, 1),
(rope.args_2[pos[2]] / 3.16)
.view(1, 1, 1, width, -1)
.repeat(batch, frames, height, 1, 1),
],
dim=-1,
)
cosine, sine = args.cos(), args.sin()
expected = torch.stack([cosine, -sine, sine, cosine], dim=-1)
expected = expected.view(*expected.shape[:-1], 2, 2).unsqueeze(-4)
def no_repeat(*args, **kwargs):
pytest.fail("RoPE axes should broadcast without materializing repeats")
monkeypatch.setattr(torch.Tensor, "repeat", no_repeat)
actual = rope(shape, pos, (1.0, 2.0, 3.16))
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
assert actual.is_contiguous()
@pytest.mark.parametrize("strided", [False, True])
def test_rotary_cpu_and_autograd(strided):
x = torch.randn(2, 17, 3, 64, requires_grad=True)
if strided:
x = x[:, ::2]
rope = torch.randn(1, x.shape[1], 1, 32, 2, 2)
expected = (rope * x.reshape(2, x.shape[1], 3, 32, 1, 2)).sum(-1).reshape_as(x)
actual = kandinsky6._apply_rotary(x, rope)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
torch.testing.assert_close(
torch.autograd.grad(actual.sum(), x, retain_graph=True)[0],
torch.autograd.grad(expected.sum(), x)[0],
rtol=0,
atol=0,
)
@torch.no_grad()
@pytest.mark.skipif(
not torch.cuda.is_available() or torch.version.hip is not None,
reason="CUDA required",
)
def test_rotary_uses_fused_kernel(monkeypatch):
original = kandinsky6.apply_matrix_rope
calls = []
def record(x, rope, dtype=None):
calls.append(x.shape)
return original(x, rope, dtype)
monkeypatch.setattr(kandinsky6, "apply_matrix_rope", record)
x = torch.randn(1, 257, 8, 128, dtype=torch.bfloat16, device="cuda")
rope = torch.randn(1, 257, 1, 64, 2, 2, device="cuda")
expected = (
(rope * x.float().reshape(1, 257, 8, 64, 1, 2))
.sum(-1)
.reshape_as(x)
.to(x.dtype)
)
assert torch.equal(kandinsky6._apply_rotary(x, rope), expected)
assert len(calls) == 1
@pytest.mark.parametrize("device", ["cpu", "cuda"])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("strided", [False, True])
def test_rotary_cast_preserves_rounding_and_autograd(device, dtype, strided):
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("requires CUDA")
x = torch.randn(2, 17, 3, 64, device=device, requires_grad=True)
if strided:
x = x[:, ::2]
rope = torch.randn(1, x.shape[1], 1, 32, 2, 2, device=device)
rounded = x.to(dtype).float().reshape(2, x.shape[1], 3, 32, 1, 2)
expected = (rope * rounded).sum(-1).reshape_as(x).to(dtype)
actual = kandinsky6._apply_rotary(x, rope, dtype)
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
torch.testing.assert_close(
torch.autograd.grad(actual.sum(), x, retain_graph=True)[0],
torch.autograd.grad(expected.sum(), x)[0],
rtol=0,
atol=0,
)
with torch.no_grad():
torch.testing.assert_close(
kandinsky6._apply_rotary(x, rope, dtype), expected, rtol=0, atol=0
)
@torch.no_grad()
@pytest.mark.skipif(
not torch.cuda.is_available() or torch.version.hip is not None,
reason="CUDA required",
)
def test_gate_sum_preserves_fp32_intermediates():
x = torch.randn(2, 257, 128, dtype=torch.bfloat16, device="cuda")
update = torch.randn_like(x)
gate = torch.randn(2, 1, 1152, dtype=x.dtype, device=x.device)[..., 256:384]
expected = (x.float() + gate.float() * update.float()).to(x.dtype)
assert not torch.equal(expected, x + gate * update)
assert torch.equal(kandinsky6._apply_gate_sum(x, update, gate), expected)
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: Apache-2.0
"""Compare native Ulysses/Ring joint video-audio forwards with a single GPU."""
import os
from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
Kandinsky6VideoAudioConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
Kandinsky6TI2VAPipelineConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
)
from sglang.multimodal_gen.runtime.layers.attention.selector import (
global_force_attn_backend,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
Kandinsky6Transformer3DModel,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.server_args import set_global_server_args
from sglang.multimodal_gen.runtime.utils.precision import set_mixed_precision_policy
from sglang.srt.utils.network import get_free_port_below_ephemeral
def _forward(rank, world_size, ring, tp_size, port, output_dir, model_dim):
os.environ.update(
MASTER_ADDR="127.0.0.1",
MASTER_PORT=str(port),
RANK=str(rank),
LOCAL_RANK=str(rank),
WORLD_SIZE=str(world_size),
)
torch.cuda.set_device(rank)
set_mixed_precision_policy(torch.bfloat16, torch.float32)
global_force_attn_backend(AttentionBackendEnum.FA)
set_global_server_args(
SimpleNamespace(
attention_backend="fa",
attention_backend_config=None,
kv_gather_degree=1,
sp_split_auto=False,
pipeline_config=Kandinsky6TI2VAPipelineConfig(),
)
)
maybe_init_distributed_environment_and_model_parallel(
tp_size=tp_size,
sp_size=world_size // tp_size,
ulysses_degree=1 if ring else world_size // tp_size,
ring_degree=world_size // tp_size if ring else 1,
)
config = Kandinsky6VideoAudioConfig()
config.update_model_arch(
dict(
model_dim=model_dim,
time_dim=32,
ff_dim=512,
axes_dims=(32, 48, 48),
in_visual_dim=4,
out_visual_dim=4,
in_text_dim=16,
in_text_dim2=8,
in_audio_dim=4,
out_audio_dim=4,
num_text_blocks=1,
num_visual_blocks=2,
patch_size=(1, 1, 1),
visual_cond=False,
visual_token_type_num_embeddings=0,
is_multimodal=True,
model_dim_a=model_dim,
time_dim_a=32,
ff_dim_a=256,
axes_dims_a=(32, 48, 48),
ca_rope=True,
cross_gates=True,
fix_modulation=True,
)
)
model = Kandinsky6Transformer3DModel(config, {}).eval()
generator = torch.Generator().manual_seed(7)
weights_path = Path(output_dir) / "weights.pt"
with torch.no_grad():
if world_size == 1:
for parameter in model.parameters():
parameter.copy_(
torch.randn(parameter.shape, generator=generator) * 0.02
)
torch.save(model.state_dict(), weights_path)
else:
weights = torch.load(weights_path, weights_only=True)
for name, parameter in model.named_parameters():
weight_loader = parameter.__dict__.get("weight_loader")
if weight_loader is None:
parameter.copy_(weights[name])
else:
weight_loader(parameter, weights[name])
model = model.to(device=rank, dtype=torch.bfloat16)
for frames in (2, 3):
# odd visual sequence length exercises masked tail padding; audio stays odd
generator.manual_seed(frames)
def tensor(*shape):
return torch.randn(*shape, generator=generator).to(
device=rank, dtype=torch.bfloat16
)
with (
torch.no_grad(),
set_forward_context(current_timestep=0, attn_metadata=None),
):
video, audio = model(
hidden_states=tensor(1, frames, 3, 3, 4),
hidden_states_audio=tensor(1, 5, 4),
encoder_hidden_states=tensor(1, 7, 16),
pooled_projections=tensor(1, 8),
timestep=torch.tensor([500.0], device=rank),
visual_rope_pos=[torch.arange(n, device=rank) for n in (frames, 3, 3)],
text_rope_pos=torch.arange(7, device=rank),
)
assert video.shape == (1, frames, 3, 3, 4)
assert audio.shape == (1, 5, 4)
assert torch.isfinite(video).all() and torch.isfinite(audio).all()
path = Path(output_dir) / f"{frames}.pt"
if world_size == 1:
torch.save((video.cpu(), audio.cpu()), path)
else:
reference = torch.load(path, weights_only=True, map_location=f"cuda:{rank}")
for actual, expected in zip((video, audio), reference, strict=True):
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.002)
dist.destroy_process_group()
def _launch(world_size, ring, output_dir, tp_size=1, model_dim=256):
port = get_free_port_below_ephemeral()
mp.spawn(
_forward,
args=(world_size, ring, tp_size, port, output_dir, model_dim),
nprocs=world_size,
)
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="requires two CUDA GPUs")
def test_joint_video_audio_sequence_parallel(tmp_path):
_launch(1, False, str(tmp_path))
_launch(2, False, str(tmp_path))
_launch(2, True, str(tmp_path))
_launch(2, False, str(tmp_path), tp_size=2)
@pytest.mark.skipif(torch.cuda.device_count() < 4, reason="requires four CUDA GPUs")
def test_joint_video_audio_tensor_sequence_parallel(tmp_path):
_launch(1, False, str(tmp_path), model_dim=512)
for tp_size in (1, 2, 4):
_launch(4, False, str(tmp_path), tp_size=tp_size, model_dim=512)
_launch(4, True, str(tmp_path), tp_size=2, model_dim=512)
@@ -0,0 +1,305 @@
# SPDX-License-Identifier: Apache-2.0
"""Kandinsky 6 SR checkpoint configuration and CLI request validation."""
import argparse
import re
import pytest
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRArchConfig,
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
Kandinsky6SRVAEConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.configs.sample.kandinsky6 import (
Kandinsky6TI2VASamplingParams,
)
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
TINY_DIT = dict(
in_visual_dim=4,
in_text_dim=8,
in_text_dim2=8,
time_dim=16,
out_visual_dim=4,
patch_size=[1, 2, 2],
model_dim=32,
ff_dim=64,
num_text_blocks=0,
num_visual_blocks=2,
axes_dims=[8, 4, 4],
visual_cond=False,
instruct_type="noise",
attention_params={"512": {"type": "flash"}},
use_text=False,
)
TINY_PIFLOW = dict(
nfe=2,
num_policy_substeps=8,
final_step_size_scale=0.5,
shift=5.0,
n_grid=3,
eps=1e-6,
)
SR_PARAMS = dict(
scale_factor={"512": [1.0, 1.0, 1.0]},
visual_size=[512],
scheduler_scale=5.0,
lq_noise_scale=0.7,
lq_noise_type="ddpm",
lq_channel_noise_scale=0.0,
cap_noise_timestep=False,
fps=24,
)
def flat_config(**changes):
"""A converted ``transformer/config.json`` (pi-Flow DX head) with ``changes`` applied."""
flat = dict(TINY_DIT, _class_name="Kandinsky6SRTransformer3DModel", n_grid=3)
flat.update(
piflow_nfe=2,
piflow_num_policy_substeps=8,
piflow_final_step_size_scale=0.5,
piflow_shift=5.0,
piflow_eps=1e-6,
attribute_overrides={"instruct_type": "noise"},
sr_visual_size=[512],
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
sr_scheduler_scale=5.0,
sr_lq_noise_scale=0.7,
sr_lq_noise_type="ddpm",
sr_lq_channel_noise_scale=0.0,
sr_cap_noise_timestep=False,
sr_fps=24,
)
flat.update(changes)
return flat
def official_config(**changes):
"""An official ``transformer/config.json``: constructor kwargs, ``sr_params``, the TOTAL DX head
width in ``out_visual_dim`` (``base * n_grid``) and no ``n_grid`` / ``piflow_*`` keys.
"""
official = {
key: value for key, value in TINY_DIT.items() if key not in ("out_visual_dim",)
}
official.update(
out_visual_dim=TINY_DIT["out_visual_dim"] * TINY_PIFLOW["n_grid"],
attribute_overrides=None,
sr_params=dict(SR_PARAMS),
)
official.update(changes)
return official
def parse(flat):
config = Kandinsky6SRDitConfig()
config.update_model_arch(flat)
return config.arch_config
# --------------------------------------------------------------------------- #
# DiT arch config
# --------------------------------------------------------------------------- #
def test_arch_config_parses_an_official_config_and_maps_checkpoint_names():
arch = parse(official_config())
assert arch.n_grid == 1 and not arch.is_piflow # pi-Flow lives in the scheduler
assert arch.out_visual_dim == 12 and arch.attribute_overrides == {}
assert arch.patch_size == (1, 2, 2) and arch.axes_dims == (8, 4, 4)
assert (arch.hidden_size, arch.num_attention_heads) == (32, 2)
assert arch.sr_scale_factor == {"512": [1.0, 1.0, 1.0]} # JSON keys stay strings
assert (arch.sr_visual_size, arch.sr_scheduler_scale, arch.sr_fps) == (
[512],
5.0,
24,
)
map_name = get_param_names_mapping(arch.param_names_mapping)
assert map_name("pooled_bias")[0] == "pooled_bias"
assert (
map_name("visual_transformer_blocks.3.feed_forward.net.0.proj.weight")[0]
== "visual_transformer_blocks.3.feed_forward.mlp.fc_in.weight"
)
assert (
map_name("visual_transformer_blocks.0.feed_forward.net.2.weight")[0]
== "visual_transformer_blocks.0.feed_forward.mlp.fc_out.weight"
)
assert (
map_name("time_embeddings.timestep_embedder.linear_1.weight")[0]
== "time_embeddings.in_layer.weight"
)
# the attention output projection is not a feed-forward layer and keeps its name
assert (
map_name("visual_transformer_blocks.0.self_attention.out_layer.weight")[0]
== "visual_transformer_blocks.0.self_attention.out_layer.weight"
)
def test_unknown_sr_params_keys_are_rejected():
with pytest.raises(ValueError, match="sr_params has unsupported keys"):
parse(official_config(sr_params={"fps": 24, "new_knob": 1}))
@pytest.mark.parametrize(
"changes,error,flag",
[
({"use_text": True}, NotImplementedError, "use_text"),
({"use_adapter": True}, NotImplementedError, "use_adapter"),
({"use_lq_modulation": True}, NotImplementedError, "use_lq_modulation"),
({"surprise_key": 1}, ValueError, "surprise_key"),
({"attribute_overrides": {"n_grid": 2}}, ValueError, "n_grid"),
({"attribute_overrides": {"instruct_type": "bogus"}}, ValueError, "bogus"),
({"piflow_eps": None}, ValueError, "piflow_eps"),
({"patch_size": [2, 2, 2]}, ValueError, "patch_size"),
({"sr_lq_noise_type": "gauss"}, ValueError, "gauss"),
],
)
def test_unsupported_or_unknown_config_fails_loudly_naming_the_key(
changes, error, flag
):
"""A config the port cannot honor must never build a silently different model:
unsupported flags raise NotImplementedError naming the flag, unknown / inconsistent
keys raise ValueError (``update_model_arch`` alone would park unknown keys in
``extra_attrs``)."""
with pytest.raises(error, match=flag):
parse(flat_config(**changes))
def test_sparse_attention_request_is_detected_from_attention_params():
assert parse(flat_config()).requested_sparse_attention() is None
nabla = flat_config(attention_params={"512": {"type": "nabla", "P": 0.9}})
assert parse(nabla).requested_sparse_attention() == "nabla"
# an override can switch it off again
overridden = flat_config(
attention_params={"512": {"type": "nabla"}},
attribute_overrides={"attention_params": {"512": {"type": "flash"}}},
)
assert parse(overridden).requested_sparse_attention() is None
def test_default_arch_config_is_valid_and_text_free():
"""Registry / ``PipelineConfig.from_kwargs`` instantiate the default config."""
arch = Kandinsky6SRArchConfig()
assert arch.use_text is False and not arch.is_piflow and arch.n_grid == 1
# --------------------------------------------------------------------------- #
# VAE config
# --------------------------------------------------------------------------- #
def test_vae_config_parses_bundle_config_and_rejects_bad_ones():
config = Kandinsky6SRVAEConfig()
config.update_model_arch(
dict(
vae_type="video-kvae",
encoder_config={"ch": 8},
decoder_config={"ch": 8},
scaling_factor=0.53,
spatial_factor=16,
temporal_factor=4,
)
)
arch = config.arch_config
assert arch.scaling_factor == 0.53
assert (arch.spatial_compression_ratio, arch.temporal_compression_ratio) == (16, 4)
assert config.get_vae_scale_factor() == 16
with pytest.raises(ValueError, match="video-kvae"):
Kandinsky6SRVAEConfig().update_model_arch({"vae_type": "wan"})
with pytest.raises(ValueError, match="surprise"):
Kandinsky6SRVAEConfig().update_model_arch({"surprise": 1})
# --------------------------------------------------------------------------- #
# Pipeline config and sampling params
# --------------------------------------------------------------------------- #
def test_pipeline_config_is_text_free_monolithic_and_strict():
config = Kandinsky6SRPipelineConfig()
config.check_pipeline_config() # the four per-encoder tuples must stay consistent
assert config.text_encoder_configs == ()
assert config.supports_disaggregation() is False
# These components must load strictly and re-raise (no diffusers AutoModel fallback).
assert {"transformer", "vae", "latent_upscaler"} <= set(
config.native_only_components
)
def test_sampling_params_validate_the_sr_knobs():
params = Kandinsky6SRSamplingParams(video_path="clip.mp4")
assert (params.sr_resolution_scale, params.num_inference_steps) == (2.25, 4)
assert (params.sr_tiles_batch_size, params.sr_tile_min_overlap) == (1, 0.20)
assert params.seed == 42 and params.guidance_scale == 1.0
for bad in (
{"sr_resolution_scale": 3},
{"num_inference_steps": 0},
{"sr_tiles_batch_size": 0},
{"sr_tile_min_overlap": 1.0},
{"sr_target_resize_mode": "crop"},
{"sr_target_resolution": "big"},
):
with pytest.raises(ValueError):
Kandinsky6SRSamplingParams(video_path="clip.mp4", **bad)
for good in ("hd", "fullhd", "2k", "1280x720", "none"):
Kandinsky6SRSamplingParams(video_path="clip.mp4", sr_target_resolution=good)
def test_sr_flags_reach_the_sampling_params_through_the_generate_cli():
"""New SamplingParams fields are only CLI flags if the base ``add_cli_args`` declares
them; ``get_cli_args`` then filters by the resolved subclass' dataclass fields."""
parser = argparse.ArgumentParser()
SamplingParams.add_cli_args(parser)
args = parser.parse_args(
[
"--video-path",
"clip.mp4",
"--sr-resolution-scale",
"4",
"--num-inference-steps",
"3",
"--sr-tiles-batch-size",
"2",
"--sr-tile-min-overlap",
"0.3",
"--sr-target-resolution",
"fullhd",
"--sr-target-resize-mode",
"exact",
]
)
kwargs = Kandinsky6SRSamplingParams.get_cli_args(args)
params = Kandinsky6SRSamplingParams(**kwargs)
assert (
params.video_path == ["clip.mp4"] and params.source_video_path() == "clip.mp4"
)
assert (params.sr_resolution_scale, params.num_inference_steps) == (4.0, 3)
assert (params.sr_tiles_batch_size, params.sr_tile_min_overlap) == (2, 0.3)
assert (params.sr_target_resolution, params.sr_target_resize_mode) == (
"fullhd",
"exact",
)
# the sr_* knobs are ignored by other models' params instead of choking on them; the
# shared num_inference_steps flag, unlike those, is a core field every model gets.
assert "sr_tiles_batch_size" not in Kandinsky6TI2VASamplingParams.get_cli_args(args)
assert "num_inference_steps" in Kandinsky6TI2VASamplingParams.get_cli_args(args)
def test_request_needs_exactly_one_video_and_names_its_output_after_it():
with pytest.raises(ValueError, match="--video-path"):
Kandinsky6SRSamplingParams()._validate_with_pipeline_config(
Kandinsky6SRPipelineConfig()
)
with pytest.raises(ValueError, match="exactly one video"):
Kandinsky6SRSamplingParams(video_path=["a.mp4", "b.mp4"]).source_video_path()
params = Kandinsky6SRSamplingParams(video_path="/data/my clip.mp4", prompt=" ")
params._set_output_file_name()
assert re.fullmatch(r"my_clip_sr2\.25x_\d{8}-\d{6}\.mp4", params.output_file_name)
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
"""SR decode parallelism must preserve causal state and request isolation."""
import os
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
Kandinsky6SRVAEConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_decode_parallel_group_coordinator,
get_dp_rank,
maybe_init_distributed_environment_and_model_parallel,
)
from sglang.multimodal_gen.runtime.layers.parallel_conv import (
SpatialParallelConv3d,
disable_spatial_parallel_decode,
gather_and_trim_height,
split_height_for_parallel_decode,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import (
Kandinsky6SRVAE,
_ChunkedConv3d,
_SpatialChunkedConv3d,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
Kandinsky6SRDecodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_DENOISED_KEY,
SR_TILES_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
decode_chunks,
)
from sglang.multimodal_gen.runtime.server_args import set_global_server_args
from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import TINY_KVAE
from sglang.srt.utils.network import get_free_port_below_ephemeral
def _decode(rank, world_size, tp_size, dp_size, port, spatial):
os.environ.update(
MASTER_ADDR="127.0.0.1",
MASTER_PORT=str(port),
RANK=str(rank),
LOCAL_RANK=str(rank),
WORLD_SIZE=str(world_size),
)
torch.cuda.set_device(rank)
sp_size = world_size // tp_size // dp_size
maybe_init_distributed_environment_and_model_parallel(
tp_size=tp_size, sp_size=sp_size, ulysses_degree=sp_size, dp_size=dp_size
)
group = get_decode_parallel_group_coordinator()
assert group.world_size == world_size // dp_size
spatial_active = spatial and group.world_size > 1
if spatial_active:
torch.manual_seed(3 + get_dp_rank())
conv = _ChunkedConv3d(2, 3, 3, padding=(0, 1, 1)).to(rank)
parallel_conv = _SpatialChunkedConv3d.from_conv(conv)
assert parallel_conv.weight is conv.weight
x = torch.randn(1, 2, 9, 9, 5, device=rank)
with torch.inference_mode():
expected_conv = conv(x)
shard, height = split_height_for_parallel_decode(
x, x.shape[-2], group.world_size, group.rank_in_group
)
# force different temporal chunk counts on uneven height shards
with patch(
"sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae._MAX_CONV_NUMEL",
200,
):
actual_conv = gather_and_trim_height(parallel_conv(shard), height)
torch.testing.assert_close(actual_conv, expected_conv)
with disable_spatial_parallel_decode():
torch.testing.assert_close(parallel_conv(x), expected_conv)
config = Kandinsky6SRVAEConfig()
config.update_model_arch(
dict(
encoder_config=dict(TINY_KVAE, in_channels=3),
decoder_config=dict(TINY_KVAE, out_ch=3),
scaling_factor=0.5,
spatial_factor=2 ** (len(TINY_KVAE["ch_mult"]) - 1),
)
)
torch.manual_seed(17)
reference_vae = Kandinsky6SRVAE(config).eval().to(device=rank, dtype=torch.bfloat16)
config = replace(
config, use_parallel_decode=spatial, parallel_decode_mode="spatial_shard"
)
vae = Kandinsky6SRVAE(config).eval().to(device=rank, dtype=torch.bfloat16)
vae.load_state_dict(reference_vae.state_dict())
assert (
any(isinstance(m, SpatialParallelConv3d) for m in vae.decoder.modules())
== spatial_active
)
pipeline_config = Kandinsky6SRPipelineConfig(vae_config=config)
args = SimpleNamespace(pipeline_config=pipeline_config, component_precisions={})
set_global_server_args(args)
stage = Kandinsky6SRDecodeStage(vae)
stage.server_args = args
with torch.inference_mode():
for offload in (False, True):
if offload:
vae.configure_layerwise_offload(
SimpleNamespace(
performance_mode="speed",
pin_cpu_memory=True,
layerwise_tuning_for=lambda *args, **kwargs: (
1,
0,
"leading",
"forward",
),
)
)
first = None
# Uneven batches, idle ranks, two temporal segments, then a repeat.
for sizes, frames, height in (
((2, 2, 1), 9, 5),
((1,), 1, 1),
((2, 1), 5, 4),
((2, 2, 1), 9, 5),
):
generator = torch.Generator().manual_seed(7 + get_dp_rank())
chunks = [
torch.randn(size, frames, height, 3, 4, generator=generator)
for size in sizes
]
reference_vae.prepare_for_next_req()
expected = decode_chunks(
chunks,
reference_vae,
scaling_factor=reference_vae.scaling_factor,
device=torch.device("cuda", rank),
)
for parallel in (False, True):
config.use_parallel_tiling = parallel
vae.prepare_for_next_req()
request = Req(sampling_params=Kandinsky6SRSamplingParams())
request.extra[SR_DENOISED_KEY] = chunks
with (
patch.object(vae, "decode", wraps=vae.decode) as decode,
patch.object(
vae.decoder, "forward", wraps=vae.decoder.forward
) as decoder,
):
actual = stage.forward(request, args).extra[SR_TILES_KEY]
count = len(
range(group.rank_in_group, sum(sizes), group.world_size)
)
assert decode.call_count == (
count
if parallel and not spatial_active and sum(sizes) > 1
else sum(sizes)
)
local_height = height
if spatial_active and height >= group.world_size:
local_height = height // group.world_size + (
group.rank_in_group < height % group.world_size
)
assert all(
call.args[0].shape[-2] == local_height
for call in decoder.call_args_list
)
assert len(actual) == len(expected)
for tile, reference in zip(actual, expected):
assert tile.device.type == "cpu" and tile.dtype == torch.uint8
torch.testing.assert_close(
tile, reference, rtol=0, atol=2 if spatial_active else 0
)
if frames == 9:
if first is None:
first = actual
else:
for tile, reference in zip(actual, first):
torch.testing.assert_close(tile, reference, rtol=0, atol=0)
if offload:
vae.disable_offload()
dist.destroy_process_group()
@pytest.mark.parametrize(
"world_size,tp_size,dp_size",
[(1, 1, 1), (2, 2, 1), (2, 1, 1), (4, 2, 1), (4, 1, 2)],
)
@pytest.mark.parametrize("spatial", [False, True], ids=["tiles", "spatial"])
def test_sr_parallel_decode(world_size, tp_size, dp_size, spatial):
if torch.cuda.device_count() < world_size:
pytest.skip(f"requires {world_size} CUDA GPUs")
mp.spawn(
_decode,
args=(world_size, tp_size, dp_size, get_free_port_below_ephemeral(), spatial),
nprocs=world_size,
)
@@ -0,0 +1,290 @@
# SPDX-License-Identifier: Apache-2.0
"""Check SR prompt admission and request fields without running a GPU scheduler.
JSON uses TestClient; multipart tests call the parsed endpoint directly. The
full-checkpoint test_server_kandinsky6_sr.py also exercises actual file uploads.
"""
import asyncio
import os
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.runtime.entrypoints.openai import video_api
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import VideoResponse
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
resolve_sampling_params_cls,
)
from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import (
_build_video_sampling_params,
_multipart_video_extras,
)
_SR_EXTRA_FIELDS = frozenset(
{
"sr_resolution_scale",
"sr_tiles_batch_size",
"sr_tile_min_overlap",
"sr_target_resolution",
"sr_target_resize_mode",
}
)
def _fake_server_args(**overrides) -> SimpleNamespace:
base = dict(
pipeline_config=Kandinsky6SRPipelineConfig(),
pipeline_class_name="Kandinsky6SRPipeline",
backend="auto",
model_id=None,
model_path="kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers",
served_model_name="kandinsky6-sr-test",
input_save_path=os.path.expanduser(
"~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/uploads"
),
output_path=os.path.expanduser(
"~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/output"
),
attention_backend_config={},
enable_trace=False,
comfyui_mode=False,
num_gpus=1,
)
base.update(overrides)
return SimpleNamespace(**base)
def _patched_global_server_args(server_args):
return patch(
"sglang.multimodal_gen.runtime.server_args.server_args._global_server_args",
server_args,
)
# --------------------------------------------------------------------------- #
# The two declared pieces (SamplingParams-level)
# --------------------------------------------------------------------------- #
def test_prompt_optional_defaults_false_and_sr_overrides_it():
"""A generic pipeline keeps requiring ``prompt``; only the SR params opt out."""
assert SamplingParams.prompt_optional is False
assert Kandinsky6SRSamplingParams.prompt_optional is True
@pytest.mark.parametrize(
"sampling_params_cls, status",
[(Kandinsky6SRSamplingParams, 200), (SamplingParams, 400)],
)
def test_json_prompt_requirement_matches_pipeline(
tmp_path, sampling_params_cls, status
):
app = FastAPI()
app.include_router(video_api.router)
server_args = _fake_server_args(
input_save_path=str(tmp_path / "uploads"), output_path=str(tmp_path / "outputs")
)
dispatch = AsyncMock()
with (
_patched_global_server_args(server_args),
patch.object(
video_api, "resolve_sampling_params_cls", return_value=sampling_params_cls
),
patch.object(video_api, "_dispatch_job_async", dispatch),
TestClient(app) as client,
):
response = client.post(
"/v1/videos",
json={"video_path": "unused.mp4", "sr_resolution_scale": 4},
)
assert response.status_code == status, response.text
if status == 200:
assert response.json()["status"] == "queued"
dispatch.assert_awaited_once()
_, batch = dispatch.call_args.args
assert batch.prompt == ""
assert batch.sampling_params.sr_resolution_scale == 4
else:
dispatch.assert_not_called()
def test_sr_declares_its_cli_flags_as_multipart_extra_fields():
assert Kandinsky6SRSamplingParams.video_request_extra_fields() == _SR_EXTRA_FIELDS
# The base default (what every other model still gets) stays empty.
assert SamplingParams.video_request_extra_fields() == frozenset()
# --------------------------------------------------------------------------- #
# The real multipart-field allowlist (video_api._multipart_video_extras)
# --------------------------------------------------------------------------- #
def test_multipart_extras_keep_sr_fields_only_for_the_sr_params_class():
raw_form = {
"video_path": "unused.mp4",
"sr_resolution_scale": "4",
"sr_tiles_batch_size": "3",
"not_a_declared_field": "dropped",
}
sr_extras = _multipart_video_extras(
raw_form,
extra_body=None,
extra_params=None,
sampling_params_cls=Kandinsky6SRSamplingParams,
)
assert sr_extras["sr_resolution_scale"] == 4 # JSON-decoded, not the raw "4"
assert sr_extras["sr_tiles_batch_size"] == 3
assert "not_a_declared_field" not in sr_extras
# A model without the override (the pre-fix behaviour) drops the sr_* fields too.
base_extras = _multipart_video_extras(
raw_form,
extra_body=None,
extra_params=None,
sampling_params_cls=SamplingParams,
)
assert "sr_resolution_scale" not in base_extras
assert "sr_tiles_batch_size" not in base_extras
# --------------------------------------------------------------------------- #
# The registry-backed class resolution the admission gate reads
# --------------------------------------------------------------------------- #
def test_video_sampling_params_cls_resolves_to_sr_for_the_sr_pipeline():
server_args = _fake_server_args()
assert resolve_sampling_params_cls(server_args) is Kandinsky6SRSamplingParams
# --------------------------------------------------------------------------- #
# End-to-end request parsing: a prompt-less request field set -> sampling params
# --------------------------------------------------------------------------- #
def test_build_video_sampling_params_accepts_no_prompt_and_carries_sr_fields():
"""The real kwargs-building code (``_build_video_sampling_params``), given a
request built the way the multipart branch builds one for a prompt-optional
pipeline (``prompt=prompt or ""`` + the surviving sr_* extras), must construct a
real ``Kandinsky6SRSamplingParams`` with those fields set and no prompt error."""
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
VideoGenerationsRequest,
)
server_args = _fake_server_args()
raw_form = {
"video_path": "unused.mp4",
"sr_resolution_scale": "4",
"sr_tiles_batch_size": "3",
}
extras = _multipart_video_extras(
raw_form,
extra_body=None,
extra_params=None,
sampling_params_cls=Kandinsky6SRSamplingParams,
)
req = VideoGenerationsRequest(
prompt="", # what video_api.py substitutes for a missing prompt
video_path=extras.pop("video_path"),
**extras,
)
with _patched_global_server_args(server_args):
sampling_params = _build_video_sampling_params("gap1-request-id", req)
assert isinstance(sampling_params, Kandinsky6SRSamplingParams)
assert sampling_params.prompt == ""
assert sampling_params.sr_resolution_scale == 4
assert sampling_params.sr_tiles_batch_size == 3
# --------------------------------------------------------------------------- #
# The full endpoint, called directly (see module docstring for why not TestClient)
# --------------------------------------------------------------------------- #
class _FakeMultipartRequest:
"""Duck-types the slice of ``fastapi.Request`` ``create_video`` reads directly."""
def __init__(self, form: dict):
self.headers = {"content-type": "multipart/form-data; boundary=x"}
self._form = form
async def form(self):
return self._form
def test_multipart_sr_request_without_prompt_is_accepted():
"""A multipart SR request with no ``prompt`` field must be admitted (no 400), the
``sr_*`` fields must reach the queued request, and it must fail on the pre-fix
code: the old unconditional ``if not prompt: raise HTTPException(...)`` gate would
reject this before any of the rest of ``create_video`` ran."""
server_args = _fake_server_args()
request = _FakeMultipartRequest(
{"sr_resolution_scale": "4", "sr_tiles_batch_size": "3"}
)
dispatched: list[tuple] = []
async def _fake_dispatch(job_id, batch, **kwargs):
dispatched.append((job_id, batch, kwargs))
async def run():
with (
_patched_global_server_args(server_args),
patch.object(video_api, "_dispatch_job_async", side_effect=_fake_dispatch),
):
response = await video_api.create_video(
request,
prompt=None, # <-- the field under test: no prompt supplied
enhance_prompt=None,
task_type=None,
x264_preset=None,
perf_dump_path=None,
input_reference=None,
reference_url=None,
video_reference=None,
video_url=None,
video_path="unused.mp4",
model=None,
n=1,
num_outputs_per_prompt=None,
seconds=None,
size=None,
fps=None,
num_frames=None,
seed=None,
generator_device="cuda",
negative_prompt=None,
guidance_scale=None,
guidance_scale_2=None,
true_cfg_scale=None,
num_inference_steps=None,
max_sequence_length=None,
flow_shift=None,
enable_teacache=None,
enable_frame_interpolation=None,
frame_interpolation_exp=None,
frame_interpolation_scale=None,
frame_interpolation_model_path=None,
enable_upscaling=None,
upscaling_model_path=None,
upscaling_scale=None,
output_quality=None,
output_compression=None,
output_path=None,
extra_params=None,
extra_body=None,
)
await asyncio.sleep(0)
return response
response = asyncio.run(run())
assert isinstance(response, VideoResponse)
assert response.status == "queued"
assert len(dispatched) == 1
_, batch, _ = dispatched[0]
assert batch.prompt == ""
assert batch.sampling_params.sr_resolution_scale == 4
assert batch.sampling_params.sr_tiles_batch_size == 3
@@ -0,0 +1,467 @@
# SPDX-License-Identifier: Apache-2.0
"""Strict loading, dtype and config contracts for tiny Kandinsky 6 SR components."""
import copy
import json
import os
import pytest
import torch
from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL
from safetensors.torch import load_file, save_file
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
Kandinsky6SRVAEConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.loader.component_loaders import (
component_loader as component_loader_module,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.latent_upscaler_loader import (
LatentUpscalerLoader,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.vae_loader import VAELoader
from sglang.multimodal_gen.runtime.loader.fsdp_load import (
load_model_from_full_model_state_dict,
)
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
Kandinsky6SRTransformer3DModel,
)
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
Kandinsky6SRLatentUpscalerBank,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
TINY_DIT = dict(
in_visual_dim=4,
in_text_dim=8,
in_text_dim2=8,
time_dim=16,
out_visual_dim=4,
patch_size=[1, 2, 2],
model_dim=32,
ff_dim=64,
num_text_blocks=0,
num_visual_blocks=2,
axes_dims=[8, 4, 4],
visual_cond=False,
instruct_type="noise",
use_text=False,
n_grid=3,
attribute_overrides={"instruct_type": "noise"},
)
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29508",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
@pytest.fixture(autouse=True)
def cpu_only(monkeypatch):
"""Load on CPU; the forward test moves its DiT to CUDA for fused kernels."""
monkeypatch.setattr(
component_loader_module.ComponentLoader,
"target_device",
staticmethod(lambda component_starts_on_cpu: torch.device("cpu")),
)
def _dit_config(**changes):
flat = dict(TINY_DIT, **changes)
config = Kandinsky6SRDitConfig()
config.update_model_arch(flat)
return config, flat
def _dit(**changes):
"""Tiny DiT with deterministic random weights (the linears start uninitialized)."""
config, flat = _dit_config(**changes)
model = Kandinsky6SRTransformer3DModel(config, flat)
generator = torch.Generator().manual_seed(0)
with torch.no_grad():
for param in model.parameters():
param.copy_(torch.randn(param.shape, generator=generator) * 0.05)
return model
def _official_state_dict(model):
"""State dict as the current Diffusers checkpoint stores it."""
return {
name.replace(".feed_forward.mlp.fc_in.", ".feed_forward.net.0.proj.")
.replace(".feed_forward.mlp.fc_out.", ".feed_forward.net.2.")
.replace(
"time_embeddings.in_layer.",
"time_embeddings.timestep_embedder.linear_1.",
)
.replace(
"time_embeddings.out_layer.",
"time_embeddings.timestep_embedder.linear_2.",
): tensor
for name, tensor in model.state_dict().items()
}
def _load_through_the_real_loader(checkpoint, **changes):
"""Meta-init the DiT and load ``checkpoint`` like ``maybe_load_fsdp_model`` does."""
config, flat = _dit_config(**changes)
with torch.device("meta"):
model = Kandinsky6SRTransformer3DModel(config, flat)
load_model_from_full_model_state_dict(
model,
model.preprocess_loaded_state_dict(iter(checkpoint.items())),
torch.device("cpu"),
torch.float32,
strict=False,
param_names_mapping=get_param_names_mapping(model.param_names_mapping),
)
return model
# --------------------------------------------------------------------------- #
# DiT
# --------------------------------------------------------------------------- #
def test_complete_checkpoint_loads_and_meta_buffers_are_rebuilt():
"""Positive control for the failure tests below, and the meta-device path: RoPE
tables and time-embedding frequencies are not in the checkpoint and must be rebuilt
by ``post_load_weights`` so that a loaded model actually runs."""
reference = _dit()
checkpoint = _official_state_dict(reference)
model = _load_through_the_real_loader(checkpoint)
model.post_load_weights()
assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()])
assert not model.time_embeddings.freqs.is_meta
torch.testing.assert_close(model.pooled_bias, reference.pooled_bias)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device)
x = torch.randn(1, 2, 4, 4, 4, device=device)
rope_pos = [torch.arange(2, device=device) for _ in range(3)]
out = model(x, torch.tensor([500.0], device=device), rope_pos)
assert out.shape == (1, 2, 4, 4, 12) and torch.isfinite(out).all()
@pytest.mark.parametrize(
"missing",
[
"pooled_bias",
"visual_transformer_blocks.0.visual_modulation.out_layer.bias",
],
)
def test_missing_checkpoint_tensor_fails_the_load(missing):
"""The generic loader zero-fills missing parameters whose names contain ``bias`` (and
would only warn about others). For this model every parameter is required, so a
partial checkpoint must raise instead of yielding a silently different network."""
checkpoint = _official_state_dict(_dit())
del checkpoint[missing]
with pytest.raises(ValueError, match="was not loaded"):
_load_through_the_real_loader(checkpoint)
def test_unexpected_checkpoint_tensor_fails_the_load():
checkpoint = _official_state_dict(_dit())
checkpoint["text_embeddings.in_layer.weight"] = torch.zeros(2, 2)
with pytest.raises(ValueError, match="unexpected keys"):
_load_through_the_real_loader(checkpoint)
def test_official_config_builds_the_head_from_the_total_width_and_loads_official_keys():
"""An official config stores the TOTAL DX head width in ``out_visual_dim`` (n_grid lives
in the scheduler config) plus nested ``sr_params`` and ``attribute_overrides: null``.
"""
official = {
key: value
for key, value in TINY_DIT.items()
if key not in ("n_grid", "attribute_overrides")
}
official.update(
out_visual_dim=12,
attribute_overrides=None,
sr_params=dict(
scale_factor={"512": [1.0, 1.0, 1.0]},
visual_size=[512],
scheduler_scale=5.0,
lq_noise_scale=0.7,
lq_noise_type="ddpm",
lq_channel_noise_scale=0.0,
cap_noise_timestep=False,
fps=24,
),
)
config = Kandinsky6SRDitConfig()
config.update_model_arch(official)
with torch.device("meta"):
model = Kandinsky6SRTransformer3DModel(config, official)
# head width = prod(patch_size) * the TOTAL out_visual_dim
assert model.state_dict()["out_layer.out_layer.weight"].shape[0] == 4 * 12
assert model.base_out_visual_dim == 4 and model.n_grid == 1
load_model_from_full_model_state_dict(
model,
model.preprocess_loaded_state_dict(
iter(
_official_state_dict(_dit()).items()
) # same shapes as the legacy DX head
),
torch.device("cpu"),
torch.float32,
strict=False,
param_names_mapping=get_param_names_mapping(model.param_names_mapping),
)
model.post_load_weights()
assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()])
def test_wide_input_checkpoint_needs_visual_cond_to_run_under_noise():
"""A hybrid_anchor checkpoint has a ``2 * in + 1`` input layer; the default ``noise``
override would feed ``in`` channels. That mismatch must be reported, and the
documented ``visual_cond`` override must make it run."""
wide = dict(
instruct_type="hybrid_anchor", attribute_overrides={"instruct_type": "noise"}
)
with pytest.raises(ValueError, match="visual_cond"):
_dit(**wide)
model = _dit(
instruct_type="hybrid_anchor",
attribute_overrides={"instruct_type": "noise", "visual_cond": True},
)
assert model.instruct_type == "noise" and model.visual_cond is True
assert model.visual_embed_dim == 9
assert model.config.instruct_type == "hybrid_anchor" # the trained value is kept
def test_rope_angle_tables_stay_fp32_when_the_module_is_cast():
"""The residency manager casts a module to its target dtype (bf16) on every move to
the GPU; the RoPE angle tables must not be rounded to bf16 with the weights."""
model = _dit().to(torch.bfloat16)
assert model.pooled_bias.dtype == torch.bfloat16
for name, buffer in model.visual_rope_embeddings.named_buffers():
assert buffer.dtype == torch.float32, name
# --------------------------------------------------------------------------- #
# Fake server args for the component loaders
# --------------------------------------------------------------------------- #
class FakeServerArgs:
def __init__(self):
self.pipeline_config = Kandinsky6SRPipelineConfig()
self.model_paths = {}
self.component_paths = {}
self.component_precisions = {}
self.component_weights_paths = {}
self.component_quantizations = {}
self.num_gpus = 1
self.disable_autocast = False
self.revision = None
self.trust_remote_code = False
def should_start_component_on_cpu(self, component_name):
return False
def should_direct_gpu_weight_load_component(self, component_name):
return False
def should_use_fsdp_for_component(self, component_name):
return False
def requested_component_attention_backend(self, component_name):
return None
def resolve_component_attention_backend(self, *names):
return None, None
def _write_component(directory, config, tensors):
directory.mkdir(parents=True, exist_ok=True)
(directory / "config.json").write_text(json.dumps(config))
save_file(tensors, str(directory / "diffusion_pytorch_model.safetensors"))
def _tiny_vae_bundle(tmp_path):
raw_config = dict(
_class_name="Kandinsky6SRVAE",
vae_type="video-kvae",
encoder_config=dict(TINY_KVAE, in_channels=3),
decoder_config=dict(TINY_KVAE, out_ch=3),
scaling_factor=0.5,
spatial_factor=16,
temporal_factor=4,
)
config = Kandinsky6SRVAEConfig()
config.update_model_arch(raw_config)
wrapper = Kandinsky6SRVAE(config)
directory = tmp_path / "vae"
_write_component(
directory,
raw_config,
{k: v.contiguous() for k, v in wrapper.state_dict().items()},
)
return directory
def test_vae_loader_loads_the_wrapper_strictly(tmp_path):
directory = _tiny_vae_bundle(tmp_path)
server_args = FakeServerArgs()
vae = VAELoader().load_customized(str(directory), server_args, "vae")
assert isinstance(vae, Kandinsky6SRVAE) and vae.scaling_factor == 0.5
assert next(vae.parameters()).dtype == torch.bfloat16 # vae_precision
assert (vae.spatial_factor, vae.temporal_factor) == (16, 4)
# one missing tensor must fail: non-strict VAE loading would only log a warning
weights_path = directory / "diffusion_pytorch_model.safetensors"
weights = load_file(str(weights_path))
torch.testing.assert_close(
vae.state_dict(),
{name: value.bfloat16() for name, value in weights.items()},
rtol=0,
atol=0,
)
weights.pop(next(iter(weights)))
save_file(weights, str(weights_path))
with pytest.raises(RuntimeError, match="Missing key"):
VAELoader().load_customized(str(directory), FakeServerArgs(), "vae")
def _tiny_lu_bundle(tmp_path, models=None, scales=(2, 4), include_scales=True):
models = models or [
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
for scale in ("2x", "4x")
]
bank = Kandinsky6SRLatentUpscalerBank(models, 0.5, scales=scales)
directory = tmp_path / "latent_upscaler"
config = {
"_class_name": "Kandinsky6SRLatentUpscalerBank",
"models": models,
"scaling_factor": 0.5,
}
if include_scales:
config["scales"] = list(scales)
_write_component(
directory,
config,
{k: v.contiguous() for k, v in bank.state_dict().items()},
)
return directory
def test_latent_upscaler_loader_loads_bank_in_bf16_eval(tmp_path):
directory = _tiny_lu_bundle(tmp_path)
bank, _ = LatentUpscalerLoader().load(
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
)
assert bank.scales == (2, 4)
assert not bank.training
assert all(
p.dtype == torch.bfloat16 and not p.requires_grad for p in bank.parameters()
)
z = torch.randn(1, 4, 3, 4, 6, dtype=torch.bfloat16)
assert bank.upscale(z, scale=2).shape == (1, 4, 3, 8, 12)
assert bank.upscale(z, scale=4).shape == (1, 4, 3, 16, 24)
with pytest.raises(ValueError, match="no x8 entry"):
bank.upscale(z, scale=8)
def test_latent_upscaler_legacy_config_uses_current_default_scale_order(tmp_path):
models = [
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
for scale in ("4x", "2x")
]
directory = _tiny_lu_bundle(
tmp_path,
models=models,
include_scales=False,
)
bank, _ = LatentUpscalerLoader().load(
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
)
assert bank.scales == (2, 4)
assert bank._models[0] is bank.for_scale(2)
assert bank._models[1] is bank.for_scale(4)
def test_latent_upscaler_may_be_declared_with_the_kandinsky6_library():
"""The official ``save_pretrained`` writes ["kandinsky6", ...], the Hub repo ["diffusers", ...]."""
for library in ("diffusers", "kandinsky6"):
loader = component_loader_module.ComponentLoader.for_component_type(
"latent_upscaler", library
)
assert isinstance(loader, LatentUpscalerLoader)
with pytest.raises(AssertionError, match="latent_upscaler must be loaded from"):
component_loader_module.ComponentLoader.for_component_type(
"latent_upscaler", "transformers"
)
def test_latent_upscaler_load_errors_are_raised_not_swallowed(tmp_path):
"""``ComponentLoader.load`` falls back to a diffusers ``AutoModel`` when the
customized loader raises, which hides real errors. This component must re-raise,
with the offending config key or tensor named."""
directory = _tiny_lu_bundle(tmp_path)
weights_path = directory / "diffusion_pytorch_model.safetensors"
weights = load_file(str(weights_path))
weights.pop(next(iter(weights)))
save_file(weights, str(weights_path))
with pytest.raises(RuntimeError, match="native fallback is disabled") as error:
LatentUpscalerLoader().load(
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
)
assert "Missing key" in str(error.value.__cause__)
bad = copy.deepcopy(TINY_LU_MODEL)
bad["bogus_field"] = 1
directory = _tiny_lu_bundle(
tmp_path / "bad",
models=[{"target_scale": "2x", "model": TINY_LU_MODEL}],
scales=(2,),
)
config_path = directory / "config.json"
config = json.loads(config_path.read_text())
config["models"][0]["model"] = bad
config_path.write_text(json.dumps(config))
with pytest.raises(RuntimeError, match="native fallback is disabled") as error:
LatentUpscalerLoader().load(
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
)
assert "bogus_field" in str(error.value.__cause__)
def test_motion_attention_is_not_a_supported_entry_key():
"""The consolidated architecture (mirroring FastVideo's ``kandinsky6_sr.py`` /
commit 950a5edb) only implements the released checkpoint, which never trains
natten/shifted-window motion attention: a config that asks for it must fail loudly
at construction instead of silently building an unsupported backend."""
with_motion = {
**copy.deepcopy(TINY_LU_MODEL),
"enable_x2_entry": False,
"motion_attention": {
"spatial_kernel_size": 3,
"temporal_offsets": [-1, 1],
"after_mid_blocks": [1],
"num_heads": 2,
},
}
with pytest.raises(ValueError, match="motion_attention"):
Kandinsky6SRLatentUpscalerBank(
[{"target_scale": "4x", "model": with_motion}], 0.5
)
@@ -0,0 +1,369 @@
# SPDX-License-Identifier: Apache-2.0
"""Validate release config shapes and component loading without downloading weights."""
import copy
import json
import os
from collections import Counter
from types import SimpleNamespace
from typing import NamedTuple
import pytest
import torch
from kandinsky6_sr_release_configs import (
DISTILLED_MODEL_INDEX,
DISTILLED_SCHEDULER_CONFIG,
DISTILLED_TRANSFORMER_CONFIG,
FLOW_MODEL_INDEX,
FLOW_SCHEDULER_CONFIG,
FLOW_SR_CONFIG,
FLOW_TRANSFORMER_CONFIG,
LATENT_UPSCALER_CONFIG,
VAE_CONFIG,
)
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
Kandinsky6SRVAEConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
PipelineComponentLoader,
)
from sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader import (
SchedulerLoader,
)
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
Kandinsky6SRTransformer3DModel,
)
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
PiflowScheduler,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
)
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
Kandinsky6SRLatentUpscalerBank,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import (
Kandinsky6SRPipeline,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
build_sampling_spec,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
PiflowParams,
)
class Bundle(NamedTuple):
"""One official repo: its configs and what they must resolve to."""
model_index: dict
transformer: dict
scheduler: dict
sr_config: dict | None
scheduler_cls: type
head_width: int # ``out_visual_dim`` of the transformer: the TOTAL head width
n_grid: int # grids of the head: 1 (flow-matching) or the scheduler's ``n_grid``
scheduler_scale: float # ``sr_params.scheduler_scale``
calls_per_tile: int # DiT calls per tile at the default ``num_inference_steps`` 5
FLOW = Bundle(
model_index=FLOW_MODEL_INDEX,
transformer=FLOW_TRANSFORMER_CONFIG,
scheduler=FLOW_SCHEDULER_CONFIG,
sr_config=FLOW_SR_CONFIG,
scheduler_cls=FlowMatchEulerDiscreteScheduler,
head_width=64,
n_grid=1,
scheduler_scale=5.0,
calls_per_tile=5,
)
DISTILLED = Bundle(
model_index=DISTILLED_MODEL_INDEX,
transformer=DISTILLED_TRANSFORMER_CONFIG,
scheduler=DISTILLED_SCHEDULER_CONFIG,
sr_config=None,
scheduler_cls=PiflowScheduler,
head_width=640,
n_grid=10,
scheduler_scale=3.5,
calls_per_tile=2,
)
both_repos = pytest.mark.parametrize(
"bundle", [FLOW, DISTILLED], ids=["flow_matching", "distilled_pi_flow"]
)
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29509",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
def _prefixes(keys, depth=2):
return Counter(".".join(key.split(".")[:depth]) for key in keys)
def _dit_config(bundle):
config = Kandinsky6SRDitConfig()
config.update_model_arch(copy.deepcopy(bundle.transformer))
return config
def _load_scheduler(bundle, tmp_path):
"""The scheduler component through SGLang's own loader (a ``scheduler/`` dir)."""
directory = tmp_path / "scheduler"
directory.mkdir()
(directory / "scheduler_config.json").write_text(json.dumps(bundle.scheduler))
server_args = SimpleNamespace(pipeline_config=Kandinsky6SRPipelineConfig())
return SchedulerLoader().load_customized(str(directory), server_args)
@both_repos
def test_real_transformer_config_builds_the_release_dit(bundle):
config = _dit_config(bundle)
with torch.device("meta"):
model = Kandinsky6SRTransformer3DModel(
config, copy.deepcopy(bundle.transformer)
)
state = model.state_dict()
per_block = Counter(
key.split(".")[1]
for key in state
if key.startswith("visual_transformer_blocks.")
)
assert (
len(state) == 459 and len(per_block) == 32 and set(per_block.values()) == {14}
)
non_block = [k for k in state if not k.startswith("visual_transformer_blocks.")]
assert len(non_block) == 11
# ``out_visual_dim`` is the TOTAL head width (64 * n_grid); a pi-Flow repo keeps its
# n_grid in the scheduler config
assert state["out_layer.out_layer.weight"].shape == (bundle.head_width, 1792)
assert model.base_out_visual_dim == 64 and model.n_grid == 1
assert bundle.n_grid * model.base_out_visual_dim == bundle.head_width
assert model.instruct_type == "hybrid_anchor" and model.visual_cond
assert model.visual_embed_dim == 2 * 64 + 1 # wide [x | cond | mask] input
arch = config.arch_config
assert arch.sr_scale_factor == {"512": [1.0, 2.0, 2.0]}
assert arch.sr_scheduler_scale == bundle.scheduler_scale and not arch.is_piflow
@both_repos
def test_real_scheduler_config_loads_through_the_scheduler_loader(bundle, tmp_path):
scheduler = _load_scheduler(bundle, tmp_path)
# PiflowScheduler subclasses the flow-matching scheduler: compare exact types
assert type(scheduler) is bundle.scheduler_cls
assert scheduler.config["shift"] == bundle.scheduler["shift"]
if bundle is DISTILLED:
assert (scheduler.config["nfe"], scheduler.config["n_grid"]) == (2, 10)
@both_repos
def test_real_scheduler_selects_the_sampler_of_its_repo(bundle, tmp_path):
"""The distilled repo resolves to pi-Flow (``nfe`` = 2 calls per tile, whatever
``num_inference_steps`` is), the flow-matching repo to flow-Euler
(``num_inference_steps`` calls per tile directly -- this repo's own convention, not
the upstream Diffusers pipeline's timestep-grid-point count).
"""
scheduler = _load_scheduler(bundle, tmp_path)
arch = _dit_config(bundle).arch_config
def spec(num_steps):
return build_sampling_spec(
arch=arch,
tiling_scale=2,
seed=42,
num_steps=num_steps,
tiles_batch_size=1,
tile_min_overlap=0.2,
scheduler=scheduler,
)
assert spec(5).steps_per_chunk == bundle.calls_per_tile
assert spec(5).scheduler_scale == bundle.scheduler_scale
if bundle is DISTILLED:
assert spec(5).piflow == PiflowParams(
nfe=2,
num_policy_substeps=128,
final_step_size_scale=0.5,
shift=3.5,
n_grid=10,
eps=1e-6,
)
assert spec(3).steps_per_chunk == spec(9).steps_per_chunk == 2
else:
assert spec(5).piflow is None
assert (spec(3).steps_per_chunk, spec(9).steps_per_chunk) == (3, 9)
def test_real_vae_config_builds_the_release_kvae_and_maps_official_keys():
config = Kandinsky6SRVAEConfig()
config.update_model_arch(dict(VAE_CONFIG))
with torch.device("meta"):
vae = Kandinsky6SRVAE(config)
# `self.encoder` / `self.decoder` are direct attributes of the wrapper (no intermediate
# `model.*` nesting), so its state_dict keys already are the official checkpoint's own
# `encoder.*` / `decoder.*` keys -- no stripping needed.
official = dict(vae.state_dict())
assert len(official) == 378
assert _prefixes(official) == Counter(
{
"decoder.up": 238,
"encoder.down": 86,
"decoder.mid": 28,
"encoder.mid": 12,
"decoder.norm_out": 5,
"decoder.conv_in": 2,
"decoder.conv_out": 2,
"encoder.conv_in": 2,
"encoder.conv_out": 2,
"encoder.norm_out": 1,
}
)
assert vae.scaling_factor == pytest.approx(0.910344004631042)
assert vae.spatial_factor == 16
# the official keys load strictly, with no remapping
vae.load_state_dict(official, strict=True, assign=True)
def test_real_latent_upscaler_config_builds_the_release_bank():
with torch.device("meta"):
bank = Kandinsky6SRLatentUpscalerBank(
LATENT_UPSCALER_CONFIG["models"],
LATENT_UPSCALER_CONFIG["scaling_factor"],
scales=(2, 4),
)
state = bank.state_dict()
assert len(state) == 489
assert _prefixes(state) == Counter({"_models.0": 311, "_models.1": 178})
x2 = _prefixes((key.split(".", 2)[2] for key in state if "_models.0." in key))
assert x2["x2_branch.blocks"] == 44 and x2["x2_branch.adapter"] == 28
assert bank.scales == (2, 4)
@both_repos
@pytest.mark.parametrize("upscaler", ["present", "missing", "null"])
def test_real_model_index_is_accepted_by_the_module_loading_path(
bundle, tmp_path, monkeypatch, upscaler
):
"""``ComposedPipelineBase.load_modules`` sees a model_index.json whose extra
``_kandinsky6_sr`` entry is a dict (not a ``[library, class]`` pair), next to a root
``sr_config.json``: only the four components of the pipeline may be requested, with
the scheduler class of the repo."""
model_index = dict(bundle.model_index)
if upscaler == "missing":
model_index.pop("latent_upscaler")
elif upscaler == "null":
model_index["latent_upscaler"] = None
(tmp_path / "model_index.json").write_text(json.dumps(model_index))
if bundle.sr_config is not None:
(tmp_path / "sr_config.json").write_text(json.dumps(bundle.sr_config))
components = ["transformer", "vae", "scheduler"]
if upscaler == "present":
components.append("latent_upscaler")
for name in components:
(tmp_path / name).mkdir()
if name != "scheduler": # the scheduler is config-only
(tmp_path / name / "diffusion_pytorch_model.safetensors").touch()
requested = {}
def record_component(
component_name,
component_model_path,
transformers_or_diffusers,
server_args,
component_architecture=None,
component_type=None,
loader_cls=None,
component_attn_backend=None,
component_attn_name=None,
):
assert component_type == component_name
assert loader_cls is None and component_attn_backend is None
assert component_attn_name == component_name
requested[component_name] = (transformers_or_diffusers, component_architecture)
return SimpleNamespace(name=component_name), 0.0
monkeypatch.setattr(
PipelineComponentLoader, "load_component", staticmethod(record_component)
)
server_args = SimpleNamespace(
model_subfolder=None,
revision=None,
comfyui_mode=False,
component_paths={},
component_direct_gpu_weight_loading={},
pipeline_config=Kandinsky6SRPipelineConfig(),
resolve_component_attention_backend=lambda *names: (None, None),
)
# the state ``ComposedPipelineBase.__init__`` sets up before it calls load_modules
pipeline = object.__new__(Kandinsky6SRPipeline)
pipeline.server_args = server_args
pipeline.model_path = str(tmp_path)
pipeline._required_config_modules = list(
Kandinsky6SRPipeline._required_config_modules
)
pipeline._extra_config_module_map = {}
pipeline._disagg_role = RoleType.MONOLITHIC
pipeline.memory_usages = {}
modules = pipeline.load_modules(server_args)
expected = {
"transformer": ("diffusers", "Kandinsky6SRTransformer3DModel"),
"vae": ("diffusers", "Kandinsky6SRVAE"),
"latent_upscaler": ("diffusers", "Kandinsky6SRLatentUpscalerBank"),
"scheduler": ("diffusers", bundle.scheduler_cls.__name__),
}
if upscaler != "present":
expected.pop("latent_upscaler")
assert set(modules) == set(expected)
assert requested == expected
assert "latent_upscaler" in Kandinsky6SRPipeline._required_config_modules
def test_legacy_bundle_requires_official_diffusers_layout(tmp_path):
(tmp_path / "model_index.json").write_text(
json.dumps(
{
"_class_name": "Kandinsky6SRPipeline",
"_diffusers_version": "0.37.0",
"dit": ["x", "Kandinsky6SRDiT"],
"vae": ["x", "Kandinsky6SRVAE"],
}
)
)
for name in ("dit", "vae"):
(tmp_path / name).mkdir()
(tmp_path / name / "diffusion_pytorch_model.safetensors").touch()
pipeline = object.__new__(Kandinsky6SRPipeline)
pipeline.server_args = SimpleNamespace(model_subfolder=None, revision=None)
pipeline.model_path = str(tmp_path)
with pytest.raises(
ValueError, match="k6_video SR bundle.*kandinskylab/Kandinsky-6.0-VSR"
):
pipeline.load_modules(pipeline.server_args)
@@ -0,0 +1,829 @@
# SPDX-License-Identifier: Apache-2.0
"""CPU parity of the Kandinsky 6 SR port against the k6_video reference (``kandinsky_sr``).
The reference package is not part of this repository. Point
``KANDINSKY_SR_REFERENCE_SRC`` at the directory that contains ``kandinsky_sr`` (for
example ``k6_video/src``); without it the whole module is skipped. The reference needs
``flash_attn`` and ``loguru``, which are stubbed here (per-sequence SDPA for the dense
varlen attention), and its base resolutions are shrunk so that tiles stay tiny. All
components use tiny random weights (every parameter re-randomized: the reference
zero-initializes most modulation / output layers, which would hide bugs) and float32
math on CPU. The port loads the reference ``state_dict()`` (unprefixed, i.e. exactly the official
Diffusers layout) with ``strict=True`` through its own name mapping.
"""
import copy
import importlib.machinery
import os
import re
import sys
import types
from functools import partial
from types import SimpleNamespace
import pytest
import torch
from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL, super_resolve
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
Kandinsky6SRVAEConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.loader.utils import (
get_param_names_mapping,
hf_to_custom_state_dict,
)
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
Kandinsky6SRTransformer3DModel,
)
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
PiflowScheduler,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
)
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
Kandinsky6SRLatentUpscalerBank,
)
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr import (
tiling,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
build_dit_spec,
build_sampling_spec,
effective_scheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
denoise_with_scheduler,
euler_start_timestep,
make_dit_fn,
module_dtype,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
plan_tiles,
stitch_tiles,
)
REFERENCE_ENV = "KANDINSKY_SR_REFERENCE_SRC"
_reference_dir = os.environ.get(REFERENCE_ENV)
if not _reference_dir or not os.path.isdir(
os.path.join(_reference_dir, "kandinsky_sr")
):
pytest.skip(
f"set {REFERENCE_ENV} to the directory containing the kandinsky_sr package",
allow_module_level=True,
)
# --------------------------------------------------------------------------- #
# Reference harness (test-only): stubs, tiny resolutions, tiny random components
# --------------------------------------------------------------------------- #
TINY_RESOLUTIONS = {512: [(64, 64), (64, 96), (96, 64)]}
TINY_KVAE_ENC = dict(TINY_KVAE, in_channels=3)
TINY_KVAE_DEC = dict(TINY_KVAE, out_ch=3)
TINY_SCALING_FACTOR = 0.5
TINY_DIT_CFG = dict(
in_visual_dim=4,
in_text_dim=8,
in_text_dim2=8,
time_dim=16,
out_visual_dim=4,
patch_size=(1, 2, 2),
model_dim=32,
ff_dim=64,
num_text_blocks=0,
num_visual_blocks=2,
axes_dims=(8, 4, 4),
visual_cond=False,
instruct_type="noise",
attention_params={"512": {"type": "flash"}},
use_text=False,
)
TINY_PIFLOW = dict(
nfe=2,
num_policy_substeps=8,
final_step_size_scale=0.5,
shift=5.0,
n_grid=3,
eps=1e-6,
)
TINY_SR_PARAMS = dict(
scale_factor={512: [1.0, 1.0, 1.0]},
visual_size=[512],
scheduler_scale=5.0,
lq_noise_scale=0.7,
lq_noise_type="ddpm",
lq_channel_noise_scale=0.0,
cap_noise_timestep=False,
fps=24,
)
def _varlen_sdpa(q, k, v, cu_q, cu_k):
outs = []
for i in range(len(cu_q) - 1):
qs, qe = int(cu_q[i]), int(cu_q[i + 1])
ks, ke = int(cu_k[i]), int(cu_k[i + 1])
qi = q[qs:qe].transpose(0, 1).unsqueeze(0)
ki = k[ks:ke].transpose(0, 1).unsqueeze(0)
vi = v[ks:ke].transpose(0, 1).unsqueeze(0)
out = torch.nn.functional.scaled_dot_product_attention(qi, ki, vi)
outs.append(out.squeeze(0).transpose(0, 1))
return torch.cat(outs, dim=0)
def _flash_attn_stub() -> types.ModuleType:
module = types.ModuleType("flash_attn")
def varlen(
q,
k,
v,
cu_seqlens_q,
cu_seqlens_k,
max_seqlen_q,
max_seqlen_k,
return_attn_probs=False,
**_,
):
out = _varlen_sdpa(q, k, v, cu_seqlens_q, cu_seqlens_k)
return (out, torch.zeros(1), None) if return_attn_probs else out
def qkvpacked(qkv, cu_seqlens, max_seqlen, return_attn_probs=False, **_):
q, k, v = qkv.unbind(dim=1)
out = _varlen_sdpa(q, k, v, cu_seqlens, cu_seqlens)
return (out, torch.zeros(1), None) if return_attn_probs else out
module.flash_attn_varlen_func = varlen
module.flash_attn_varlen_qkvpacked_func = qkvpacked
# transformers probes flash_attn with importlib.util.find_spec
module.__spec__ = importlib.machinery.ModuleSpec("flash_attn", loader=None)
return module
def _loguru_stub() -> types.ModuleType:
module = types.ModuleType("loguru")
class _Logger:
def __getattr__(self, name):
return lambda *args, **kwargs: self
module.logger = _Logger()
module.__spec__ = importlib.machinery.ModuleSpec("loguru", loader=None)
return module
def _install_reference() -> None:
os.environ.setdefault("TORCH_COMPILE_DISABLE", "1")
os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
if _reference_dir not in sys.path:
sys.path.insert(0, _reference_dir)
if "flash_attn" not in sys.modules:
sys.modules["flash_attn"] = _flash_attn_stub()
if "loguru" not in sys.modules:
sys.modules["loguru"] = _loguru_stub()
from kandinsky_sr import constants
constants.RESOLUTIONS.clear()
constants.RESOLUTIONS.update(TINY_RESOLUTIONS)
_install_reference()
from kandinsky_sr.core.components.model.dit import get_dit # noqa: E402
from kandinsky_sr.core.components.model.dx_dit import DXDiTWrapper # noqa: E402
from kandinsky_sr.core.components.video_kvae.cached_model import ( # noqa: E402
CachedCausalVAE,
)
from omegaconf import OmegaConf # noqa: E402
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29507",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
def randomize(module: torch.nn.Module, seed: int, std: float = 0.05) -> None:
generator = torch.Generator().manual_seed(seed)
with torch.no_grad():
for param in module.parameters():
param.copy_(torch.randn(param.shape, generator=generator) * std)
def build_reference_dit(*, piflow: dict | None, seed: int = 2, cfg: dict | None = None):
cfg = dict(cfg or TINY_DIT_CFG)
if piflow is not None:
dit = DXDiTWrapper(
cfg, out_visual_dim=cfg["out_visual_dim"], n_grid=piflow["n_grid"]
)
dit.piflow_params = dict(piflow)
else:
dit = get_dit(cfg)
dit.eval()
randomize(dit, seed)
return dit
def flat_transformer_config(*, cfg: dict, piflow: dict | None, overrides: dict) -> dict:
"""The flat legacy dict form of a transformer config (n_grid / piflow_* / sr_* fields)."""
flat = dict(cfg)
flat["patch_size"] = list(flat["patch_size"])
flat["axes_dims"] = list(flat["axes_dims"])
flat["n_grid"] = piflow["n_grid"] if piflow is not None else 1
if piflow is not None:
flat.update(
piflow_nfe=piflow["nfe"],
piflow_num_policy_substeps=piflow["num_policy_substeps"],
piflow_final_step_size_scale=piflow["final_step_size_scale"],
piflow_shift=piflow["shift"],
piflow_eps=piflow["eps"],
)
flat["attribute_overrides"] = dict(overrides)
flat.update(
sr_visual_size=[512],
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
sr_scheduler_scale=TINY_SR_PARAMS["scheduler_scale"],
sr_lq_noise_scale=TINY_SR_PARAMS["lq_noise_scale"],
sr_lq_noise_type=TINY_SR_PARAMS["lq_noise_type"],
sr_lq_channel_noise_scale=TINY_SR_PARAMS["lq_channel_noise_scale"],
sr_cap_noise_timestep=TINY_SR_PARAMS["cap_noise_timestep"],
sr_fps=TINY_SR_PARAMS["fps"],
)
return flat
def build_port_dit(
reference_dit, *, cfg: dict | None = None, piflow: dict | None, overrides: dict
) -> Kandinsky6SRTransformer3DModel:
"""Port DiT loaded with the reference weights (``model.`` prefix), strictly."""
flat = flat_transformer_config(
cfg=dict(cfg or TINY_DIT_CFG), piflow=piflow, overrides=overrides
)
dit_config = Kandinsky6SRDitConfig()
dit_config.update_model_arch(flat)
model = Kandinsky6SRTransformer3DModel(dit_config, flat)
official = dict(reference_dit.state_dict()) # unprefixed = the official layout
mapped, _ = hf_to_custom_state_dict(
official,
get_param_names_mapping(model.param_names_mapping),
valid_target_names=set(model.state_dict()),
)
model.load_state_dict(mapped, strict=True)
return model.eval()
# --------------------------------------------------------------------------- #
# DiT forward parity
# --------------------------------------------------------------------------- #
def _reference_forward(dit, x, time, *, scale_factor, motion_score=None):
"""Run the packed reference DiT on a batched ``[B, T, H, W, C]`` input."""
batch, frames, height, width, channels = x.shape
packed = x.reshape(batch * frames, height, width, channels)
visual_cu = frames * torch.arange(batch + 1, dtype=torch.int32)
text_cu = torch.zeros(batch + 1, dtype=torch.int32)
rope_pos = [
torch.cat([torch.arange(frames) for _ in range(batch)]),
torch.arange(height // dit.patch_size[1]),
torch.arange(width // dit.patch_size[2]),
]
out = dit(
packed,
torch.zeros(0, 1),
torch.zeros(batch, 1),
time,
visual_cu,
text_cu,
rope_pos,
torch.zeros(0, dtype=torch.long),
scale_factor=scale_factor,
motion_score=motion_score,
)
return out.reshape(batch, frames, *out.shape[1:]) if out.dim() == 4 else out
def _port_forward(model, x, time, *, scale_factor, motion_score=None):
batch, frames, height, width, _ = x.shape
rope_pos = [
torch.arange(frames),
torch.arange(height // model.patch_size[1]),
torch.arange(width // model.patch_size[2]),
]
return model(
x,
time,
rope_pos,
scale_factor=scale_factor,
motion_score=motion_score,
)
def _random_latent(
width_channels: int, *, batch=2, frames=3, height=4, width=6, seed=0
):
generator = torch.Generator().manual_seed(seed)
return torch.randn(
batch, frames, height, width, width_channels, generator=generator
)
@pytest.mark.parametrize("piflow", [TINY_PIFLOW, None], ids=["dx_head", "plain_head"])
def test_dit_forward_matches_reference(piflow):
"""Guards the block math, RoPE, modulation, time bias and the DX head layout.
The port is batched ``[B, T, H, W, C]`` while the reference packs samples with
``cu_seqlens``; different per-sample times make a wrong batching/broadcast visible.
"""
reference = build_reference_dit(piflow=piflow)
model = build_port_dit(
reference, piflow=piflow, overrides={"instruct_type": "noise"}
)
x = _random_latent(4)
time = torch.tensor([300.0, 850.0])
scale = (1.0, 2.0, 2.0)
with torch.no_grad():
expected = _reference_forward(reference, x, time, scale_factor=scale)
actual = _port_forward(model, x, time, scale_factor=scale)
n_grid = 1 if piflow is None else piflow["n_grid"]
if piflow is not None:
# reference DX wrapper already splits the grids: [B*T, n_grid, H, W, C]
expected = expected.movedim(1, -2).reshape(*actual.shape)
assert actual.shape[-1] == 4 * n_grid
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
def test_dit_forward_matches_reference_with_motion_score_and_wide_input():
"""Guards the optional motion embedding and the ``2 * in + 1`` conditioned input."""
cfg = dict(
TINY_DIT_CFG,
use_motion_score=True,
instruct_type="hybrid_anchor",
use_lq_noise_cond=True,
)
reference = build_reference_dit(piflow=TINY_PIFLOW, cfg=cfg)
model = build_port_dit(
reference,
cfg=cfg,
piflow=TINY_PIFLOW,
overrides={"instruct_type": "noise", "visual_cond": True},
)
reference.instruct_type = "noise"
reference.visual_cond = True
x = _random_latent(2 * 4 + 1)
time = torch.tensor([120.0, 999.0])
motion = torch.full((1,), 900.0)
with torch.no_grad():
expected = _reference_forward(
reference, x, time, scale_factor=(1.0, 1.0, 1.0), motion_score=motion
)
actual = _port_forward(
model, x, time, scale_factor=(1.0, 1.0, 1.0), motion_score=motion
)
without_motion = _port_forward(model, x, time, scale_factor=(1.0, 1.0, 1.0))
expected = expected.movedim(1, -2).reshape(*actual.shape)
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
assert not torch.allclose(actual, without_motion)
def test_state_dict_keys_equal_mapped_reference_keys():
"""Guards the module tree / key mapping: extra or missing keys would break real checkpoints."""
reference = build_reference_dit(piflow=TINY_PIFLOW)
model = build_port_dit(reference, piflow=TINY_PIFLOW, overrides={})
mapped, _ = hf_to_custom_state_dict(
dict(reference.state_dict()),
get_param_names_mapping(model.param_names_mapping),
)
assert set(mapped) == set(model.state_dict())
# --------------------------------------------------------------------------- #
# Sampler loops (pi-Flow / Euler) against the reference generate functions
# --------------------------------------------------------------------------- #
def _reference_loop_inputs(x):
batch, frames, height, width, _ = x.shape
visual_cu = frames * torch.arange(batch + 1, dtype=torch.int32)
text = {"text_embeds": torch.zeros(0, 1), "pooled_embed": torch.zeros(batch, 1)}
text_cu = torch.zeros(batch + 1, dtype=torch.int32)
rope_pos = [
torch.cat([torch.arange(frames) for _ in range(batch)]),
torch.arange(height // 2),
torch.arange(width // 2),
]
return visual_cu, text, text_cu, rope_pos, torch.zeros(0, dtype=torch.long)
def _port_dit_fn(model, x, scale_factor, use_motion_score=False):
return make_dit_fn(
model,
latent_frames_hw=tuple(x.shape[1:4]),
patch_size=model.patch_size,
scale_factor=scale_factor,
use_motion_score=use_motion_score,
)
def test_piflow_loop_matches_reference_piflow_generate():
"""Guards the segment schedule, DXPolicy grid layout and rollout on a batch of tiles.
The reference packs the tiles along time with one per-frame sigma; the port batches
``[B, T, H, W, C]``. Same weights and same start latent, fp32 on CPU.
The port no longer re-derives the pi-Flow segment schedule itself (``latents.piflow_schedule``
/ ``sampling.denoise_piflow`` are gone): it drives a real ``PiflowScheduler`` object through
``denoise_with_scheduler`` instead, which the analysis in commit d8d0e79f's port (and the
``PiflowScheduler._policy_step`` implementation itself) shows reproduces the exact same
per-segment ``(raw_src, seg, raw_dst, sigma_src)`` schedule as the removed pure function did.
"""
from kandinsky_sr.core.algo.piflow_sampler import piflow_generate
reference = build_reference_dit(piflow=TINY_PIFLOW)
model = build_port_dit(
reference, piflow=TINY_PIFLOW, overrides={"instruct_type": "noise"}
)
x = _random_latent(4, seed=5)
scale = (1.0, 2.0, 2.0)
visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x)
packed = x.reshape(-1, *x.shape[2:]).clone()
scheduler = PiflowScheduler(**TINY_PIFLOW)
scheduler.set_timesteps(TINY_PIFLOW["nfe"], device="cpu")
with torch.no_grad():
expected = piflow_generate(
packed,
reference,
text,
visual_cu,
text_cu,
rope_pos,
text_rope,
scale,
out_dim=4,
start_timestep=1.0,
device="cpu",
**TINY_PIFLOW,
)
actual = denoise_with_scheduler(
x.clone(),
_port_dit_fn(model, x, scale),
scheduler,
channels=4,
is_piflow=True,
)
torch.testing.assert_close(
actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5
)
@pytest.mark.parametrize(
"capped", [False, True], ids=["full_range", "capped_noise_start"]
)
def test_euler_loop_matches_reference_generate(capped):
"""Guards the warped timesteps, per-step deltas and (for a wide input) the in-place
update of the first channels only, incl. the ``cap_noise_timestep`` start time.
The reference's ``generate(..., 5, ...)`` counts timestep *grid points* (5 points -> 4
Euler steps); the port's own convention counts DiT *calls* directly, so the equivalent
port call is ``num_inference_steps=4``. Both grids are the same 5-point warped-linspace
array (``FlowMatchEulerDiscreteScheduler.set_timesteps`` applies the configured ``shift``
warp to the 4 unwarped sigmas this test passes in, then appends the terminal 0 itself --
algebraically the same 5th point the reference's own ``warp(0) == 0`` produces), so the 4
step-to-step deltas ``denoise_with_scheduler`` consumes via ``scheduler.step`` are bit-for-
bit the 4 deltas the old, removed ``denoise_euler`` consumed from that same array.
"""
from kandinsky_sr.core.algo.utils import generate
cfg = dict(TINY_DIT_CFG, use_motion_score=True, instruct_type="hybrid_anchor")
reference = build_reference_dit(piflow=None, cfg=cfg)
model = build_port_dit(
reference,
cfg=cfg,
piflow=None,
overrides={"instruct_type": "noise", "visual_cond": True},
)
reference.instruct_type, reference.visual_cond = "noise", True
x = _random_latent(2 * 4 + 1, seed=9)
scale = (1.0, 1.5, 1.5)
start = euler_start_timestep(
cap_noise_timestep=capped, lq_noise_scale=0.7, instruct_type="noise"
)
assert start == (0.7 if capped else 1.0)
visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x)
packed = x.reshape(-1, *x.shape[2:]).clone()
num_inference_steps = 4 # the reference's 5 grid points == 4 actual Euler steps
scheduler = FlowMatchEulerDiscreteScheduler(shift=5.0)
sigmas = torch.linspace(start, 0.0, num_inference_steps + 1)[:-1].tolist()
scheduler.set_timesteps(sigmas=sigmas, device="cpu")
with torch.no_grad():
expected = generate(
packed,
reference,
"cpu",
5,
text,
text,
visual_cu,
text_cu,
text_cu,
rope_pos,
text_rope,
text_rope,
scale,
1.0,
5.0,
start_timestep=start,
)
actual = denoise_with_scheduler(
x.clone(),
_port_dit_fn(model, x, scale, use_motion_score=True),
scheduler,
channels=4,
is_piflow=False,
)
torch.testing.assert_close(
actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5
)
# --------------------------------------------------------------------------- #
# Tiling / stitching against the reference helpers
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
"frame_hw,scale", [((64, 96), 2), ((64, 64), 4), ((96, 64), 2), ((80, 144), 2)]
)
def test_tile_plan_and_stitch_match_reference(frame_hw, scale):
"""Guards the tile grid (aspect -> base resolution -> even grid) and that the
frame-chunked stitch equals one reference stitch over the whole clip (T=19, not a
multiple of the chunk size)."""
from kandinsky_sr.core.algo.tiling_utils import stitch_tiles_hanning
from kandinsky_sr.pipeline.sr_pipeline import _tile_geometry
height, width = frame_hw
plan = plan_tiles(
frame_hw=frame_hw,
visual_size=512,
tiling_scale=scale,
tile_min_overlap=0.2,
resolutions=TINY_RESOLUTIONS,
)
base, tile_hw, grid = _tile_geometry(
height, width, 512, scale, 0.25, 16, "even", 0.2
)
assert (plan.base_hw, plan.tile_hw) == (base, tile_hw)
assert tuple(plan.pixel_grid) == tuple(grid)
generator = torch.Generator().manual_seed(3)
tiles = [
torch.randint(0, 256, (3, 19, *base), dtype=torch.uint8, generator=generator)
for _ in range(grid.total_tiles)
]
expected = (
stitch_tiles_hanning(
[t.float() for t in tiles], grid, height, width, scale=scale
)
.clamp(0, 255)
.to(torch.uint8)
)
assert torch.equal(stitch_tiles(tiles, plan), expected)
# --------------------------------------------------------------------------- #
# End to end: pure orchestration vs kandinsky_sr.pipeline.stages.run_tiled_sr*
# --------------------------------------------------------------------------- #
def build_reference_lu_bank(target_scales=("2x", "4x"), seed=0):
"""Reference LU bank with random weights and its bundle-style ``models`` config."""
import pydantic
from kandinsky_sr.core.algo.latent_upscaler import LatentUpscalerBank
from kandinsky_sr.core.components.latent_upscaler.config import ModelConfig
from kandinsky_sr.core.components.latent_upscaler.model.factory import (
build_upsampler,
)
models = [
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
for scale in target_scales
]
adapter = pydantic.TypeAdapter(ModelConfig)
bank = LatentUpscalerBank()
for index, spec in enumerate(models):
upsampler = build_upsampler(adapter.validate_python(spec["model"])).eval()
randomize(upsampler, seed + 10 + index)
upsampler.target_scale = spec["target_scale"]
upsampler.scaling_factor = TINY_SCALING_FACTOR
bank[spec["target_scale"]] = upsampler
return bank, models
def build_stacks(*, piflow, use_lu):
"""(reference SRComponents, port dict) sharing identical random weights."""
from kandinsky_sr.pipeline.sr_pipeline import SRComponents, SRParams
torch.manual_seed(0)
ref_vae = CachedCausalVAE(
encoder_conf=OmegaConf.create(dict(TINY_KVAE_ENC)),
decoder_conf=OmegaConf.create(dict(TINY_KVAE_DEC)),
).eval()
randomize(ref_vae, 1, std=0.1)
ref_vae.config = SimpleNamespace(scaling_factor=TINY_SCALING_FACTOR)
ref_dit = build_reference_dit(piflow=piflow, seed=2)
ref_bank, models = build_reference_lu_bank() if use_lu else (None, None)
reference = SRComponents(
dit=ref_dit,
vae=ref_vae,
latent_upscaler=ref_bank,
sr_params=SRParams(**TINY_SR_PARAMS),
spatial_factor=16,
)
vae_config = Kandinsky6SRVAEConfig()
vae_config.update_model_arch(
dict(
vae_type="video-kvae",
encoder_config=dict(TINY_KVAE_ENC),
decoder_config=dict(TINY_KVAE_DEC),
scaling_factor=TINY_SCALING_FACTOR,
spatial_factor=16,
temporal_factor=4,
)
)
vae = Kandinsky6SRVAE(vae_config)
# unprefixed reference keys = the official layout, mapped by the wrapper's load hook
vae.load_state_dict(dict(ref_vae.state_dict()), strict=True)
port = {
"vae": vae.eval(),
"dit": build_port_dit(
ref_dit, piflow=piflow, overrides={"instruct_type": "noise"}
),
"bank": None,
}
if use_lu:
bank = Kandinsky6SRLatentUpscalerBank(models, TINY_SCALING_FACTOR)
converted = {}
scale_indices = {"2x": 0, "4x": 1}
for key, value in ref_bank.state_dict().items():
scale, suffix = key.split(".", 1)
suffix = suffix.replace("private_", "")
suffix = re.sub(r"(output_proj\.)0\.", r"\1norm.", suffix)
suffix = re.sub(r"(output_proj\.)2\.", r"\1conv.", suffix)
converted[f"_models.{scale_indices[scale]}.{suffix}"] = value
bank.load_state_dict(
converted,
strict=True,
)
port["bank"] = bank.eval()
return reference, port
def run_reference(video, reference, *, scale, seed, tiles_batch_size, num_steps):
from kandinsky_sr.core.algo.latent_upscaler import latent_upscaler_for_scale
from kandinsky_sr.pipeline.sr_pipeline import (
RunConfig,
encode_lq_video_to_lr_latent,
)
from kandinsky_sr.pipeline.stages import run_tiled_sr, run_tiled_sr_from_pixels
from kandinsky_sr.pipeline.upscale_utils import (
pre_upscale_video,
resolve_scale_request,
)
tiling_scale, pre = resolve_scale_request(float(scale))
if pre != 1.0:
video = pre_upscale_video(video, pre, 16)
run_config = RunConfig(
device="cpu",
num_steps=num_steps,
seed=seed,
overlap=0.25,
tiles_batch_size=tiles_batch_size,
resolution_scale=tiling_scale,
)
if latent_upscaler_for_scale(reference.latent_upscaler, tiling_scale) is not None:
lr_latent = encode_lq_video_to_lr_latent(video, reference.vae, "cpu")
return run_tiled_sr(lr_latent, reference, run_config)
return run_tiled_sr_from_pixels(video, reference, run_config)
def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps):
"""The stage logic (scale request, pre-upscale, specs) around the pure function.
``num_steps`` here is the reference's convention (timestep grid points); for a
flow-matching (non pi-Flow) bundle the port's own ``num_inference_steps`` is
``num_steps - 1`` (DiT calls), so it is translated once, here, at the one remaining
boundary between the two conventions. A pi-Flow bundle ignores it either way (its
scheduler's own ``nfe`` always wins), so no translation is needed for that case.
"""
tiling_scale, pre = tiling.resolve_scale_request(float(scale))
if pre != 1.0:
video = tiling.pre_upscale_video(video, pre, 16)
arch = port["dit"].config
port_num_steps = num_steps if arch.is_piflow else num_steps - 1
spec = build_sampling_spec(
arch=arch,
tiling_scale=tiling_scale,
seed=seed,
num_steps=port_num_steps,
tiles_batch_size=tiles_batch_size,
tile_min_overlap=0.2,
)
bank = port["bank"]
use_lu = bank is not None and bank.for_scale(tiling_scale) is not None
return super_resolve(
video,
vae=port["vae"],
scaling_factor=port["vae"].scaling_factor,
dit=port["dit"],
dit_spec=build_dit_spec(port["dit"]),
spec=spec,
scheduler=effective_scheduler(spec, None),
device="cpu",
upscale_fn=partial(bank.upscale, scale=tiling_scale) if use_lu else None,
lu_dtype=module_dtype(bank) if use_lu else None,
resolutions=TINY_RESOLUTIONS,
)
def _random_video(frames, height, width, seed=0):
generator = torch.Generator().manual_seed(seed)
return torch.randint(
0, 256, (frames, 3, height, width), dtype=torch.uint8, generator=generator
)
E2E_CASES = [
# scale, (H, W), tiles_batch_size
(2, (64, 96), 1),
(2, (64, 96), 3),
(4, (64, 64), 2),
(2.25, (64, 96), 1),
]
def _assert_uint8_close(actual, expected, *, max_diff: int = 1):
assert actual.shape == expected.shape and actual.dtype == expected.dtype
diff = (actual.int() - expected.int()).abs()
exact = (diff == 0).float().mean().item()
assert diff.max().item() <= max_diff and exact > 0.999, (diff.max().item(), exact)
@pytest.mark.parametrize("use_lu", [False, True], ids=["pixel_path", "lu_path"])
@pytest.mark.parametrize("scale,hw,tiles_batch_size", E2E_CASES)
def test_tiled_piflow_super_resolution_matches_reference(
use_lu, scale, hw, tiles_batch_size
):
"""Guards the whole orchestration: scale request + pre-upscale, tile geometry, LU or
tile encode, chunk seeding (``seed + first tile index``), pi-Flow loop, decode with
truncation to uint8 and Hann stitching, against ``run_tiled_sr`` /
``run_tiled_sr_from_pixels`` with identical random weights.
``max_diff=2`` (not the default 1): the reference's ``piflow_generate`` is the
training-only inline schedule, which clamps the *final* segment's ``raw_dst`` to
0.0 (dead code on every real checkpoint - see GAP 2 / ``latents.piflow_schedule``).
The port now deliberately matches the real ``PiflowScheduler._policy_step``
convention (``raw_dst=eps``) instead, which real checkpoints always run through.
That is a ~1e-6 raw-timestep difference in the last segment only, so it shows up
here as at most a 1-LSB-wider uint8 rounding tie on top of the pre-existing
rounding-boundary noise (still >99.99% exact pixels), not as a structural bug.
"""
reference, port = build_stacks(piflow=TINY_PIFLOW, use_lu=use_lu)
video = _random_video(9, *hw)
kwargs = dict(scale=scale, seed=42, tiles_batch_size=tiles_batch_size, num_steps=5)
expected = run_reference(video, reference, **kwargs)
actual = run_port(video, port, **kwargs)
_assert_uint8_close(actual, expected, max_diff=2)
@pytest.mark.parametrize("use_lu", [False, True], ids=["pixel_path", "lu_path"])
def test_tiled_euler_super_resolution_matches_reference(use_lu):
"""Same as above for a plain (non pi-Flow) DiT: the Euler loop with ``num_steps``."""
reference, port = build_stacks(piflow=None, use_lu=use_lu)
video = _random_video(9, 64, 96, seed=1)
kwargs = dict(scale=2, seed=7, tiles_batch_size=2, num_steps=4)
expected = run_reference(video, reference, **kwargs)
actual = run_port(video, port, **kwargs)
_assert_uint8_close(actual, expected)
def test_seed_and_chunking_change_the_result_like_the_reference():
"""The initial noise of chunk k is seeded with ``seed + chunk start``: another seed or
another chunk size must change the output (a port that ignored them would not)."""
_, port = build_stacks(piflow=TINY_PIFLOW, use_lu=False)
video = _random_video(9, 64, 96, seed=2)
base = run_port(video, port, scale=2, seed=42, tiles_batch_size=1, num_steps=5)
assert not torch.equal(
base, run_port(video, port, scale=2, seed=43, tiles_batch_size=1, num_steps=5)
)
assert not torch.equal(
base, run_port(video, port, scale=2, seed=42, tiles_batch_size=4, num_steps=5)
)
@@ -0,0 +1,253 @@
# SPDX-License-Identifier: Apache-2.0
"""Validate scheduler selection, DiT call counts and incompatible head rejection."""
import os
from typing import NamedTuple
import pytest
from kandinsky6_sr_release_configs import (
DISTILLED_SCHEDULER_CONFIG,
DISTILLED_TRANSFORMER_CONFIG,
FLOW_SCHEDULER_CONFIG,
FLOW_TRANSFORMER_CONFIG,
)
from kandinsky6_sr_tiny_components import (
TINY_DIT,
build_components,
make_request,
make_stage,
random_video,
)
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
PiflowScheduler,
)
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
FlowMatchEulerDiscreteScheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
Kandinsky6SRDenoisingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
Kandinsky6SRLatentPrepStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_VIDEO_KEY,
build_sampling_spec,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
RESOLUTIONS,
)
LATENT_WIDTH = TINY_DIT["in_visual_dim"]
N_GRID = (
3 # the distilled bundle's own grid width, independent of TINY_DIT's legacy fields
)
NUM_TILES = 9 # the 64x96 clip below, cut into 32x48 tiles
def official_dit(head_width):
"""Tiny transformer config in the official layout: ``out_visual_dim`` is the TOTAL
head width, and ``n_grid`` / ``piflow_*`` live in the scheduler config."""
arch = {
key: value
for key, value in TINY_DIT.items()
if key != "out_visual_dim" and not key.startswith(("n_grid", "piflow_", "sr_"))
}
return dict(
arch,
out_visual_dim=head_width,
sr_params=dict(scale_factor={"512": [1.0, 1.0, 1.0]}, visual_size=[512]),
)
FLOW_DIT = official_dit(LATENT_WIDTH) # one latent per token: flow-matching head
DX_DIT = official_dit(N_GRID * LATENT_WIDTH) # N_GRID latents per token: pi-Flow head
def flow_scheduler():
return FlowMatchEulerDiscreteScheduler(shift=5.0)
def pi_flow_scheduler(*, nfe=2, n_grid=N_GRID):
return PiflowScheduler(
nfe=nfe,
n_grid=n_grid,
shift=5.0,
eps=1e-6,
final_step_size_scale=0.5,
num_policy_substeps=8,
)
def release_scheduler(scheduler_cls, config):
return scheduler_cls(**{k: v for k, v in config.items() if not k.startswith("_")})
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29510",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
@pytest.fixture(autouse=True)
def tiny_resolutions(monkeypatch):
monkeypatch.setitem(RESOLUTIONS, 512, [(64, 64), (64, 96), (96, 64)])
class SamplingRun(NamedTuple):
tiles: int
dit_calls: int
progress_totals: list
def run_denoising(dit_arch, scheduler, *, num_steps=5):
"""Run latent-prep + denoising (the LU-less pixel path) on the 9-tile clip and count
the DiT calls next to the totals the denoising progress bar was opened with."""
vae, dit, _, server_args = build_components((), dit_arch)
dit_calls = []
dit.register_forward_hook(lambda module, args, output: dit_calls.append(1))
latent_prep = make_stage(
Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, scheduler
)
denoise = make_stage(Kandinsky6SRDenoisingStage, server_args, dit, scheduler)
progress_totals = []
progress_bar = denoise.progress_bar
def spy_on_progress_bar(**kwargs):
progress_totals.append(kwargs["total"])
return progress_bar(**kwargs)
denoise.progress_bar = spy_on_progress_bar
batch = make_request(random_video(9, 64, 96), num_steps=num_steps)
batch = latent_prep.forward(batch, server_args)
tiles = len(batch.extra["kandinsky6_sr_chunks"])
denoise.forward(batch, server_args)
return SamplingRun(tiles, len(dit_calls), progress_totals)
def sampling_spec(dit_arch, scheduler):
config = Kandinsky6SRDitConfig()
config.update_model_arch(dict(dit_arch))
return build_sampling_spec(
arch=config.arch_config,
tiling_scale=2,
seed=42,
num_steps=5,
tiles_batch_size=1,
tile_min_overlap=0.2,
scheduler=scheduler,
)
# --------------------------------------------------------------------------- #
# DiT calls per tile
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize("nfe,num_steps", [(2, 3), (2, 5), (2, 9), (3, 5)])
def test_pi_flow_scheduler_makes_nfe_calls_per_tile_whatever_num_inference_steps(
nfe, num_steps
):
run = run_denoising(DX_DIT, pi_flow_scheduler(nfe=nfe), num_steps=num_steps)
assert run.tiles == NUM_TILES
assert run.dit_calls == NUM_TILES * nfe
assert run.progress_totals == [NUM_TILES * nfe]
@pytest.mark.parametrize("num_steps,calls_per_tile", [(3, 3), (5, 5), (9, 9)])
def test_flow_euler_scheduler_makes_num_inference_steps_calls_per_tile(
num_steps, calls_per_tile
):
run = run_denoising(FLOW_DIT, flow_scheduler(), num_steps=num_steps)
assert run.tiles == NUM_TILES
assert run.dit_calls == NUM_TILES * calls_per_tile
assert run.progress_totals == [NUM_TILES * calls_per_tile]
@pytest.mark.parametrize("num_steps", [3, 9])
def test_legacy_config_with_its_own_pi_flow_fields_keeps_its_sampler(num_steps):
"""A flat DiT config carries n_grid / piflow_* and needs no scheduler component: the
denoising stage synthesizes a matching PiflowScheduler from those arch fields
(run_spec.effective_scheduler) instead of leaving nothing to step."""
run = run_denoising(TINY_DIT, None, num_steps=num_steps)
assert run.dit_calls == NUM_TILES * TINY_DIT["piflow_nfe"]
assert run.progress_totals == [run.dit_calls]
# --------------------------------------------------------------------------- #
# Samplers that do not fit the DiT head
# --------------------------------------------------------------------------- #
@pytest.mark.parametrize(
"scheduler",
[flow_scheduler(), PiflowScheduler(n_grid=N_GRID, nfe=None), None],
ids=["flow_euler_scheduler", "pi_flow_scheduler_without_nfe", "no_scheduler"],
)
def test_dx_head_never_runs_flow_euler(scheduler):
with pytest.raises(ValueError, match="flow-Euler"):
sampling_spec(DX_DIT, scheduler)
@pytest.mark.parametrize(
"dit_arch,n_grid,needed",
[
(FLOW_DIT, N_GRID, "3 x 4 = 12"),
(DX_DIT, 2, "2 x 4 = 8"),
(DX_DIT, 4, "4 x 4 = 16"),
],
ids=["single_head", "too_few_grids", "too_many_grids"],
)
def test_pi_flow_needs_a_head_of_n_grid_latents(dit_arch, n_grid, needed):
with pytest.raises(ValueError, match=f"needs a {needed} channel DX head"):
sampling_spec(dit_arch, pi_flow_scheduler(n_grid=n_grid))
@pytest.mark.parametrize("nfe", [0, -1])
def test_pi_flow_needs_at_least_one_call_per_tile(nfe):
with pytest.raises(ValueError, match="nfe >= 1"):
sampling_spec(DX_DIT, pi_flow_scheduler(nfe=nfe))
with pytest.raises(ValueError, match="nfe >= 1"):
sampling_spec(dict(TINY_DIT, piflow_nfe=nfe), None)
def test_release_transformer_and_scheduler_of_different_repos_are_rejected():
"""The real configs of the two official repos: their transformers differ only in
the head width, so one repo's transformer with the other's scheduler must fail."""
flow = release_scheduler(FlowMatchEulerDiscreteScheduler, FLOW_SCHEDULER_CONFIG)
distilled = release_scheduler(PiflowScheduler, DISTILLED_SCHEDULER_CONFIG)
assert sampling_spec(FLOW_TRANSFORMER_CONFIG, flow).steps_per_chunk == 5
assert sampling_spec(DISTILLED_TRANSFORMER_CONFIG, distilled).steps_per_chunk == 2
with pytest.raises(ValueError, match="640 channels wide.*flow-Euler"):
sampling_spec(DISTILLED_TRANSFORMER_CONFIG, flow)
with pytest.raises(ValueError, match="needs a 10 x 64 = 640 channel DX head"):
sampling_spec(FLOW_TRANSFORMER_CONFIG, distilled)
def test_a_mismatched_bundle_fails_before_any_tile_is_processed():
vae, dit, _, server_args = build_components((), DX_DIT)
dit_calls = []
dit.register_forward_hook(lambda module, args, output: dit_calls.append(1))
latent_prep = make_stage(
Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, flow_scheduler()
)
batch = make_request(random_video(9, 64, 96))
with pytest.raises(ValueError, match="flow-Euler"):
latent_prep.forward(batch, server_args)
assert dit_calls == []
assert SR_VIDEO_KEY in batch.extra # the clip was never cut into tiles
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
"""Verify SR token sharding, TP weight loading, and repeated distributed forwards."""
import os
from pathlib import Path
from types import SimpleNamespace
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
Kandinsky6SRDitConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
Kandinsky6SRPipelineConfig,
)
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
)
from sglang.multimodal_gen.runtime.layers.attention.selector import (
global_force_attn_backend,
)
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
Kandinsky6SRTransformer3DModel,
)
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
from sglang.multimodal_gen.runtime.server_args import set_global_server_args
from sglang.multimodal_gen.runtime.utils.precision import set_mixed_precision_policy
from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import TINY_DIT
from sglang.srt.utils.network import get_free_port_below_ephemeral
def _forward(rank, world_size, tp_size, ring, port, output_dir):
os.environ.update(
MASTER_ADDR="127.0.0.1",
MASTER_PORT=str(port),
RANK=str(rank),
LOCAL_RANK=str(rank),
WORLD_SIZE=str(world_size),
)
torch.cuda.set_device(rank)
set_mixed_precision_policy(torch.bfloat16, torch.float32)
global_force_attn_backend(AttentionBackendEnum.FA)
set_global_server_args(
SimpleNamespace(
attention_backend="fa",
attention_backend_config=None,
kv_gather_degree=1,
sp_split_auto=False,
pipeline_config=Kandinsky6SRPipelineConfig(),
)
)
sp_size = world_size // tp_size
maybe_init_distributed_environment_and_model_parallel(
tp_size=tp_size,
sp_size=sp_size,
ulysses_degree=1 if ring else sp_size,
ring_degree=sp_size if ring else 1,
)
config = Kandinsky6SRDitConfig()
config.update_model_arch(
TINY_DIT | dict(model_dim=512, time_dim=32, ff_dim=1024, axes_dims=[32, 48, 48])
)
model = Kandinsky6SRTransformer3DModel(config, {}).eval()
generator = torch.Generator().manual_seed(7)
directory = Path(output_dir)
with torch.no_grad():
if world_size == 1:
for parameter in model.parameters():
parameter.copy_(
torch.randn(parameter.shape, generator=generator) * 0.02
)
torch.save(model.state_dict(), directory / "weights.pt")
else:
weights = torch.load(directory / "weights.pt", weights_only=True)
for name, parameter in model.named_parameters():
loader = parameter.__dict__.get("weight_loader")
if loader is None:
parameter.copy_(weights[name])
else:
loader(parameter, weights[name])
model.to(device=rank, dtype=torch.bfloat16)
assert model.visual_transformer_blocks[0].self_attention.local_num_heads == (
4 // tp_size
)
observed = []
def record_shard(module, args):
visual, _, rope, _ = args
observed.append((visual.shape[0], visual.shape[1], rope.shape[1]))
handle = model.visual_transformer_blocks[0].register_forward_pre_hook(record_shard)
first = None
for batch_size, frames in ((1, 2), (2, 3), (1, 2)):
generator.manual_seed(frames)
inputs = torch.randn(batch_size, frames, 6, 6, 4, generator=generator).to(
device=rank, dtype=torch.bfloat16
)
with (
torch.no_grad(),
set_forward_context(current_timestep=0, attn_metadata=None),
):
actual = model(
inputs,
torch.full((batch_size,), 500.0, device=rank),
[torch.arange(n, device=rank) for n in (frames, 3, 3)],
)
local_len = (frames * 3 * 3 + sp_size - 1) // sp_size
assert observed[-1] == (batch_size, local_len, local_len)
assert actual.shape == (batch_size, frames, 6, 6, 12)
assert torch.isfinite(actual).all()
if frames == 2:
if first is None:
first = actual.clone()
else:
torch.testing.assert_close(actual, first, rtol=0, atol=0)
path = directory / f"{batch_size}-{frames}.pt"
if world_size == 1:
torch.save(actual.cpu(), path)
else:
expected = torch.load(path, weights_only=True, map_location=f"cuda:{rank}")
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.002)
handle.remove()
dist.destroy_process_group()
def _launch(world_size, tp_size, ring, output_dir):
mp.spawn(
_forward,
args=(world_size, tp_size, ring, get_free_port_below_ephemeral(), output_dir),
nprocs=world_size,
)
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="requires two CUDA GPUs")
def test_sr_two_gpus(tmp_path):
_launch(1, 1, False, str(tmp_path))
_launch(2, 1, False, str(tmp_path))
_launch(2, 1, True, str(tmp_path))
_launch(2, 2, False, str(tmp_path))
@pytest.mark.skipif(torch.cuda.device_count() < 4, reason="requires four CUDA GPUs")
def test_sr_four_gpus(tmp_path):
_launch(1, 1, False, str(tmp_path))
for tp_size in (1, 2, 4):
_launch(4, tp_size, False, str(tmp_path))
_launch(4, 2, True, str(tmp_path))
@@ -0,0 +1,353 @@
# SPDX-License-Identifier: Apache-2.0
"""Native SR stage-chain parity, component residency and video/audio hand-offs."""
import copy
import os
from functools import partial
from unittest.mock import MagicMock
import pytest
import torch
from kandinsky6_sr_tiny_components import (
TINY_LU_MODEL,
build_components,
make_request,
make_stage,
random_video,
super_resolve,
)
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
maybe_init_distributed_environment_and_model_parallel,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
Kandinsky6SRLatentUpscalerBank,
)
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import (
Kandinsky6SRPipeline,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
Kandinsky6SRDecodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
Kandinsky6SRDenoisingStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import (
Kandinsky6SREncodeStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
WARMUP_CLIP_FRAMES,
WARMUP_CLIP_HW,
Kandinsky6SRInputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
Kandinsky6SRLatentPrepStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import (
Kandinsky6SROutputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_LR_LATENT_KEY,
SR_PLAN_KEY,
SR_REQUESTED_HW_KEY,
SR_TILES_KEY,
SR_VIDEO_KEY,
build_dit_spec,
build_sampling_spec,
effective_scheduler,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
module_dtype,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
TilePlan,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
RESOLUTIONS,
)
@pytest.fixture(scope="module", autouse=True)
def single_process_model_parallel():
if not model_parallel_is_initialized():
for key, value in dict(
MASTER_ADDR="127.0.0.1",
MASTER_PORT="29509",
RANK="0",
LOCAL_RANK="0",
WORLD_SIZE="1",
).items():
os.environ.setdefault(key, value)
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
@pytest.fixture(autouse=True)
def tiny_resolutions(monkeypatch):
monkeypatch.setitem(RESOLUTIONS, 512, [(64, 64), (64, 96), (96, 64)])
@pytest.mark.parametrize(
"bank_scales",
[("2x", "4x"), (), ("4x",)],
ids=["lu_path", "pixel_path", "pixel_fallback"],
)
def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
"""encode -> latent-prep -> denoise -> decode -> output must give what
``super_resolve`` gives on the same clip, for the LU path, the pixel path, and a bank
that has no entry for the requested scale (which must fall back to the pixel path, not
fail). The 64x128 clip holds 12 overlapping 32x48 tiles, denoised in chunks of 5, 5 and
2."""
vae, dit, bank, server_args = build_components(bank_scales)
video = random_video(9, 64, 128)
batch = make_request(video, tiles_batch_size=5)
pipeline = object.__new__(Kandinsky6SRPipeline)
pipeline.modules = dict(vae=vae, transformer=dit, scheduler=None)
if bank is not None:
pipeline.modules["latent_upscaler"] = bank
pipeline._stages, pipeline._stage_name_mapping = [], {}
pipeline._disagg_role = RoleType.MONOLITHIC
pipeline.create_pipeline_stages(server_args)
for stage in pipeline.stages:
stage.server_args = server_args
input_stage, encode, latent_prep, denoise, decode, output = pipeline.stages
assert [type(stage) for stage in pipeline.stages] == [
Kandinsky6SRInputStage,
Kandinsky6SREncodeStage,
Kandinsky6SRLatentPrepStage,
Kandinsky6SRDenoisingStage,
Kandinsky6SRDecodeStage,
Kandinsky6SROutputStage,
]
assert [
(u.component_name, u.start_at_stage_entry)
for u in encode.component_uses(server_args)
] == [("vae", False)]
expected_phases = [("vae", "encode_tiles")]
if bank is not None:
expected_phases.insert(0, ("latent_upscaler", "upscale_tiles"))
assert [
(u.component_name, u.phase) for u in latent_prep.component_uses(server_args)
] == expected_phases
(dit_use,) = denoise.component_uses(server_args)
assert (dit_use.component_name, dit_use.phase) == ("transformer", "denoise_tiles")
assert dit_use.memory_intensive and dit_use.target_dtype == torch.bfloat16
assert [
(u.component_name, u.phase) for u in decode.component_uses(server_args)
] == [("vae", "decode_tiles")]
batch = encode.forward(batch, server_args)
assert (SR_LR_LATENT_KEY in batch.extra) == ("2x" in bank_scales)
batch = latent_prep.forward(batch, server_args)
assert SR_LR_LATENT_KEY not in batch.extra and SR_VIDEO_KEY not in batch.extra
assert len(batch.extra["kandinsky6_sr_chunks"]) == 3 # 12 tiles / 5 per chunk
assert isinstance(batch.extra[SR_PLAN_KEY], TilePlan)
batch = denoise.forward(batch, server_args)
batch = decode.forward(batch, server_args)
assert len(batch.extra[SR_TILES_KEY]) == 12
result = output.forward(batch, server_args)
arch = dit.config
use_lu = "2x" in bank_scales
spec = build_sampling_spec(
arch=arch,
tiling_scale=2,
seed=42,
num_steps=5,
tiles_batch_size=5,
tile_min_overlap=0.2,
)
expected = super_resolve(
video,
vae=vae,
scaling_factor=vae.scaling_factor,
dit=dit,
dit_spec=build_dit_spec(dit),
spec=spec,
scheduler=effective_scheduler(spec, None),
device=next(dit.parameters()).device,
upscale_fn=partial(bank.upscale, scale=2) if use_lu else None,
lu_dtype=module_dtype(bank) if use_lu else None,
)
assert isinstance(result, OutputBatch)
assert result.output.dtype == torch.float16
assert result.output.shape == (1, 3, 9, 128, 256)
restored = (result.output[0].float() * 255).to(torch.uint8)
assert torch.equal(restored, expected)
assert (batch.height, batch.width) == (128, 256)
assert SR_TILES_KEY not in batch.extra and SR_PLAN_KEY not in batch.extra
@pytest.mark.parametrize(
"role", [RoleType.ENCODER, RoleType.DENOISER, RoleType.DECODER]
)
def test_sr_rejects_disaggregation(role):
pipeline = object.__new__(Kandinsky6SRPipeline)
with pytest.raises(ValueError, match="monolithic"):
pipeline.validate_disagg_role(role)
pipeline.validate_disagg_role(RoleType.MONOLITHIC)
def test_encode_stage_leaves_the_vae_alone_on_the_pixel_path():
"""No LU entry for the scale -> the whole-video encode is skipped, and the VAE (which
a CPU-offloaded run would otherwise move to the GPU for nothing) is never touched.
"""
vae = MagicMock(name="vae")
bank = Kandinsky6SRLatentUpscalerBank(
[{"target_scale": "4x", "model": copy.deepcopy(TINY_LU_MODEL)}],
0.5,
scales=(4,),
)
_, _, _, server_args = build_components(())
video = random_video(9, 64, 64)
batch = make_request(video)
stage = make_stage(Kandinsky6SREncodeStage, server_args, vae, bank)
assert stage.forward(batch, server_args) is batch
assert torch.equal(batch.extra[SR_VIDEO_KEY], video)
assert SR_LR_LATENT_KEY not in batch.extra
assert vae.method_calls == [] and not vae.called
def test_input_stage_runs_on_a_synthetic_clip_for_warmup_requests():
"""Warmup requests carry no video (synthetic warmup) or a real one that must not be
decoded twice (request warmup): both run the real path on a small synthetic clip,
without audio, and without needing a prompt."""
params = Kandinsky6SRSamplingParams(video_path="missing_on_purpose.mp4")
batch = Req(sampling_params=params)
batch.is_warmup = True
Kandinsky6SRInputStage().forward(batch, MagicMock())
video = batch.extra[SR_VIDEO_KEY]
height, width = WARMUP_CLIP_HW
# default scale 2.25 = x1.125 pre-upscale (aligned to 16) then tiling scale 2
assert video.shape[0] == WARMUP_CLIP_FRAMES and video.dtype == torch.uint8
assert video.shape[2:] == (288, 432)
# 288x432 is already a multiple of 16, so alignment padding is a no-op here, and the
# requested size the output stage will crop back to equals the padded (encoded) size.
assert batch.extra[SR_REQUESTED_HW_KEY] == (288, 432)
assert batch.extra["kandinsky6_sr_tiling_scale"] == 2
assert (batch.fps, batch.num_frames) == (24, WARMUP_CLIP_FRAMES)
assert (batch.height, batch.width) == (288 * 2, 432 * 2)
assert batch.audio is None and height == 256
def test_input_stage_pads_a_misaligned_source_to_the_spatial_factor():
"""A source whose pre-upscaled size is not already a multiple of 16 (every integer
scale, which runs no pixel resize at all) must still be padded before whole-video KVAE
encoding, and the *requested* (pre-pad) size recorded for the output stage to crop back
to -- the fix for "whole-video KVAE encoding fails at the fourth PixelUnshuffle" for
scales other than 2.25."""
params = Kandinsky6SRSamplingParams(
video_path="missing_on_purpose.mp4", sr_resolution_scale=2
)
batch = Req(sampling_params=params)
batch.is_warmup = True
Kandinsky6SRInputStage().forward(batch, MagicMock())
# WARMUP_CLIP_HW is already 16-aligned, so exercise a genuinely misaligned size by
# checking the padding helper directly against the same video the stage produced, cut
# down to a misaligned size.
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
pad_to_spatial_factor,
)
misaligned = batch.extra[SR_VIDEO_KEY][:, :, :100, :70]
padded, requested_hw = pad_to_spatial_factor(misaligned, 16)
assert requested_hw == (100, 70)
assert padded.shape[-2] % 16 == 0 and padded.shape[-1] % 16 == 0
assert torch.equal(padded[:, :, :100, :70], misaligned)
def test_input_stage_rejects_a_missing_video_file():
batch = Req(sampling_params=Kandinsky6SRSamplingParams(video_path="nope.mp4"))
with pytest.raises(ValueError, match="not found"):
Kandinsky6SRInputStage().forward(batch, MagicMock())
def test_output_stage_stitches_crops_resizes_and_carries_the_source_audio():
"""The output stage blends tiles, restores the originally requested (pre-alignment-pad)
size, applies the delivery resize (fit keeps the aspect ratio and never upscales),
publishes the *final* size and fps on the request / result, and hands the source audio
through unchanged."""
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
compute_tile_grid_even,
)
grid = compute_tile_grid_even(32, 48, (16, 24), 0.2, 16)
plan = TilePlan(
tiling_scale=2,
frame_hw=(32, 48),
base_hw=(32, 48),
tile_hw=(16, 24),
pixel_grid=grid,
)
tiles = [
torch.full((3, 9, 32, 48), 100 + i, dtype=torch.uint8)
for i in range(grid.total_tiles)
]
audio = torch.linspace(-1, 1, 1000)
params = Kandinsky6SRSamplingParams(
video_path="unused.mp4",
sr_target_resolution="48x32",
sr_target_resize_mode="fit",
)
batch = Req(sampling_params=params)
batch.extra[SR_TILES_KEY], batch.extra[SR_PLAN_KEY] = tiles, plan
batch.extra[SR_REQUESTED_HW_KEY] = (32, 48) # no alignment padding happened
batch.audio, batch.audio_sample_rate = audio, 44100
batch.fps = 12
result = Kandinsky6SROutputStage().forward(batch, MagicMock())
assert result.output.shape == (1, 3, 9, 32, 48) # 64x96 fitted into 48x32
assert (batch.height, batch.width) == (32, 48)
assert result.audio is audio and result.audio_sample_rate == 44100
assert result.fps == 12 # the worker's effective fps reaches OutputBatch
plain = Kandinsky6SRSamplingParams(video_path="unused.mp4")
batch = Req(sampling_params=plain)
batch.extra[SR_TILES_KEY], batch.extra[SR_PLAN_KEY] = tiles, plan
assert Kandinsky6SROutputStage().forward(batch, MagicMock()).output.shape == (
1,
3,
9,
64,
96,
)
def test_output_stage_crops_the_alignment_padding_back_off():
"""A plan whose stitched frame is larger than what was actually requested (the input
stage padded for alignment) must be cropped down to the requested size before any
delivery resize -- not left at the padded size."""
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
compute_tile_grid_even,
)
# Padded frame is 32x48 (aligned), but only 25x40 pixels were actually requested.
grid = compute_tile_grid_even(32, 48, (16, 24), 0.2, 16)
plan = TilePlan(
tiling_scale=2,
frame_hw=(32, 48),
base_hw=(32, 48),
tile_hw=(16, 24),
pixel_grid=grid,
)
tiles = [
torch.full((3, 9, 32, 48), 100 + i, dtype=torch.uint8)
for i in range(grid.total_tiles)
]
batch = Req(sampling_params=Kandinsky6SRSamplingParams(video_path="unused.mp4"))
batch.extra[SR_TILES_KEY], batch.extra[SR_PLAN_KEY] = tiles, plan
batch.extra[SR_REQUESTED_HW_KEY] = (25, 40)
result = Kandinsky6SROutputStage().forward(batch, MagicMock())
assert result.output.shape == (1, 3, 9, 50, 80) # (25, 40) * tiling_scale(2)
assert (batch.height, batch.width) == (50, 80)
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
"""Tile blending and initial latent precision."""
import pytest
import torch
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
DitSpec,
SamplingSpec,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
build_chunk_latent,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
compute_tile_grid_even,
stitch_tiles_hanning,
)
@pytest.mark.parametrize(
"height,width,tile_hw",
[
(512, 512, (512, 512)),
(512, 768, (512, 768)),
(768, 512, (768, 512)),
(768, 1536, (512, 768)),
],
)
@pytest.mark.parametrize("value", [0.0, 1.0, 255.0])
def test_hann_blending_preserves_constant_color_including_corners(
height, width, tile_hw, value
):
# tiny but nonzero Hann corner weights must not be clamped before normalization
grid = compute_tile_grid_even(height, width, tile_hw, 0.2, 16)
tiles = [torch.full((3, 1, *tile_hw), value) for _ in range(grid.total_tiles)]
result = stitch_tiles_hanning(tiles, grid, height, width, scale=1)
torch.testing.assert_close(
result, torch.full_like(result, value), rtol=1e-5, atol=1e-2
)
def test_uncovered_pixels_are_zero_not_nan():
grid = compute_tile_grid_even(64, 64, (64, 64), 0.2, 16)
tile = torch.full((3, 1, 64, 64), 100.0)
result = stitch_tiles_hanning([tile], grid, 64, 128, scale=1)
assert result.shape == (3, 1, 64, 128)
torch.testing.assert_close(result[..., :64], tile, rtol=1e-5, atol=1e-2)
assert torch.equal(result[..., 64:], torch.zeros_like(result[..., 64:]))
@pytest.mark.parametrize("dit_dtype", [torch.bfloat16, torch.float32, None])
def test_initial_latent_uses_dit_dtype_or_preserves_input(dit_dtype):
dit_spec = DitSpec(
instruct_type="noise",
visual_cond=False,
in_visual_dim=4,
use_motion_score=False,
patch_size=(1, 1, 1),
dtype=dit_dtype,
)
spec = SamplingSpec(
tiling_scale=2,
tiles_batch_size=1,
seed=1,
num_steps=5,
tile_min_overlap=0.2,
visual_size=512,
scale_factor=(1.0, 1.0, 1.0),
scheduler_scale=5.0,
lq_noise_scale=0.7,
lq_noise_type="linear",
lq_channel_noise_scale=0.0,
cap_noise_timestep=False,
piflow=None,
)
tile = torch.randn(4, 4, 4, 4, dtype=torch.float64)
result = build_chunk_latent(
[tile],
seed=1,
dit_spec=dit_spec,
spec=spec,
device=torch.device("cpu"),
)
assert result.dtype == (dit_dtype or tile.dtype)
@@ -0,0 +1,240 @@
# SPDX-License-Identifier: Apache-2.0
"""Streaming input / output helpers of Kandinsky 6 video SR (fps and frame budget).
The reference decodes every frame and then applies ``resample_to_target_fps`` and
``clip_to_aligned_frames``; the port streams. These tests pin that both give the same
frames, and that the output tensor survives the framework's uint8 conversion.
"""
import numpy as np
import pytest
import torch
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
Kandinsky6SRSamplingParams,
)
from sglang.multimodal_gen.runtime.entrypoints.utils import _sample_to_uint8_frames
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
Kandinsky6SRInputStage,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
SR_TILING_SCALE_KEY,
SR_VIDEO_KEY,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
FrameSelector,
decode_clip,
extract_source_audio,
to_output_video,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
clip_to_aligned_frames,
resample_to_target_fps,
)
FRAME_COUNTS = [1, 2, 9, 24, 50, 121, 125, 200, 301, 302, 400, 1000]
SOURCE_FPS = [12, 23.976, 24, 25, 29.97, 30, 48, 50, 59.94, 60, 120]
def _stream_select(total: int, fps: float) -> tuple[list[int], int]:
"""Run the streaming selector over frames ``0 .. total - 1`` (early stop included)."""
selector = FrameSelector(fps)
kept: list[int] = []
seen = 0
for index in range(total):
if selector.keep(index):
kept.append(index)
selector.mark_kept()
seen += 1
if selector.enough(seen):
break
return kept[: selector.limit(seen, len(kept))], selector.effective_fps
def _reference_select(total: int, fps: float) -> tuple[list[int], int] | None:
ids = torch.arange(total).view(total, 1, 1, 1)
resampled, effective_fps = resample_to_target_fps(ids, fps)
try:
clipped = clip_to_aligned_frames(resampled)
except ValueError:
return None
return clipped.flatten().tolist(), effective_fps
@pytest.mark.parametrize("fps", SOURCE_FPS)
def test_streaming_selection_equals_reference_resample_and_clip(fps):
"""The stream may stop early once 121 frames are kept, but must return exactly the
frames (and the effective fps) of decode-all -> resample_to_target_fps ->
clip_to_aligned_frames, including sources too short for the 121-frame budget and
``int(total / step)`` truncation of the last selectable frame."""
for total in FRAME_COUNTS:
expected = _reference_select(total, fps)
kept, effective_fps = _stream_select(total, fps)
if expected is None:
# the reference raises "no readable frames"; nothing usable was selected
assert not kept, (total, fps)
continue
ids = torch.tensor(kept).view(-1, 1, 1, 1)
actual = clip_to_aligned_frames(ids).flatten().tolist()
assert (actual, effective_fps) == expected, (total, fps)
def test_effective_fps_is_an_int_by_the_reference_rule():
"""Downsampled sources play at exactly 24, in-tolerance / slow sources keep
``round(source fps)`` (never a float: SamplingParams.fps must be a positive int)."""
assert FrameSelector(60.0).effective_fps == 24
assert FrameSelector(25.0).effective_fps == 25
assert FrameSelector(23.976).effective_fps == 24
assert FrameSelector(12.0).effective_fps == 12
assert all(isinstance(FrameSelector(fps).effective_fps, int) for fps in SOURCE_FPS)
# --------------------------------------------------------------------------- #
# Real files (PyAV)
# --------------------------------------------------------------------------- #
av = pytest.importorskip("av")
def write_test_video(path, *, frames, fps, size=(48, 64), audio_seconds=None):
"""Tiny mp4 whose frame ``i`` is a flat colour ``i`` (so index errors are visible)."""
from fractions import Fraction
height, width = size
with av.open(str(path), mode="w") as container:
video = container.add_stream("mpeg4", rate=fps)
video.width, video.height, video.pix_fmt = width, height, "yuv420p"
audio = None
if audio_seconds is not None:
audio = container.add_stream("aac", rate=22050)
audio.layout = "mono"
for index in range(frames):
gray = (index * 5) % 250
array = np.full((height, width, 3), gray, dtype=np.uint8)
frame = av.VideoFrame.from_ndarray(array, format="rgb24")
frame.pts, frame.time_base = index, Fraction(1, fps)
for packet in video.encode(frame):
container.mux(packet)
if audio is not None:
t = np.arange(int(22050 * audio_seconds)) / 22050
samples = (0.3 * np.sin(2 * np.pi * 440 * t)).astype(np.float32)[None]
frame = av.AudioFrame.from_ndarray(samples, format="fltp", layout="mono")
frame.sample_rate, frame.pts = 22050, 0
frame.time_base = Fraction(1, 22050)
for packet in audio.encode(frame):
container.mux(packet)
for packet in audio.encode():
container.mux(packet)
for packet in video.encode():
container.mux(packet)
def _decode_everything(path):
with av.open(str(path)) as container:
stream = container.streams.video[0]
fps = float(stream.average_rate)
frames = [
torch.from_numpy(f.to_ndarray(format="rgb24")).permute(2, 0, 1)
for f in container.decode(stream)
]
return torch.stack(frames), fps
@pytest.mark.parametrize("fps,frames", [(30, 40), (24, 26), (12, 20), (60, 80)])
def test_decode_clip_equals_reference_cli_pipeline(tmp_path, fps, frames):
"""decode_clip == read_video_tchw_uint8 -> resample_to_target_fps ->
clip_to_aligned_frames: same pixels, same frame count (1 + 8k), same output fps."""
path = tmp_path / "clip.mp4"
write_test_video(path, frames=frames, fps=fps)
full, source_fps = _decode_everything(path)
expected, expected_fps = resample_to_target_fps(full, source_fps)
expected = clip_to_aligned_frames(expected)
clip = decode_clip(str(path))
assert clip.frames.dtype == torch.uint8 and clip.frames.shape[1] == 3
assert clip.frames.shape[0] % 8 == 1
assert torch.equal(clip.frames, expected)
assert clip.fps == expected_fps and isinstance(clip.fps, int)
def test_extract_source_audio_is_mono_float_trimmed_and_optional(tmp_path):
"""Audio is decoded to mono float32 at 44.1 kHz, trimmed to the processed duration
(the reference muxes the whole track, leaving audio longer than the video), and a
file without an audio stream yields None instead of a silent track."""
with_audio = tmp_path / "with_audio.mp4"
write_test_video(with_audio, frames=10, fps=10, audio_seconds=1.0)
full = extract_source_audio(str(with_audio))
trimmed = extract_source_audio(str(with_audio), max_seconds=0.25)
assert full.dtype == np.float32 and full.ndim == 1
assert abs(len(full) - 44100) < 2048 # aac priming / resampler delay
assert len(trimmed) == 11025
assert np.abs(full).max() <= 1.0 and np.abs(full).max() > 0.05
silent = tmp_path / "silent.mp4"
write_test_video(silent, frames=10, fps=10)
assert extract_source_audio(str(silent)) is None
def test_output_video_round_trips_through_uint8_conversions():
"""Every uint8 value must survive the framework writer's ``(x * 255)`` truncation,
also when a consumer upcasts the fp16 tensor before multiplying (a plain ``k / 255``
stored in fp16 fails that order for over half of the values)."""
values = torch.arange(256, dtype=torch.uint8)
video = values.view(1, 1, 8, 32).expand(3, 2, 8, 32).contiguous()
output = to_output_video(video)
assert output.dtype == torch.float16 and output.shape == (1, 3, 2, 8, 32)
frames = _sample_to_uint8_frames(output[0]) # framework writer: list of [H, W, 3]
restored = torch.from_numpy(np.stack(frames)).permute(3, 0, 1, 2)
assert torch.equal(restored, video)
assert torch.equal((output.float() * 255).to(torch.uint8)[0], video)
naive = (video.float() / 255.0).to(torch.float16)
assert not torch.equal((naive.float() * 255).to(torch.uint8), video)
@pytest.mark.parametrize(
"scale,size,expected_frame,tiling_scale",
[
(2, (48, 64), (48, 64), 2),
(2.25, (64, 96), (64, 112), 2),
(4, (48, 64), (48, 64), 4),
],
)
def test_input_stage_fills_the_request_from_a_video_file(
tmp_path, scale, size, expected_frame, tiling_scale
):
"""A 30 fps source is thinned to 24 fps (an int), floored to 1 + 8k frames, and the
request's output geometry follows the clip: x1.125 pre-upscale aligned to 16 for the
2.25 scale, the tile scale for the rest. The audio is mono float32 at 44.1 kHz and
trimmed to the processed duration."""
path = tmp_path / "clip.mp4"
write_test_video(path, frames=40, fps=30, size=size, audio_seconds=1.5)
params = Kandinsky6SRSamplingParams(video_path=str(path), sr_resolution_scale=scale)
batch = Req(sampling_params=params)
Kandinsky6SRInputStage().forward(batch, None)
video = batch.extra[SR_VIDEO_KEY]
assert video.dtype == torch.uint8 and video.shape[1] == 3
assert video.shape[0] == 25 and batch.num_frames == 25 # int(40 / 1.25) = 32 -> 25
assert tuple(video.shape[2:]) == expected_frame
assert batch.extra[SR_TILING_SCALE_KEY] == tiling_scale
assert batch.fps == 24 and isinstance(batch.fps, int)
assert (batch.height, batch.width) == (
expected_frame[0] * tiling_scale,
expected_frame[1] * tiling_scale,
)
assert batch.audio.dtype == torch.float32 and batch.audio.ndim == 1
assert batch.audio.shape[0] == round(25 / 24 * 44100)
assert batch.audio_sample_rate == 44100
def test_input_stage_produces_no_audio_for_a_silent_source(tmp_path):
path = tmp_path / "silent.mp4"
write_test_video(path, frames=9, fps=24)
batch = Req(sampling_params=Kandinsky6SRSamplingParams(video_path=str(path)))
Kandinsky6SRInputStage().forward(batch, None)
assert batch.audio is None and batch.audio_sample_rate is None
assert batch.fps == 24 and batch.num_frames == 9

Some files were not shown because too many files have changed in this diff Show More