[diffusion] model: support Kandinsky 6 (#42743)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Lev Novitskiy <57654885+leffff@users.noreply.github.com> Co-authored-by: Kirill <kykozlov@gmail.com> Co-authored-by: Artem Borisov <divotionchannel@gmail.com>
This commit is contained in:
co-authored by
Mick Qian
Lev Novitskiy
Kirill
Artem Borisov
parent
730f1f3e5b
commit
16a23a672b
@@ -151,6 +151,10 @@ Several norms look interchangeable and are not. Start here.
|
||||
| Entry point | Backend | Contract | Applies to |
|
||||
|---|---|---|---|
|
||||
| `residual_gate_add` | JIT CUDA (contiguous), Triton (transposed) | bit-exact `residual + update * gate` | contiguous tensors, or a transposed-dense `[B, tokens, hidden]` residual/output with contiguous update and row-broadcast gate (SANA-Video) |
|
||||
| `residual_gate_fp32` | Triton | separate FP32 multiply/add, then cast to input dtype | contiguous `[B,S,D]`, strided `[B or 1,1,D]` gate; Kandinsky6 |
|
||||
|
||||
These contracts differ: `residual_gate_fp32` does not round the product to
|
||||
BF16/FP16, and disables FMA to preserve the eager FP32 rounding boundary.
|
||||
|
||||
The transposed-dense path tiles along the physical stride-1 token dimension
|
||||
for coalesced residual reads and output writes. The contiguous layouts use
|
||||
|
||||
@@ -49,6 +49,20 @@ _HIP = frozenset({CapabilityRequirement.HIP})
|
||||
# inventory: callers name the one they want via ``select_kernel``.
|
||||
# ---------------------------------------------------------------------------
|
||||
_SPECS: tuple[tuple[str, KernelBackend, str, frozenset, str], ...] = (
|
||||
(
|
||||
"diffusion.residual_gate_fp32",
|
||||
KernelBackend.TRITON,
|
||||
"modulate.residual_gate_fp32:residual_gate_fp32",
|
||||
_CUDA,
|
||||
"Residual gating with separate FP32 product and sum rounding.",
|
||||
),
|
||||
(
|
||||
"diffusion.matrix_rope",
|
||||
KernelBackend.TRITON,
|
||||
"rope.matrix_rope:apply_matrix_rope",
|
||||
_CUDA,
|
||||
"Interleaved FP32 matrix RoPE with separate product and sum rounding.",
|
||||
),
|
||||
(
|
||||
"diffusion.fp8_rowwise",
|
||||
KernelBackend.TRITON,
|
||||
@@ -590,6 +604,8 @@ for _op, _backend, _target, _caps, _description in _SPECS:
|
||||
# then symbol; a new public kernel belongs here and nowhere else.
|
||||
# ---------------------------------------------------------------------------
|
||||
_EXPORTS: dict[str, str] = {
|
||||
"apply_matrix_rope": "rope.matrix_rope",
|
||||
"residual_gate_fp32": "modulate.residual_gate_fp32",
|
||||
"can_use_fp8_rowwise": "quantization.fp8_rowwise_triton",
|
||||
"fp8_rowwise": "quantization.fp8_rowwise_triton",
|
||||
"fused_gelu_tanh_cat": "activation.gelu_tanh_cat_jit",
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Residual gating with FP32 intermediates and a single final storage cast."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _residual_gate_fp32_kernel(
|
||||
x,
|
||||
update,
|
||||
gate,
|
||||
out,
|
||||
count,
|
||||
TOKENS: tl.constexpr,
|
||||
HIDDEN: tl.constexpr,
|
||||
GATE_BATCH_STRIDE: tl.constexpr,
|
||||
GATE_CHANNEL_STRIDE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
index = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
|
||||
valid = index < count
|
||||
gate_index = (
|
||||
index // (TOKENS * HIDDEN) * GATE_BATCH_STRIDE
|
||||
+ index % HIDDEN * GATE_CHANNEL_STRIDE
|
||||
)
|
||||
residual = tl.load(x + index, valid, other=0).to(tl.float32)
|
||||
value = tl.load(update + index, valid, other=0).to(tl.float32)
|
||||
weight = tl.load(gate + gate_index, valid, other=0).to(tl.float32)
|
||||
# eager rounds the multiply to FP32, not to the input dtype or an FMA
|
||||
result = residual + weight * value
|
||||
tl.store(out + index, result, valid)
|
||||
|
||||
|
||||
def _fake_residual_gate_fp32(
|
||||
x: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(x)
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
op_name="diffusion_residual_gate_fp32",
|
||||
mutates_args=[],
|
||||
fake_impl=_fake_residual_gate_fp32,
|
||||
)
|
||||
def _residual_gate_fp32(
|
||||
x: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
output = torch.empty_like(x)
|
||||
if x.numel():
|
||||
with torch.cuda.device(x.device):
|
||||
_residual_gate_fp32_kernel[(triton.cdiv(x.numel(), 1024),)](
|
||||
x,
|
||||
update,
|
||||
gate,
|
||||
output,
|
||||
x.numel(),
|
||||
TOKENS=x.shape[1],
|
||||
HIDDEN=x.shape[2],
|
||||
GATE_BATCH_STRIDE=gate.stride(0) if gate.shape[0] != 1 else 0,
|
||||
GATE_CHANNEL_STRIDE=gate.stride(2),
|
||||
BLOCK=1024,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def residual_gate_fp32(
|
||||
x: torch.Tensor, update: torch.Tensor, gate: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Compute ``(x.float() + gate.float() * update.float()).to(x.dtype)``.
|
||||
|
||||
Unlike ``residual_gate_add``, the product is not rounded to BF16/FP16.
|
||||
The CUDA path accepts contiguous [B,S,D] and a [B or 1,1,D] gate.
|
||||
"""
|
||||
if (
|
||||
x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and not torch.is_grad_enabled()
|
||||
and x.ndim == 3
|
||||
and x.shape == update.shape
|
||||
and x.is_contiguous()
|
||||
and update.is_contiguous()
|
||||
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
and update.dtype == x.dtype
|
||||
and gate.dtype in (x.dtype, torch.float32)
|
||||
and gate.device == update.device == x.device
|
||||
and gate.ndim == 3
|
||||
and gate.shape[0] in (1, x.shape[0])
|
||||
and gate.shape[1:] == (1, x.shape[2])
|
||||
):
|
||||
return _residual_gate_fp32(x, update, gate)
|
||||
return (x.float() + gate.float() * update.float()).to(x.dtype)
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Interleaved RoPE with the eager FP32 matrix-product rounding boundaries."""
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
|
||||
from sglang.srt.utils.custom_op import register_custom_op
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _matrix_rope_kernel(
|
||||
x,
|
||||
rope,
|
||||
out,
|
||||
pairs,
|
||||
SEQ: tl.constexpr,
|
||||
HEADS: tl.constexpr,
|
||||
HALF_DIM: tl.constexpr,
|
||||
BATCH_STRIDE: tl.constexpr,
|
||||
SEQ_STRIDE: tl.constexpr,
|
||||
PAIR_STRIDE: tl.constexpr,
|
||||
ROW_STRIDE: tl.constexpr,
|
||||
COL_STRIDE: tl.constexpr,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
idx = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
|
||||
valid = idx < pairs
|
||||
token = idx // (HEADS * HALF_DIM)
|
||||
pos = (
|
||||
token // SEQ * BATCH_STRIDE
|
||||
+ token % SEQ * SEQ_STRIDE
|
||||
+ idx % HALF_DIM * PAIR_STRIDE
|
||||
)
|
||||
# preserve the input cast before FP32 products when fusing a dtype conversion
|
||||
x0 = tl.load(x + 2 * idx, valid, other=0).to(out.dtype.element_ty).to(tl.float32)
|
||||
x1 = (
|
||||
tl.load(x + 2 * idx + 1, valid, other=0).to(out.dtype.element_ty).to(tl.float32)
|
||||
)
|
||||
r00 = tl.load(rope + pos, valid, other=0)
|
||||
r01 = tl.load(rope + pos + COL_STRIDE, valid, other=0)
|
||||
r10 = tl.load(rope + pos + ROW_STRIDE, valid, other=0)
|
||||
r11 = tl.load(rope + pos + ROW_STRIDE + COL_STRIDE, valid, other=0)
|
||||
# do not contract the products: eager materializes them before the pair sum
|
||||
y0 = x0 * r00 + x1 * r01
|
||||
y1 = x0 * r10 + x1 * r11
|
||||
tl.store(out + 2 * idx, y0, valid)
|
||||
tl.store(out + 2 * idx + 1, y1, valid)
|
||||
|
||||
|
||||
def _fake_apply_matrix_rope(
|
||||
x: torch.Tensor, rope: torch.Tensor, dtype: torch.dtype | None = None
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(x, dtype=dtype)
|
||||
|
||||
|
||||
@register_custom_op(
|
||||
op_name="diffusion_apply_matrix_rope",
|
||||
mutates_args=[],
|
||||
fake_impl=_fake_apply_matrix_rope,
|
||||
)
|
||||
def apply_matrix_rope(
|
||||
x: torch.Tensor, rope: torch.Tensor, dtype: torch.dtype | None = None
|
||||
) -> torch.Tensor:
|
||||
"""Apply FP32 matrices to [B,S,H,D], optionally casting x to dtype first.
|
||||
|
||||
The result has the cast dtype; the input rounding precedes FP32 products.
|
||||
"""
|
||||
dtype = x.dtype if dtype is None else dtype
|
||||
if not (
|
||||
x.is_cuda
|
||||
and torch.version.hip is None
|
||||
and x.ndim == 4
|
||||
and x.is_contiguous()
|
||||
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
and dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
):
|
||||
raise ValueError(
|
||||
"matrix RoPE requires contiguous FP16/BF16/FP32 CUDA [B,S,H,D]"
|
||||
)
|
||||
batch, seq, heads, dim = x.shape
|
||||
if not (
|
||||
dim % 2 == 0
|
||||
and rope.device == x.device
|
||||
and rope.dtype == torch.float32
|
||||
and rope.ndim == 6
|
||||
and rope.shape[0] in (1, batch)
|
||||
and rope.shape[1:] == (seq, 1, dim // 2, 2, 2)
|
||||
):
|
||||
raise ValueError("matrix RoPE requires matching FP32 [B or 1,S,1,D/2,2,2]")
|
||||
output = torch.empty_like(x, dtype=dtype)
|
||||
pairs = x.numel() // 2
|
||||
if pairs:
|
||||
with torch.cuda.device(x.device):
|
||||
_matrix_rope_kernel[(triton.cdiv(pairs, 512),)](
|
||||
x,
|
||||
rope,
|
||||
output,
|
||||
pairs,
|
||||
SEQ=seq,
|
||||
HEADS=heads,
|
||||
HALF_DIM=dim // 2,
|
||||
BATCH_STRIDE=rope.stride(0) if rope.shape[0] != 1 else 0,
|
||||
SEQ_STRIDE=rope.stride(1),
|
||||
PAIR_STRIDE=rope.stride(3),
|
||||
ROW_STRIDE=rope.stride(4),
|
||||
COL_STRIDE=rope.stride(5),
|
||||
BLOCK=512,
|
||||
enable_fp_fusion=False,
|
||||
)
|
||||
return output
|
||||
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Kandinsky 6 joint video/audio checkpoint architecture.
|
||||
|
||||
Distilled checkpoints have n_grid velocity outputs per latent channel."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6ArchConfig(DiTArchConfig):
|
||||
"""Checkpoint fields and derived dimensions for Kandinsky6Transformer3DModel."""
|
||||
|
||||
# Chained layer renames apply to checkpoint tensors and LoRA A/B/alpha keys.
|
||||
param_names_mapping: dict = field(
|
||||
default_factory=lambda: {
|
||||
r"^(visual_transformer_blocks\.\d+)\.video_dec_block\.(.*)$": r"\1.videoT.\2",
|
||||
r"^(visual_transformer_blocks\.\d+)\.audio_dec_block\.(.*)$": r"\1.audioT.\2",
|
||||
r"^(.*feed_forward)\.net\.0\.proj\.(.*)$": r"\1.mlp.fc_in.\2",
|
||||
r"^(.*feed_forward)\.net\.2\.(.*)$": r"\1.mlp.fc_out.\2",
|
||||
r"^(video_time_embeddings|audio_time_embeddings)\.timestep_embedder\.linear_1\.(.*)$": r"\1.in_layer.\2",
|
||||
r"^(video_time_embeddings|audio_time_embeddings)\.timestep_embedder\.linear_2\.(.*)$": r"\1.out_layer.\2",
|
||||
r"^((?:video|audio)_text_transformer_blocks\.\d+)\.attn_norm\.(.*)$": r"\1.self_attention_norm.\2",
|
||||
r"^((?:video|audio)_text_transformer_blocks\.\d+)\.attn\.(.*)$": r"\1.self_attention.\2",
|
||||
}
|
||||
)
|
||||
lora_param_names_mapping: dict = field(
|
||||
default_factory=lambda: {r"^transformer\.(.*)$": r"\1"}
|
||||
)
|
||||
|
||||
# Diffusers Kandinsky6Transformer3DModel config fields (mirror
|
||||
# transformer/config.json 1:1).
|
||||
in_visual_dim: int = 16
|
||||
out_visual_dim: int = 16
|
||||
in_text_dim: int = 3584 # Qwen2.5-7B hidden size (Reason1 text encoder)
|
||||
in_text_dim2: int = 768 # CLIP pooled dim
|
||||
time_dim: int = 1024
|
||||
patch_size: tuple[int, int, int] = (1, 2, 2)
|
||||
# checkpoint RoPE scaling is fixed across request resolutions
|
||||
scale_factor: tuple[float, float, float] = (1.0, 2.0, 2.0)
|
||||
model_dim: int = 4096
|
||||
ff_dim: int = 16384
|
||||
num_text_blocks: int = 4
|
||||
num_visual_blocks: int = 60
|
||||
axes_dims: tuple[int, int, int] = (32, 48, 48) # 3D RoPE T/H/W split
|
||||
visual_cond: bool = True
|
||||
|
||||
# False selects the video-only architecture; TI2VA pipelines use True
|
||||
is_multimodal: bool = True
|
||||
out_audio_dim: int | None = None
|
||||
in_audio_dim: int = 20
|
||||
model_dim_a: int | None = None
|
||||
time_dim_a: int | None = None
|
||||
ff_dim_a: int | None = None
|
||||
axes_dims_a: tuple[int, int, int] | None = None
|
||||
audio_freqs_scaling: float = 1.0
|
||||
|
||||
# checkpoint metadata, not a runtime backend override; NABLA is rejected
|
||||
attention_engine: str = "auto"
|
||||
attention_causal: bool | None = None
|
||||
attention_local: bool | None = None
|
||||
attention_glob: bool | None = None
|
||||
attention_window: int | None = None
|
||||
attention_P: float | None = None
|
||||
attention_wT: int | None = None
|
||||
attention_wW: int | None = None
|
||||
attention_wH: int | None = None
|
||||
attention_add_sta: bool | None = None
|
||||
attention_method: str | None = None
|
||||
|
||||
# checkpoint metadata; the pipeline passes unpadded text rather than masks
|
||||
text_token_padding: bool = False
|
||||
|
||||
# Video<->audio fused block knobs (Kandinsky6FusedTransformerDecoderBlock).
|
||||
ca_rope: bool = False
|
||||
cross_gates: bool = False
|
||||
fix_modulation: bool = False
|
||||
# generated/reference token embeddings for tail-frame image conditioning
|
||||
visual_token_type_num_embeddings: int = 2
|
||||
|
||||
# Derived, DiTArchConfig-contract fields (set in __post_init__ below;
|
||||
# not part of the diffusers config.json).
|
||||
in_channels: int = 0
|
||||
out_channels: int = 0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
super().__post_init__()
|
||||
if isinstance(self.patch_size, list):
|
||||
self.patch_size = tuple(self.patch_size)
|
||||
if len(self.patch_size) != 3:
|
||||
raise ValueError(f"patch_size must have 3 values, got {self.patch_size}.")
|
||||
if isinstance(self.scale_factor, list):
|
||||
self.scale_factor = tuple(self.scale_factor)
|
||||
if len(self.scale_factor) != 3:
|
||||
raise ValueError(
|
||||
f"scale_factor must have 3 values, got {self.scale_factor}."
|
||||
)
|
||||
self.scale_factor = tuple(float(value) for value in self.scale_factor)
|
||||
|
||||
head_dim = sum(self.axes_dims)
|
||||
if self.model_dim % head_dim != 0:
|
||||
raise ValueError(
|
||||
f"model_dim ({self.model_dim}) must be divisible by head_dim "
|
||||
f"({head_dim})"
|
||||
)
|
||||
self.hidden_size = self.model_dim
|
||||
self.num_attention_heads = self.model_dim // head_dim
|
||||
self.in_channels = self.in_visual_dim
|
||||
self.out_channels = self.out_visual_dim
|
||||
self.num_channels_latents = self.in_visual_dim
|
||||
|
||||
# resolve omitted audio dimensions from the video tower
|
||||
self.model_dim_a = self.model_dim_a or self.model_dim
|
||||
self.time_dim_a = self.time_dim_a or self.time_dim
|
||||
self.ff_dim_a = self.ff_dim_a or self.ff_dim
|
||||
self.axes_dims_a = self.axes_dims_a or self.axes_dims
|
||||
head_dim_a = sum(self.axes_dims_a)
|
||||
if self.model_dim_a % head_dim_a != 0:
|
||||
raise ValueError(
|
||||
f"model_dim_a ({self.model_dim_a}) must be divisible by "
|
||||
f"head_dim_a ({head_dim_a})"
|
||||
)
|
||||
|
||||
# without fix_modulation, cross-modal modulation uses the other tower's time
|
||||
# embedding, so video and audio time widths must match
|
||||
if (
|
||||
self.is_multimodal
|
||||
and not self.fix_modulation
|
||||
and self.time_dim_a != self.time_dim
|
||||
):
|
||||
raise ValueError(
|
||||
f"time_dim_a ({self.time_dim_a}) must equal time_dim "
|
||||
f"({self.time_dim}) unless fix_modulation=True "
|
||||
"(Kandinsky6FusedTransformerDecoderBlock's cross-modal "
|
||||
"modulation is driven by the other modality's time "
|
||||
"embedding by default)."
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6VideoAudioConfig(DiTConfig):
|
||||
arch_config: DiTArchConfig = field(default_factory=Kandinsky6ArchConfig)
|
||||
prefix: str = "Kandinsky6"
|
||||
|
||||
|
||||
__all__ = ["Kandinsky6ArchConfig", "Kandinsky6VideoAudioConfig"]
|
||||
@@ -0,0 +1,269 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""SR checkpoint architecture and sampling metadata.
|
||||
|
||||
Official out_visual_dim is the total DX head width; n_grid lives in the scheduler.
|
||||
Unsupported architecture options and unknown fields are rejected."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.base import DiTArchConfig, DiTConfig
|
||||
|
||||
WIDE_INPUT_INSTRUCT_TYPES = ("channel", "hybrid", "hybrid_anchor")
|
||||
SUPPORTED_INSTRUCT_TYPES = ("noise", *WIDE_INPUT_INSTRUCT_TYPES)
|
||||
# Post-load behaviour overrides the port accepts (no blind ``setattr``).
|
||||
ATTRIBUTE_OVERRIDE_WHITELIST = ("instruct_type", "visual_cond", "attention_params")
|
||||
# Bookkeeping keys the loader leaves in the component config.
|
||||
_TOLERATED_EXTRA_KEYS = frozenset({"_class_name", "_diffusers_version"})
|
||||
_SPARSE_ATTENTION_TYPES = ("nabla", "nabla_framewise_causal")
|
||||
_PIFLOW_FIELDS = (
|
||||
"piflow_nfe",
|
||||
"piflow_num_policy_substeps",
|
||||
"piflow_final_step_size_scale",
|
||||
"piflow_shift",
|
||||
"piflow_eps",
|
||||
)
|
||||
_LQ_NOISE_TYPES = ("ddpm", "linear")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6SRArchConfig(DiTArchConfig):
|
||||
"""Static architecture metadata of ``Kandinsky6SRTransformer3DModel``."""
|
||||
|
||||
# Map the current Diffusers checkpoint names to the native SGLang port.
|
||||
param_names_mapping: dict = field(
|
||||
default_factory=lambda: {
|
||||
r"^(.*feed_forward)\.net\.0\.proj\.(weight|bias)$": r"\1.mlp.fc_in.\2",
|
||||
r"^(.*feed_forward)\.net\.2\.(weight|bias)$": r"\1.mlp.fc_out.\2",
|
||||
r"^time_embeddings\.timestep_embedder\.linear_1\.(weight|bias)$": r"time_embeddings.in_layer.\1",
|
||||
r"^time_embeddings\.timestep_embedder\.linear_2\.(weight|bias)$": r"time_embeddings.out_layer.\1",
|
||||
}
|
||||
)
|
||||
|
||||
# --- DiffusionTransformer3D constructor keys (trained architecture) ---
|
||||
in_visual_dim: int = 4
|
||||
in_text_dim: int = 3584
|
||||
in_text_dim2: int = 768
|
||||
time_dim: int = 512
|
||||
out_visual_dim: int = 4
|
||||
patch_size: tuple[int, int, int] = (1, 2, 2)
|
||||
model_dim: int = 2048
|
||||
ff_dim: int = 5120
|
||||
num_text_blocks: int = 2
|
||||
num_visual_blocks: int = 32
|
||||
axes_dims: tuple[int, int, int] = (16, 24, 24)
|
||||
visual_cond: bool = False
|
||||
instruct_type: str | None = None
|
||||
attention_params: dict[str, Any] | None = None
|
||||
use_motion_score: bool = False
|
||||
use_lq_modulation: bool = False
|
||||
lq_modulation_sublayers: dict[str, bool] | None = None
|
||||
zero_lq_in_main_path: bool = False
|
||||
use_adapter: bool = False
|
||||
adapter_gamma: float = 0.0
|
||||
# The reference default is True; the SR release is text-free and its config says
|
||||
# so explicitly.
|
||||
use_text: bool = False
|
||||
use_lq_noise_cond: bool = False
|
||||
|
||||
# --- optional legacy / test-only DX head / pi-Flow fields ---
|
||||
# An official checkpoint stores the TOTAL head width in ``out_visual_dim`` and the
|
||||
# pi-Flow values in its scheduler config; with ``n_grid > 1`` here the head is
|
||||
# ``out_visual_dim * n_grid`` wide instead (``out_visual_dim`` being the base dim).
|
||||
n_grid: int = 1
|
||||
piflow_nfe: int | None = None
|
||||
piflow_num_policy_substeps: int | None = None
|
||||
piflow_final_step_size_scale: float | None = None
|
||||
piflow_shift: float | None = None
|
||||
piflow_eps: float | None = None
|
||||
|
||||
# Post-load attribute overrides (whitelist: instruct_type, visual_cond,
|
||||
# attention_params); the architecture above stays as trained. The official
|
||||
# config writes ``null`` when there are none.
|
||||
attribute_overrides: dict[str, Any] | None = field(default_factory=dict)
|
||||
|
||||
# --- SR sampling parameters of the training config ---
|
||||
# As stored in ``transformer/config.json`` (``sr_params``); expanded into the
|
||||
# flat ``sr_*`` fields below in ``__post_init__``.
|
||||
sr_params: dict[str, Any] | None = None
|
||||
sr_visual_size: list[int] = field(default_factory=lambda: [512])
|
||||
sr_scale_factor: dict[str, list[float]] | None = None
|
||||
sr_scheduler_scale: float = 5.0
|
||||
sr_lq_noise_scale: float = 0.7
|
||||
sr_lq_noise_type: str = "ddpm"
|
||||
sr_lq_channel_noise_scale: float = 0.0
|
||||
sr_cap_noise_timestep: bool = False
|
||||
sr_fps: int = 24
|
||||
|
||||
# DiTArchConfig contract fields, derived in __post_init__.
|
||||
in_channels: int = 0
|
||||
out_channels: int = 0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
super().__post_init__()
|
||||
if self.attribute_overrides is None:
|
||||
self.attribute_overrides = {}
|
||||
if self.sr_params is not None:
|
||||
self._expand_sr_params(self.sr_params)
|
||||
self._reject_unknown_keys()
|
||||
self._reject_unsupported_flags()
|
||||
self._normalize_and_validate_shapes()
|
||||
self._validate_overrides()
|
||||
self._validate_sampler_fields()
|
||||
self.hidden_size = self.model_dim
|
||||
self.num_attention_heads = self.model_dim // sum(self.axes_dims)
|
||||
self.in_channels = self.in_visual_dim
|
||||
self.out_channels = self.out_visual_dim
|
||||
self.num_channels_latents = self.in_visual_dim
|
||||
|
||||
@property
|
||||
def is_piflow(self) -> bool:
|
||||
return self.piflow_nfe is not None
|
||||
|
||||
@property
|
||||
def head_width(self) -> int:
|
||||
"""Channels the DiT head emits per latent position (all grids of a DX head)."""
|
||||
return self.out_visual_dim * self.n_grid
|
||||
|
||||
@property
|
||||
def base_out_visual_dim(self) -> int:
|
||||
"""Channels per latent grid; legacy n_grid configs store the base output width."""
|
||||
return self.out_visual_dim if self.n_grid > 1 else self.in_visual_dim
|
||||
|
||||
@property
|
||||
def trained_wide_input(self) -> bool:
|
||||
"""Whether the input layer was trained on ``[x | cond | mask]`` channels."""
|
||||
return bool(self.visual_cond or self.instruct_type in WIDE_INPUT_INSTRUCT_TYPES)
|
||||
|
||||
def effective_override(self, name: str) -> Any:
|
||||
"""Post-load value of ``name``: the override if present, else as trained."""
|
||||
if name in self.attribute_overrides:
|
||||
return self.attribute_overrides[name]
|
||||
return getattr(self, name)
|
||||
|
||||
def requested_sparse_attention(self) -> str | None:
|
||||
"""Sparse attention type asked for by ``attention_params``, if any."""
|
||||
params = self.effective_override("attention_params")
|
||||
if not isinstance(params, dict):
|
||||
return None
|
||||
for value in params.values():
|
||||
attention_type = value.get("type") if isinstance(value, dict) else None
|
||||
if attention_type in _SPARSE_ATTENTION_TYPES:
|
||||
return attention_type
|
||||
return None
|
||||
|
||||
_SR_PARAM_FIELDS = {
|
||||
"visual_size": "sr_visual_size",
|
||||
"scale_factor": "sr_scale_factor",
|
||||
"scheduler_scale": "sr_scheduler_scale",
|
||||
"lq_noise_scale": "sr_lq_noise_scale",
|
||||
"lq_noise_type": "sr_lq_noise_type",
|
||||
"lq_channel_noise_scale": "sr_lq_channel_noise_scale",
|
||||
"cap_noise_timestep": "sr_cap_noise_timestep",
|
||||
"fps": "sr_fps",
|
||||
}
|
||||
|
||||
def _expand_sr_params(self, sr_params: dict[str, Any]) -> None:
|
||||
"""Nested ``sr_params`` of ``transformer/config.json`` -> the flat ``sr_*`` fields."""
|
||||
unknown = sorted(set(sr_params) - set(self._SR_PARAM_FIELDS))
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR sr_params has unsupported keys {unknown}; accepted "
|
||||
f"keys: {sorted(self._SR_PARAM_FIELDS)}"
|
||||
)
|
||||
for key, value in sr_params.items():
|
||||
setattr(self, self._SR_PARAM_FIELDS[key], value)
|
||||
|
||||
def _reject_unknown_keys(self) -> None:
|
||||
unknown = sorted(set(self.extra_attrs) - _TOLERATED_EXTRA_KEYS)
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
"Kandinsky6SR transformer config has unknown keys "
|
||||
f"{unknown}; declare them on Kandinsky6SRArchConfig or fix the checkpoint config."
|
||||
)
|
||||
|
||||
def _reject_unsupported_flags(self) -> None:
|
||||
if self.use_text:
|
||||
raise NotImplementedError(
|
||||
"Kandinsky6SR: use_text=True (text-conditioned SR DiT) is not "
|
||||
"supported; only the text-free SR DiT (use_text=False) is implemented."
|
||||
)
|
||||
if self.use_adapter:
|
||||
raise NotImplementedError(
|
||||
"Kandinsky6SR: use_adapter=True (VideoAdapter) is not supported."
|
||||
)
|
||||
if self.use_lq_modulation:
|
||||
raise NotImplementedError(
|
||||
"Kandinsky6SR: use_lq_modulation=True (per-block LQ modulation) is "
|
||||
"not supported."
|
||||
)
|
||||
|
||||
def _normalize_and_validate_shapes(self) -> None:
|
||||
self.patch_size = tuple(self.patch_size)
|
||||
self.axes_dims = tuple(self.axes_dims)
|
||||
if len(self.patch_size) != 3 or self.patch_size[0] != 1:
|
||||
raise ValueError(
|
||||
"Kandinsky6SR patch_size must be (1, ph, pw) with a unit temporal "
|
||||
f"patch, got {self.patch_size}"
|
||||
)
|
||||
if len(self.axes_dims) != 3 or any(dim % 2 for dim in self.axes_dims):
|
||||
raise ValueError(
|
||||
f"axes_dims must be three even sizes, got {self.axes_dims}"
|
||||
)
|
||||
head_dim = sum(self.axes_dims)
|
||||
if self.model_dim % head_dim != 0:
|
||||
raise ValueError(
|
||||
f"model_dim ({self.model_dim}) must be divisible by head_dim "
|
||||
f"({head_dim}) = sum(axes_dims)"
|
||||
)
|
||||
if self.model_dim % 2 != 0:
|
||||
raise ValueError(f"model_dim must be even, got {self.model_dim}")
|
||||
if self.n_grid < 1:
|
||||
raise ValueError(f"n_grid must be >= 1, got {self.n_grid}")
|
||||
|
||||
def _validate_overrides(self) -> None:
|
||||
unknown = sorted(
|
||||
set(self.attribute_overrides) - set(ATTRIBUTE_OVERRIDE_WHITELIST)
|
||||
)
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
f"attribute_overrides keys {unknown} are not allowed; "
|
||||
f"supported keys: {list(ATTRIBUTE_OVERRIDE_WHITELIST)}"
|
||||
)
|
||||
instruct_type = self.effective_override("instruct_type")
|
||||
if instruct_type not in (None, *SUPPORTED_INSTRUCT_TYPES):
|
||||
raise ValueError(
|
||||
f"instruct_type must be one of {list(SUPPORTED_INSTRUCT_TYPES)}, "
|
||||
f"got {instruct_type!r}"
|
||||
)
|
||||
visual_cond = self.effective_override("visual_cond")
|
||||
if not isinstance(visual_cond, bool):
|
||||
raise ValueError(f"visual_cond must be a bool, got {visual_cond!r}")
|
||||
|
||||
def _validate_sampler_fields(self) -> None:
|
||||
present = [name for name in _PIFLOW_FIELDS if getattr(self, name) is not None]
|
||||
if present and len(present) != len(_PIFLOW_FIELDS):
|
||||
missing = sorted(set(_PIFLOW_FIELDS) - set(present))
|
||||
raise ValueError(
|
||||
f"pi-Flow config is incomplete: {missing} missing next to {present}"
|
||||
)
|
||||
if self.sr_lq_noise_type not in _LQ_NOISE_TYPES:
|
||||
raise ValueError(
|
||||
f"sr_lq_noise_type must be one of {list(_LQ_NOISE_TYPES)}, "
|
||||
f"got {self.sr_lq_noise_type!r}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6SRDitConfig(DiTConfig):
|
||||
arch_config: DiTArchConfig = field(default_factory=Kandinsky6SRArchConfig)
|
||||
prefix: str = "Kandinsky6SR"
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ATTRIBUTE_OVERRIDE_WHITELIST",
|
||||
"Kandinsky6SRArchConfig",
|
||||
"Kandinsky6SRDitConfig",
|
||||
"SUPPORTED_INSTRUCT_TYPES",
|
||||
"WIDE_INPUT_INSTRUCT_TYPES",
|
||||
]
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Reason1 text encoder, reusing the Qwen2.5-VL runtime without its vision tower.
|
||||
|
||||
Defaults describe the 3584-wide language trunk; checkpoint config overrides them."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from sglang.multimodal_gen.configs.models.encoders.base import (
|
||||
TextEncoderArchConfig,
|
||||
TextEncoderConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.fsdp import (
|
||||
is_embed_tokens,
|
||||
is_final_norm,
|
||||
is_layer,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Reason1ArchConfig(TextEncoderArchConfig):
|
||||
"""Architecture metadata (defaults match Qwen2.5-VL-7B-Instruct)."""
|
||||
|
||||
architectures: list[str] = field(
|
||||
default_factory=lambda: ["Qwen2_5_VLForConditionalGeneration"]
|
||||
)
|
||||
|
||||
vocab_size: int = 152064
|
||||
hidden_size: int = 3584
|
||||
intermediate_size: int = 18944
|
||||
num_hidden_layers: int = 28
|
||||
num_attention_heads: int = 28
|
||||
num_key_value_heads: int = 4
|
||||
hidden_act: str = "silu"
|
||||
max_position_embeddings: int = 128000
|
||||
initializer_range: float = 0.02
|
||||
rms_norm_eps: float = 1e-6
|
||||
use_cache: bool = False
|
||||
tie_word_embeddings: bool = False
|
||||
attention_dropout: float = 0.0
|
||||
|
||||
rope_theta: float = 1_000_000.0
|
||||
# 3D mRoPE (temporal/height/width) section split; Qwen2.5-VL-specific.
|
||||
rope_scaling: dict[str, Any] | None = field(
|
||||
default_factory=lambda: {"type": "mrope", "mrope_section": [16, 24, 24]}
|
||||
)
|
||||
|
||||
# Kandinsky6's text-conditioning stack always consumes the last hidden
|
||||
# state (see kandinsky6_qwen_postprocess_text in
|
||||
# configs/pipeline_configs/kandinsky6.py), so hidden states must be kept.
|
||||
output_hidden_states: bool = True
|
||||
hidden_state_skip_layer: int = 0
|
||||
|
||||
# text_len caps user tokens; tokenizer length also includes the 129-token template
|
||||
text_len: int = 512
|
||||
|
||||
bos_token_id: int = 151643
|
||||
pad_token_id: int = 151643
|
||||
eos_token_id: int = 151645
|
||||
vision_start_token_id: int = 151652
|
||||
vision_end_token_id: int = 151653
|
||||
vision_token_id: int = 151654
|
||||
image_token_id: int = 151655
|
||||
video_token_id: int = 151656
|
||||
|
||||
# Same fused-projection naming as the sglang qwen2_5vl/qwen_image runtime
|
||||
# encoders (MergedColumnParallelLinear "gate_up_proj", fused "qkv_proj").
|
||||
stacked_params_mapping: list[tuple[str, str, str]] = field(
|
||||
default_factory=lambda: [
|
||||
(".qkv_proj", ".q_proj", "q"),
|
||||
(".qkv_proj", ".k_proj", "k"),
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0), # type: ignore
|
||||
(".gate_up_proj", ".up_proj", 1), # type: ignore
|
||||
]
|
||||
)
|
||||
_fsdp_shard_conditions: list = field(
|
||||
default_factory=lambda: [is_layer, is_embed_tokens, is_final_norm]
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Reason1Config(TextEncoderConfig):
|
||||
arch_config: TextEncoderArchConfig = field(default_factory=Reason1ArchConfig)
|
||||
prefix: str = "reason1"
|
||||
|
||||
|
||||
__all__ = ["Reason1ArchConfig", "Reason1Config"]
|
||||
@@ -0,0 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from sglang.multimodal_gen.configs.models.upsamplers.kandinsky6_sr import (
|
||||
Kandinsky6SRLatentUpscalerEntryConfig,
|
||||
)
|
||||
|
||||
__all__ = ["Kandinsky6SRLatentUpscalerEntryConfig"]
|
||||
@@ -0,0 +1,225 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Configuration of an entry in the SR latent-upscaler bank.
|
||||
|
||||
Only released architectures are implemented; reject unsupported settings."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
|
||||
# key -> (required value, value a config that omits the key stands for). The omitted-key values
|
||||
# are the training code's defaults, so e.g. a config without ``input_skip`` means
|
||||
# ``input_skip=True`` and is rejected.
|
||||
_FIXED_KEYS: dict[str, tuple[Any, Any]] = {
|
||||
"upscale_factor": (4, 4),
|
||||
"dims": (3, 2),
|
||||
"temporal_padding": ("replicate", "zeros"),
|
||||
"upsample_mode": ("pxs_v2", "pixel_shuffle"),
|
||||
"upsample_padding_mode": ("zeros", "reflect"),
|
||||
"modulated_norm": (True, False),
|
||||
"modulated_output_proj": (True, False),
|
||||
"bare_stem": (True, False),
|
||||
"input_skip": (False, True),
|
||||
"global_skip": (False, False),
|
||||
"grn": (False, False),
|
||||
"layer_scale_init": (None, None),
|
||||
"depthwise": (False, False),
|
||||
"bottleneck_channels": (None, None),
|
||||
"motion_attention": (None, None),
|
||||
# Only read by depthwise blocks; the residual convs are always 3x3x3.
|
||||
"kernel_size": (3, 3),
|
||||
}
|
||||
# Fixed for entries built with the x2 entry (``enable_x2_entry=true``).
|
||||
_X2_FIXED_KEYS: dict[str, tuple[Any, Any]] = {
|
||||
"x2_tail_mode": ("private_full", "shared"),
|
||||
"x2_finisher": ("pxs_residual", "none"),
|
||||
}
|
||||
# Accepted without effect at inference: training-time settings, and switches that only apply to
|
||||
# the pixel-shuffle upsample modes (the pxs_v2 upsample has no temporal kernel and no ICNR init).
|
||||
_IGNORED_KEYS = frozenset(
|
||||
{
|
||||
"gradient_checkpointing",
|
||||
"stochastic_depth_rate",
|
||||
"loss_weight_2x",
|
||||
"loss_weight_4x",
|
||||
"temporal_mix",
|
||||
"icnr",
|
||||
"upsample_position",
|
||||
}
|
||||
)
|
||||
_REQUIRED_INT_KEYS = (
|
||||
"in_channels",
|
||||
"hidden_channels",
|
||||
"num_pre_blocks",
|
||||
"num_mid_blocks",
|
||||
"num_post_blocks",
|
||||
"expand_ratio",
|
||||
)
|
||||
_SUPPORTED_SCALES = (2, 4)
|
||||
|
||||
|
||||
def _positive_int(where: str, key: str, value: Any) -> int:
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < 1:
|
||||
raise ValueError(f"{where}: `{key}` must be a positive integer, got {value!r}")
|
||||
return value
|
||||
|
||||
|
||||
def _parse_target_scale(where: str, value: Any) -> int:
|
||||
text = str(value).removesuffix("x")
|
||||
if (
|
||||
isinstance(value, bool)
|
||||
or not text.isdigit()
|
||||
or int(text) not in _SUPPORTED_SCALES
|
||||
):
|
||||
raise ValueError(
|
||||
f"{where}: `target_scale` must be one of '2x', '4x', got {value!r}"
|
||||
)
|
||||
return int(text)
|
||||
|
||||
|
||||
class Kandinsky6SRLatentUpscalerEntryConfig(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Cascaded 2x+2x upsampler, optionally with a private x2 entry path."""
|
||||
|
||||
target_scale: int
|
||||
in_channels: int
|
||||
hidden_channels: int
|
||||
stage_channels: tuple[int, int, int]
|
||||
num_pre_blocks: int
|
||||
num_mid_blocks: int
|
||||
num_post_blocks: int
|
||||
expand_ratio: int
|
||||
enable_x2_entry: bool = False
|
||||
x2_adapter_blocks: int = 0
|
||||
|
||||
@classmethod
|
||||
def from_dict(
|
||||
cls, spec: Any, index: int = 0
|
||||
) -> Kandinsky6SRLatentUpscalerEntryConfig:
|
||||
"""Validate one ``models[index]`` entry of ``latent_upscaler/config.json``."""
|
||||
where = f"latent upscaler models[{index}]"
|
||||
if (
|
||||
not isinstance(spec, Mapping)
|
||||
or set(spec) != {"target_scale", "model"}
|
||||
or not isinstance(spec["model"], Mapping)
|
||||
):
|
||||
raise ValueError(
|
||||
f"{where} must be a mapping with exactly `target_scale` and a `model` "
|
||||
f"mapping, got {spec!r}"
|
||||
)
|
||||
target_scale = _parse_target_scale(where, spec["target_scale"])
|
||||
model = dict(spec["model"])
|
||||
|
||||
if model.get("architecture") != "multi_scale":
|
||||
raise ValueError(
|
||||
f"{where}: `architecture` must be 'multi_scale', got "
|
||||
f"{model.get('architecture')!r}"
|
||||
)
|
||||
enable_x2_entry = model.get("enable_x2_entry", False)
|
||||
if not isinstance(enable_x2_entry, bool):
|
||||
raise ValueError(
|
||||
f"{where}: `enable_x2_entry` must be a boolean, got {enable_x2_entry!r}"
|
||||
)
|
||||
|
||||
fixed = dict(_FIXED_KEYS)
|
||||
if enable_x2_entry:
|
||||
fixed.update(_X2_FIXED_KEYS)
|
||||
for key, (required, omitted) in fixed.items():
|
||||
value = model.get(key, omitted)
|
||||
if value != required or type(value) is not type(required):
|
||||
raise ValueError(
|
||||
f"{where}: unsupported `{key}`={value!r}"
|
||||
f"{' (omitted)' if key not in model else ''}; only {required!r} is "
|
||||
"implemented"
|
||||
)
|
||||
|
||||
x2_defaults = {
|
||||
"x2_adapter_blocks": 0,
|
||||
"x2_adapter_sources": None,
|
||||
"x2_tail_mode": "shared",
|
||||
"x2_finisher": "none",
|
||||
}
|
||||
known = {
|
||||
"architecture",
|
||||
"enable_x2_entry",
|
||||
"stage_channels",
|
||||
*_REQUIRED_INT_KEYS,
|
||||
*fixed,
|
||||
*x2_defaults,
|
||||
*_IGNORED_KEYS,
|
||||
}
|
||||
unknown = sorted(set(model) - known)
|
||||
if unknown:
|
||||
raise ValueError(f"{where}: unknown keys {unknown}")
|
||||
if not enable_x2_entry:
|
||||
present = sorted(
|
||||
key
|
||||
for key, default in x2_defaults.items()
|
||||
if model.get(key, default) != default
|
||||
)
|
||||
if present:
|
||||
raise ValueError(f"{where}: {present} require `enable_x2_entry`=true")
|
||||
if target_scale == 2 and not enable_x2_entry:
|
||||
raise ValueError(
|
||||
f"{where}: `target_scale`='2x' requires `enable_x2_entry`=true"
|
||||
)
|
||||
|
||||
missing = [key for key in _REQUIRED_INT_KEYS if key not in model]
|
||||
if missing:
|
||||
raise ValueError(f"{where}: missing keys {missing}")
|
||||
ints = {
|
||||
key: _positive_int(where, key, model[key]) for key in _REQUIRED_INT_KEYS
|
||||
}
|
||||
hidden = ints["hidden_channels"]
|
||||
|
||||
raw_stages = model.get("stage_channels")
|
||||
if raw_stages is None:
|
||||
stage_channels = (hidden, hidden, hidden)
|
||||
else:
|
||||
if not isinstance(raw_stages, (list, tuple)) or len(raw_stages) != 3:
|
||||
raise ValueError(
|
||||
f"{where}: `stage_channels` must list 3 widths, got {raw_stages!r}"
|
||||
)
|
||||
first, second, third = (
|
||||
_positive_int(where, "stage_channels", width) for width in raw_stages
|
||||
)
|
||||
stage_channels = (first, second, third)
|
||||
if stage_channels[0] != hidden:
|
||||
raise ValueError(
|
||||
f"{where}: `hidden_channels` ({hidden}) must equal `stage_channels[0]` "
|
||||
f"({stage_channels[0]})"
|
||||
)
|
||||
|
||||
x2_adapter_blocks = 0
|
||||
if enable_x2_entry:
|
||||
x2_adapter_blocks = _positive_int(
|
||||
where, "x2_adapter_blocks", model.get("x2_adapter_blocks", 0)
|
||||
)
|
||||
# Training warm-starts the adapter from these pre_blocks; only their consistency
|
||||
# matters here.
|
||||
sources = model.get("x2_adapter_sources")
|
||||
if sources is not None and (
|
||||
not isinstance(sources, (list, tuple))
|
||||
or len(sources) != x2_adapter_blocks
|
||||
or any(
|
||||
not isinstance(i, int) or not 0 <= i < ints["num_pre_blocks"]
|
||||
for i in sources
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
f"{where}: `x2_adapter_sources` must list {x2_adapter_blocks} indices in "
|
||||
f"[0, num_pre_blocks={ints['num_pre_blocks']}), got {sources!r}"
|
||||
)
|
||||
|
||||
return cls(
|
||||
target_scale=target_scale,
|
||||
stage_channels=stage_channels,
|
||||
enable_x2_entry=enable_x2_entry,
|
||||
x2_adapter_blocks=x2_adapter_blocks,
|
||||
**ints,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["Kandinsky6SRLatentUpscalerEntryConfig"]
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Checkpoint configuration for the bundled MMAudio codec and BigVGAN vocoder."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6AudioVAEArchConfig(VAEArchConfig):
|
||||
mode: str = "44k"
|
||||
# The two flags below are not read by the runtime module: the official
|
||||
# checkpoints carry the encoder half's weights, so ``Kandinsky6AudioVAE``
|
||||
# always builds both halves of the nested MMAudioVAE.
|
||||
need_vae_decoder: bool = True
|
||||
need_vae_encoder: bool = False
|
||||
scaling_factor: float = 0.5302
|
||||
# Reverses a checkpoint's own audio-latent shift, matching the diffusers
|
||||
# reference's postprocess_audio (getattr(audio_vae, "mean_value", 0.0));
|
||||
# no real checkpoint has been observed setting this to anything but 0.0
|
||||
# yet.
|
||||
mean_value: float = 0.0
|
||||
# BigVGAN-v2 hyperparameters, passed straight through to the runtime
|
||||
# BigVGANV2(...) constructor unmodified.
|
||||
vocoder_config: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6AudioVAEConfig(VAEConfig):
|
||||
arch_config: VAEArchConfig = field(default_factory=Kandinsky6AudioVAEArchConfig)
|
||||
# Generation only decodes audio; the runtime module does not read these.
|
||||
load_encoder: bool = False
|
||||
load_decoder: bool = True
|
||||
|
||||
|
||||
__all__ = ["Kandinsky6AudioVAEArchConfig", "Kandinsky6AudioVAEConfig"]
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""SR causal KVAE config supporting nested and flat checkpoint layouts.
|
||||
|
||||
Normalize flat fields into encoder_config/decoder_config; reject unknown fields."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.base import VAEArchConfig, VAEConfig
|
||||
|
||||
_TOLERATED_EXTRA_KEYS = frozenset({"_class_name", "_diffusers_version"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6SRVAEArchConfig(VAEArchConfig):
|
||||
vae_type: str = "video-kvae"
|
||||
encoder_config: dict[str, Any] = field(default_factory=dict)
|
||||
decoder_config: dict[str, Any] = field(default_factory=dict)
|
||||
scaling_factor: float = 1.0
|
||||
spatial_factor: int = 16
|
||||
temporal_factor: int = 4
|
||||
|
||||
# Flat top-level KVAE architecture fields (current official vae/config.json layout -- see
|
||||
# module docstring). Left at their None default (and ignored) when the checkpoint instead
|
||||
# uses the older nested encoder_config/decoder_config layout.
|
||||
in_channels: int | None = None
|
||||
out_channels: int | None = None
|
||||
z_channels: int | None = None
|
||||
ch: int | None = None
|
||||
ch_mult: list[float] | None = None
|
||||
decoder_ch: int | None = None
|
||||
decoder_ch_mult: list[float] | None = None
|
||||
num_res_blocks: int | None = None
|
||||
padding_mode: str | None = None
|
||||
temporal_compress_times: int | None = None
|
||||
temporal_compress_start_level: int | None = None
|
||||
norm_type: str | None = None
|
||||
double_z: bool | None = None
|
||||
downsample_version: int | None = None
|
||||
fix_pxs: bool | None = None
|
||||
# Present in the flat layout but unused by Kandinsky6SRVAE (the KVAE is resolution-agnostic);
|
||||
# declared only so it doesn't trip the unknown-top-level-key check below.
|
||||
resolution: int | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
unknown = sorted(set(self.extra_attrs) - _TOLERATED_EXTRA_KEYS)
|
||||
if unknown:
|
||||
raise ValueError(f"Kandinsky6SR VAE config has unknown keys {unknown}")
|
||||
if self.vae_type != "video-kvae":
|
||||
raise ValueError(
|
||||
f"Kandinsky6SRVAE supports only 'video-kvae', got {self.vae_type!r}"
|
||||
)
|
||||
self.spatial_compression_ratio = self.spatial_factor
|
||||
self.temporal_compression_ratio = self.temporal_factor
|
||||
|
||||
if not self.encoder_config and not self.decoder_config and self.ch is not None:
|
||||
shared = dict(
|
||||
num_res_blocks=self.num_res_blocks,
|
||||
z_channels=self.z_channels,
|
||||
temporal_compress_times=self.temporal_compress_times,
|
||||
temporal_compress_start_level=self.temporal_compress_start_level,
|
||||
norm_type=self.norm_type,
|
||||
padding_mode=self.padding_mode,
|
||||
)
|
||||
self.encoder_config = {
|
||||
**shared,
|
||||
"ch": self.ch,
|
||||
"ch_mult": self.ch_mult,
|
||||
"in_channels": self.in_channels,
|
||||
"double_z": self.double_z,
|
||||
"downsample_version": self.downsample_version,
|
||||
"fix_pxs": self.fix_pxs,
|
||||
}
|
||||
self.decoder_config = {
|
||||
**shared,
|
||||
"ch": self.decoder_ch,
|
||||
"ch_mult": self.decoder_ch_mult,
|
||||
"out_ch": self.out_channels,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6SRVAEConfig(VAEConfig):
|
||||
arch_config: VAEArchConfig = field(default_factory=Kandinsky6SRVAEArchConfig)
|
||||
# SR stages own tile geometry; parallel tiling distributes these existing tiles
|
||||
# without enabling the generic VAE tiling or spatial-shard decode paths
|
||||
use_tiling: bool = False
|
||||
use_temporal_tiling: bool = False
|
||||
use_parallel_tiling: bool = False
|
||||
use_parallel_decode: bool = False
|
||||
|
||||
def get_vae_scale_factor(self) -> int:
|
||||
return self.arch_config.spatial_factor
|
||||
|
||||
|
||||
__all__ = ["Kandinsky6SRVAEArchConfig", "Kandinsky6SRVAEConfig"]
|
||||
@@ -0,0 +1,248 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Kandinsky 6 text-to-video/audio pipeline with optional image conditioning."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models import EncoderConfig
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
|
||||
Kandinsky6VideoAudioConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.encoders import (
|
||||
BaseEncoderOutput,
|
||||
CLIPTextConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.encoders.kandinsky6_reason1 import (
|
||||
Reason1Config,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.hunyuanvae import HunyuanVAEConfig
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
|
||||
Kandinsky6AudioVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ModelTaskType,
|
||||
PipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
_is_kandinsky6_sr,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.qwen_image import (
|
||||
qwen_image_postprocess_text,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy
|
||||
|
||||
# checkpoint-trained template: preserve its exact bytes, including misspellings
|
||||
KANDINSKY6_PROMPT_TEMPLATE = "\n".join(
|
||||
[
|
||||
"<|im_start|>system\nYou are a promt engineer. Describe the video in detail.", # codespell:ignore promt
|
||||
"Describe how the camera moves or shakes, describe the zoom and view angle, whether it follows the objects.",
|
||||
"Describe the location of the video, main characters or objects and their action.",
|
||||
"Describe the dynamism of the video and presented actions.",
|
||||
"Name the visual style of the video: whether it is a professional footage, user generated content, some kind of animation, video game or scren content.", # codespell:ignore scren
|
||||
"Describe the visual effects, postprocessing and transitions if they are presented in the video.",
|
||||
"Pay attention to the order of key actions shown in the scene.<|im_end|>",
|
||||
"<|im_start|>user\n{}<|im_end|>",
|
||||
]
|
||||
)
|
||||
# Reason1 template prefix length before user tokens
|
||||
KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX = 129
|
||||
|
||||
|
||||
def kandinsky6_qwen_preprocess_text(prompt: str) -> str:
|
||||
if not prompt.strip():
|
||||
prompt = "."
|
||||
return KANDINSKY6_PROMPT_TEMPLATE.format(prompt)
|
||||
|
||||
|
||||
def kandinsky6_qwen_postprocess_text(
|
||||
outputs: BaseEncoderOutput,
|
||||
text_inputs,
|
||||
return_attention_mask: bool = False,
|
||||
):
|
||||
"""Strip padding and the Reason1 template prefix, then repad embeddings and mask."""
|
||||
return qwen_image_postprocess_text(
|
||||
outputs,
|
||||
text_inputs,
|
||||
drop_idx=KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX,
|
||||
return_attention_mask=return_attention_mask,
|
||||
)
|
||||
|
||||
|
||||
def kandinsky6_clip_postprocess_text(
|
||||
outputs: BaseEncoderOutput, _text_inputs
|
||||
) -> torch.Tensor:
|
||||
return outputs.pooler_output
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6TI2VAPipelineConfig(PipelineConfig):
|
||||
"""Configuration for the Kandinsky6 TI2VA pipeline."""
|
||||
|
||||
task_type: ModelTaskType = ModelTaskType.TI2V
|
||||
native_only_components: tuple[str, ...] = (
|
||||
"transformer",
|
||||
"text_encoder",
|
||||
"text_encoder_2",
|
||||
"vae",
|
||||
"audio_vae",
|
||||
)
|
||||
|
||||
# image conditioning is handled by Kandinsky6ImageEncodingStage, not Wan TI2V preprocessing
|
||||
skip_input_image_preprocess: bool = True
|
||||
|
||||
# a single joint DiT consumes video and audio latents
|
||||
dit_config: Kandinsky6VideoAudioConfig = field(
|
||||
default_factory=Kandinsky6VideoAudioConfig
|
||||
)
|
||||
dit_precision: str = "bf16"
|
||||
cfg_policy: CFGPolicy = field(
|
||||
default_factory=lambda: CFGPolicy(parallel_uses_serial_arithmetic=True)
|
||||
)
|
||||
|
||||
# keep tiled decode: whole-clip sharding materializes a quadratic temporal mask
|
||||
vae_config: HunyuanVAEConfig = field(
|
||||
default_factory=lambda: HunyuanVAEConfig(parallel_decode_mode="tiled")
|
||||
)
|
||||
vae_precision: str = "bf16"
|
||||
vae_tiling: bool = True
|
||||
|
||||
# audio_vae includes both the mel codec and vocoder
|
||||
audio_vae_config: Kandinsky6AudioVAEConfig = field(
|
||||
default_factory=Kandinsky6AudioVAEConfig
|
||||
)
|
||||
audio_vae_precision: str = "bf16"
|
||||
|
||||
# Reason1 provides token embeddings; CLIP provides pooled text conditioning
|
||||
text_encoder_configs: tuple[EncoderConfig, ...] = field(
|
||||
default_factory=lambda: (Reason1Config(), CLIPTextConfig())
|
||||
)
|
||||
text_encoder_precisions: tuple[str, ...] = field(
|
||||
default_factory=lambda: ("bf16", "bf16")
|
||||
)
|
||||
preprocess_text_funcs: tuple[Callable[[str], str] | None, ...] = field(
|
||||
default_factory=lambda: (kandinsky6_qwen_preprocess_text, None)
|
||||
)
|
||||
postprocess_text_funcs: tuple[Callable[..., Any], ...] = field(
|
||||
default_factory=lambda: (
|
||||
kandinsky6_qwen_postprocess_text,
|
||||
kandinsky6_clip_postprocess_text,
|
||||
)
|
||||
)
|
||||
# Reason1: 129 template + 512 user tokens; CLIP: fixed 77-token context
|
||||
text_encoder_extra_args: list[dict] = field(
|
||||
default_factory=lambda: [
|
||||
dict(
|
||||
max_length=KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX + 512,
|
||||
padding=True,
|
||||
truncation=True,
|
||||
),
|
||||
dict(
|
||||
max_length=77,
|
||||
padding="max_length",
|
||||
truncation=True,
|
||||
add_special_tokens=True,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# Shift of both official checkpoints' schedulers (FlowMatchEulerDiscreteScheduler
|
||||
# for Pro-sft, PiflowScheduler for Pro-distill: scheduler_config.json "shift").
|
||||
flow_shift: float | None = 5.0
|
||||
|
||||
# audio frames = ceil(pixel_frames / fps * sample_rate / downsample_factor)
|
||||
sample_fps: float = 24.0
|
||||
audio_sample_rate: int = 44100
|
||||
audio_downsample_factor: int = 1024
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if len(self.text_encoder_configs) != 2:
|
||||
raise ValueError(
|
||||
"Kandinsky6 pipeline requires exactly 2 text encoders "
|
||||
f"(Reason1 and CLIP), but got {len(self.text_encoder_configs)}."
|
||||
)
|
||||
if len(self.text_encoder_precisions) != 2:
|
||||
raise ValueError(
|
||||
"Kandinsky6 pipeline requires exactly 2 text encoder "
|
||||
f"precisions, but got {len(self.text_encoder_precisions)}."
|
||||
)
|
||||
if len(self.text_encoder_extra_args) != 2:
|
||||
raise ValueError(
|
||||
"Kandinsky6 pipeline requires exactly 2 text encoder extra "
|
||||
f"tokenizer arg dicts, but got {len(self.text_encoder_extra_args)}."
|
||||
)
|
||||
|
||||
# keep the video VAE encoder available for optional image conditioning
|
||||
self.vae_config.load_encoder = True
|
||||
self.vae_config.load_decoder = True
|
||||
|
||||
def supports_disaggregation(self) -> bool:
|
||||
# joint denoising currently runs monolithically
|
||||
return False
|
||||
|
||||
def get_text_encoder_pooler_output(self, outputs, encoder_index):
|
||||
# only CLIP (index 1) returns pooler_output; Reason1 returns causal-LM output
|
||||
if encoder_index == 1:
|
||||
return outputs.pooler_output
|
||||
return None
|
||||
|
||||
def get_pos_prompt_embeds(self, batch):
|
||||
return batch.prompt_embeds[0]
|
||||
|
||||
def get_neg_prompt_embeds(self, batch):
|
||||
return batch.negative_prompt_embeds[0]
|
||||
|
||||
def tokenize_prompt(self, prompt, tokenizer, tok_kwargs) -> dict:
|
||||
# Reason1 uses a multimodal processor: positional input would bind to images,
|
||||
# not text. The conditioning image is handled separately by the VAE.
|
||||
return tokenizer(text=prompt, **tok_kwargs)
|
||||
|
||||
|
||||
def _is_kandinsky6_t2va_family(model_id: str) -> bool:
|
||||
normalized = model_id.lower().replace("-", "").replace("_", "")
|
||||
return "kandinsky6" in normalized and not _is_kandinsky6_sr(model_id)
|
||||
|
||||
|
||||
def _is_kandinsky6_t2va_distilled(model_id: str) -> bool:
|
||||
short_name = model_id.lower().rstrip("/").split("/")[-1]
|
||||
return _is_kandinsky6_t2va_family(model_id) and "distill" in short_name
|
||||
|
||||
|
||||
def _is_kandinsky6_t2va(model_id: str) -> bool:
|
||||
short_name = model_id.lower().rstrip("/").split("/")[-1]
|
||||
return _is_kandinsky6_t2va_family(model_id) and "distill" not in short_name
|
||||
|
||||
|
||||
def register():
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6 import (
|
||||
Kandinsky6TI2VADistilledSamplingParams,
|
||||
Kandinsky6TI2VASamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.registry import register_configs
|
||||
|
||||
register_configs(
|
||||
sampling_param_cls=Kandinsky6TI2VADistilledSamplingParams,
|
||||
pipeline_config_cls=Kandinsky6TI2VAPipelineConfig,
|
||||
hf_model_paths=["kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers"],
|
||||
model_detectors=[_is_kandinsky6_t2va_distilled],
|
||||
)
|
||||
register_configs(
|
||||
sampling_param_cls=Kandinsky6TI2VASamplingParams,
|
||||
pipeline_config_cls=Kandinsky6TI2VAPipelineConfig,
|
||||
hf_model_paths=[
|
||||
"kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers",
|
||||
"kandinskylab/Kandinsky-6.0-Pro-sft-5s-Diffusers",
|
||||
],
|
||||
model_detectors=[_is_kandinsky6_t2va],
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KANDINSKY6_PROMPT_TEMPLATE",
|
||||
"KANDINSKY6_PROMPT_TEMPLATE_ENCODE_START_IDX",
|
||||
"Kandinsky6TI2VAPipelineConfig",
|
||||
"kandinsky6_clip_postprocess_text",
|
||||
"kandinsky6_qwen_postprocess_text",
|
||||
"kandinsky6_qwen_preprocess_text",
|
||||
]
|
||||
@@ -0,0 +1,76 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Video-to-video SR without text/audio models; source audio is preserved.
|
||||
|
||||
TI2V supplies the API task type, but model-specific stages handle video_path."""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
Kandinsky6SRVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
ModelTaskType,
|
||||
PipelineConfig,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6SRPipelineConfig(PipelineConfig):
|
||||
"""Configuration for ``Kandinsky6SRPipeline``."""
|
||||
|
||||
task_type: ModelTaskType = ModelTaskType.TI2V
|
||||
skip_input_image_preprocess: bool = True
|
||||
|
||||
# A missing / unexpected weight, a bad config or an unsupported flag must fail the
|
||||
# load loudly: the loaders only load strictly and re-raise their errors (instead of
|
||||
# falling back to a diffusers ``AutoModel``) for these components.
|
||||
native_only_components: tuple[str, ...] = ("transformer", "vae", "latent_upscaler")
|
||||
|
||||
dit_config: Kandinsky6SRDitConfig = field(default_factory=Kandinsky6SRDitConfig)
|
||||
dit_precision: str = "bf16"
|
||||
vae_config: Kandinsky6SRVAEConfig = field(default_factory=Kandinsky6SRVAEConfig)
|
||||
vae_precision: str = "bf16"
|
||||
vae_tiling: bool = False
|
||||
vae_sp: bool = False
|
||||
|
||||
# Text-free: no text encoders (all four per-encoder tuples must stay equally long).
|
||||
text_encoder_configs: tuple = ()
|
||||
text_encoder_precisions: tuple = ()
|
||||
preprocess_text_funcs: tuple = ()
|
||||
postprocess_text_funcs: tuple = ()
|
||||
|
||||
should_use_guidance: bool = False
|
||||
enable_autocast: bool = False
|
||||
|
||||
def supports_disaggregation(self) -> bool:
|
||||
# The tiled loop runs in one process over shared tensors.
|
||||
return False
|
||||
|
||||
|
||||
__all__ = ["Kandinsky6SRPipelineConfig"]
|
||||
|
||||
|
||||
def _is_kandinsky6_sr(model_id: str) -> bool:
|
||||
normalized = model_id.lower().replace("-", "").replace("_", "")
|
||||
return re.search(r"kandinsky6(?:\.\d+)?(?:vsr|sr|superres)", normalized) is not None
|
||||
|
||||
|
||||
def register():
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.registry import register_configs
|
||||
|
||||
register_configs(
|
||||
sampling_param_cls=Kandinsky6SRSamplingParams,
|
||||
pipeline_config_cls=Kandinsky6SRPipelineConfig,
|
||||
hf_model_paths=[
|
||||
"kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers",
|
||||
"kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers",
|
||||
],
|
||||
model_detectors=[_is_kandinsky6_sr],
|
||||
)
|
||||
@@ -0,0 +1,65 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6TI2VASamplingParams(SamplingParams):
|
||||
"""Sampling params for Kandinsky6 TI2VA (text[+image] -> video+audio).
|
||||
|
||||
Defaults of the Diffusers ``Kandinsky6TI2VAPipeline`` (the undistilled
|
||||
``Kandinsky-6.0-Pro-sft-5s-Diffusers`` checkpoint): 512x768 @ 24fps, 121
|
||||
frames ((121 - 1) // 4 + 1 == 31 latent frames at the video VAE's
|
||||
temporal_compression_ratio=4), guidance_scale=5.0, 50 inference steps.
|
||||
"""
|
||||
|
||||
# Video parameters (Kandinsky6 TI2VA default preset)
|
||||
height: int = 512
|
||||
width: int = 768
|
||||
num_frames: int = 121
|
||||
fps: int = 24
|
||||
|
||||
# SamplingParams._adjust's multi-GPU frame-count rounding
|
||||
# (`orig_latent_num_frames` up to a multiple of `num_gpus`) is a second,
|
||||
# independent rounding rule on top of Kandinsky6LatentPreparationStage's
|
||||
# own `num_frames % 4 == 1` rounding (the diffusers reference's own
|
||||
# `latent_frames = (num_frames - 1) // 4 + 1` convention -- the single
|
||||
# source of truth for this model). The two compose for num_gpus > 1 and
|
||||
# land on a decoded frame count neither rule intended (e.g. the
|
||||
# documented default num_frames=121 becomes 125/129/125/125 decoded
|
||||
# frames at 2/3/4/8 GPUs instead of Diffusers' 121). Disabling it here
|
||||
# leaves Kandinsky6LatentPreparationStage's rounding as the only one
|
||||
# applied, regardless of num_gpus -- matching this model's precedent for
|
||||
# opting out of this same mechanism when it owns frame-count alignment
|
||||
# itself (see `enable_sequence_shard=True`, which disables it for the
|
||||
# same reason a few lines up in SamplingParams._adjust).
|
||||
adjust_frames: bool = False
|
||||
|
||||
# Denoising stage
|
||||
guidance_scale: float = 5.0
|
||||
num_inference_steps: int = 50
|
||||
|
||||
negative_prompt: str = (
|
||||
"Static, 2D cartoon, cartoon, 2d animation, paintings, images, worst "
|
||||
"quality, low quality, ugly, deformed, walking backwards"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6TI2VADistilledSamplingParams(Kandinsky6TI2VASamplingParams):
|
||||
"""Sampling params for the pi-Flow distilled Kandinsky6 TI2VA checkpoint
|
||||
(``Kandinsky-6.0-Pro-distill-5s-Diffusers``).
|
||||
|
||||
Distilled for 10 steps (model card; its ``scheduler_config.json`` leaves
|
||||
``nfe`` unset): its ``PiflowScheduler`` runs one DiT call per step and has
|
||||
no classifier-free guidance, so any guidance_scale other than 1.0 is
|
||||
rejected by the denoising stage, and the negative prompt is never encoded
|
||||
either.
|
||||
"""
|
||||
|
||||
guidance_scale: float = 1.0
|
||||
num_inference_steps: int = 10
|
||||
|
||||
|
||||
__all__ = ["Kandinsky6TI2VADistilledSamplingParams", "Kandinsky6TI2VASamplingParams"]
|
||||
@@ -0,0 +1,124 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Sampling parameters of Kandinsky 6 video super-resolution (video -> video)."""
|
||||
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
|
||||
SUPPORTED_RESOLUTION_SCALES,
|
||||
resolve_target_hw,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
|
||||
SR_RESIZE_MODES = ("fit", "exact")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Kandinsky6SRSamplingParams(SamplingParams):
|
||||
"""SR request options; dimensions and fps follow video_path, prompt is ignored."""
|
||||
|
||||
# No prompt-driven generation: keep the guidance defaults inert.
|
||||
negative_prompt: str = ""
|
||||
guidance_scale: float = 1.0
|
||||
|
||||
# 2, 4 or 2.25 (= x1.125 pixel pre-upscale, then x2 tiles).
|
||||
sr_resolution_scale: float = 2.25
|
||||
# DiT calls per tile (the repo-wide convention; counts network calls, not timestep grid
|
||||
# points). Flow-matching bundles want 4; a pi-Flow bundle ignores this and runs its own
|
||||
# ``nfe`` instead (the denoising stage warns if this is set to something else).
|
||||
num_inference_steps: int = 4
|
||||
# Tiles denoised together (memory / speed trade-off).
|
||||
sr_tiles_batch_size: int = 1
|
||||
# Minimum tile overlap of the tile grid, as a fraction of the tile size.
|
||||
sr_tile_min_overlap: float = 0.20
|
||||
# Optional delivery resolution: "hd", "fullhd", "2k" or "WxH" (never upscales).
|
||||
sr_target_resolution: str | None = None
|
||||
# "fit" keeps the aspect ratio inside the target bucket; "exact" squeezes to it.
|
||||
sr_target_resize_mode: str = "fit"
|
||||
|
||||
# The SR DiT is text-free: a multipart request needs no ``prompt`` field.
|
||||
prompt_optional = True
|
||||
|
||||
@classmethod
|
||||
def video_request_extra_fields(cls) -> frozenset[str]:
|
||||
# num_inference_steps is already in the base schema; only sr_* needs model_extra
|
||||
return frozenset(
|
||||
{
|
||||
"sr_resolution_scale",
|
||||
"sr_tiles_batch_size",
|
||||
"sr_tile_min_overlap",
|
||||
"sr_target_resolution",
|
||||
"sr_target_resize_mode",
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def lower_video_request_kwargs(
|
||||
cls, request: Any, kwargs: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
"""Lift the ``sr_*`` extras (declared above, kept by the multipart / JSON
|
||||
request layer as pydantic ``model_extra``) into the kwargs this class is
|
||||
constructed from; the base class only forwards a fixed, generic field set."""
|
||||
extra = getattr(request, "model_extra", None) or {}
|
||||
for name in cls.video_request_extra_fields():
|
||||
if name in extra and extra[name] is not None:
|
||||
kwargs[name] = extra[name]
|
||||
return kwargs
|
||||
|
||||
def _validate(self):
|
||||
super()._validate()
|
||||
if self.sr_resolution_scale not in SUPPORTED_RESOLUTION_SCALES:
|
||||
raise ValueError(
|
||||
f"sr_resolution_scale must be one of {list(SUPPORTED_RESOLUTION_SCALES)}, "
|
||||
f"got {self.sr_resolution_scale!r}"
|
||||
)
|
||||
# num_inference_steps >= 1 is already enforced generically by the base class; the
|
||||
# denoising stage additionally rejects it there with a message naming the model, and
|
||||
# warns when a pi-Flow checkpoint is asked to run something other than its own nfe.
|
||||
if self.sr_tiles_batch_size < 1:
|
||||
raise ValueError(
|
||||
f"sr_tiles_batch_size must be >= 1, got {self.sr_tiles_batch_size}"
|
||||
)
|
||||
if not 0.0 <= self.sr_tile_min_overlap < 1.0:
|
||||
raise ValueError(
|
||||
f"sr_tile_min_overlap must be in [0, 1), got {self.sr_tile_min_overlap}"
|
||||
)
|
||||
if self.sr_target_resize_mode not in SR_RESIZE_MODES:
|
||||
raise ValueError(
|
||||
f"sr_target_resize_mode must be one of {list(SR_RESIZE_MODES)}, "
|
||||
f"got {self.sr_target_resize_mode!r}"
|
||||
)
|
||||
# "exact" mode only parses the spec: a malformed tier / WxH fails early.
|
||||
resolve_target_hw(self.sr_target_resolution, (1080, 1920), "exact")
|
||||
|
||||
def _validate_with_pipeline_config(self, pipeline_config):
|
||||
super()._validate_with_pipeline_config(pipeline_config)
|
||||
if self.source_video_path() is None:
|
||||
raise ValueError(
|
||||
"Kandinsky6 SR needs an input video: pass --video-path (a local "
|
||||
"mp4/mkv file)."
|
||||
)
|
||||
|
||||
def source_video_path(self) -> str | None:
|
||||
"""The single input video path (a one-element list is accepted)."""
|
||||
path = self.video_path
|
||||
if isinstance(path, list):
|
||||
if len(path) != 1:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 SR takes exactly one video, got {len(path)} paths"
|
||||
)
|
||||
path = path[0]
|
||||
return path
|
||||
|
||||
def _set_output_file_name(self):
|
||||
path = self.source_video_path() if self.video_path is not None else None
|
||||
if self.output_file_name is None and path:
|
||||
stem = os.path.splitext(os.path.basename(path))[0]
|
||||
stamp = time.strftime("%Y%m%d-%H%M%S")
|
||||
self.output_file_name = f"{stem}_sr{self.sr_resolution_scale:g}x_{stamp}"
|
||||
super()._set_output_file_name()
|
||||
|
||||
|
||||
__all__ = ["Kandinsky6SRSamplingParams"]
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""SR resolution validation shared by sampling parameters and output stages.
|
||||
|
||||
Keep this module independent of runtime to avoid the registry import cycle."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Literal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# from-video total scales (2.25 = x1.125 pixel pre-upscale + x2 tiling).
|
||||
SUPPORTED_RESOLUTION_SCALES: tuple[float, ...] = (2, 4, 2.25)
|
||||
|
||||
# Delivery tiers for the target resolution: (H, W) candidates per tier -- landscape 16:9,
|
||||
# square, portrait 9:16; the closest-aspect entry wins.
|
||||
TARGET_RESOLUTIONS: dict[str, tuple[tuple[int, int], ...]] = {
|
||||
"hd": ((720, 1280), (720, 720), (1280, 720)),
|
||||
"fullhd": ((1080, 1920), (1080, 1080), (1920, 1080)),
|
||||
"2k": ((1440, 2560), (1440, 1440), (2560, 1440)),
|
||||
}
|
||||
|
||||
TargetResizeMode = Literal["fit", "exact"]
|
||||
|
||||
# Log a warning when the SR and target aspect ratios differ by more than this (relative) --
|
||||
# a mild anisotropic squeeze is fine, a big one means the wrong tier for the route.
|
||||
ASPECT_MISMATCH_TOLERANCE = 0.02
|
||||
|
||||
|
||||
def fit_within(
|
||||
sr_hw: tuple[int, int], bucket_hw: tuple[int, int]
|
||||
) -> tuple[int, int] | None:
|
||||
"""Fit within the bucket without upscaling; preserve aspect and codec-even sizes."""
|
||||
scale = min(bucket_hw[0] / sr_hw[0], bucket_hw[1] / sr_hw[1])
|
||||
if scale >= 1.0:
|
||||
logger.warning(
|
||||
"SR result %sx%s already fits inside the %sx%s bucket -- nothing to "
|
||||
"downscale, keeping it as is.",
|
||||
sr_hw[1],
|
||||
sr_hw[0],
|
||||
bucket_hw[1],
|
||||
bucket_hw[0],
|
||||
)
|
||||
return None
|
||||
height, width = (
|
||||
min(bucket, round(side * scale / 2) * 2)
|
||||
for bucket, side in zip(bucket_hw, sr_hw, strict=True)
|
||||
)
|
||||
return height, width
|
||||
|
||||
|
||||
def resolve_target_hw(
|
||||
spec: str | None,
|
||||
sr_hw: tuple[int, int],
|
||||
mode: TargetResizeMode = "fit",
|
||||
validate_only: bool = False,
|
||||
) -> tuple[int, int] | None:
|
||||
"""Resolve a tier or WxH bucket to (H, W), or None for no resize.
|
||||
|
||||
fit preserves aspect; exact returns the bucket dimensions. validate_only
|
||||
checks syntax without needing an actual SR result."""
|
||||
if mode not in ("fit", "exact"):
|
||||
raise ValueError(
|
||||
f"sr_target_resize_mode must be 'fit' or 'exact', got {mode!r}"
|
||||
)
|
||||
if spec is None or spec.lower() == "none":
|
||||
return None
|
||||
key = spec.lower()
|
||||
if key in TARGET_RESOLUTIONS:
|
||||
source_ratio = sr_hw[1] / sr_hw[0]
|
||||
bucket = min(
|
||||
TARGET_RESOLUTIONS[key], key=lambda hw: abs(hw[1] / hw[0] - source_ratio)
|
||||
)
|
||||
else:
|
||||
parts = key.split("x")
|
||||
if len(parts) != 2 or not all(part.strip().isdigit() for part in parts):
|
||||
raise ValueError(
|
||||
f"sr_target_resolution must be one of {sorted(TARGET_RESOLUTIONS)} or "
|
||||
f"WxH (e.g. 1280x720), got {spec!r}"
|
||||
)
|
||||
width, height = (int(part) for part in parts)
|
||||
bucket = (height, width)
|
||||
if mode == "exact" or validate_only:
|
||||
return bucket
|
||||
return fit_within(sr_hw, bucket)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ASPECT_MISMATCH_TOLERANCE",
|
||||
"SUPPORTED_RESOLUTION_SCALES",
|
||||
"TARGET_RESOLUTIONS",
|
||||
"TargetResizeMode",
|
||||
"fit_within",
|
||||
"resolve_target_hw",
|
||||
]
|
||||
@@ -298,6 +298,10 @@ class SamplingParams:
|
||||
# clamped to it, so a generic 1-step warmup stays a valid request.
|
||||
min_num_inference_steps: ClassVar[int] = 1
|
||||
|
||||
# A prompt-free pipeline (e.g. a text-free SR / video-to-video DiT) sets this to
|
||||
# ``True`` so the multipart video-generation route does not require ``prompt``.
|
||||
prompt_optional: ClassVar[bool] = False
|
||||
|
||||
height: int | None = None
|
||||
width: int | None = None
|
||||
fps: int = 24
|
||||
@@ -1442,6 +1446,42 @@ class SamplingParams:
|
||||
"frames for the generated output."
|
||||
),
|
||||
)
|
||||
# Kandinsky6 video super-resolution (ignored by other models)
|
||||
add_argument(
|
||||
"--sr-resolution-scale",
|
||||
type=float,
|
||||
help="Kandinsky6 SR total upscale factor: 2, 4 or 2.25 (default: 2.25).",
|
||||
)
|
||||
add_argument(
|
||||
"--sr-num-steps",
|
||||
type=int,
|
||||
help="Kandinsky6 SR Euler grid points (>= 2, default: 5); ignored by "
|
||||
"pi-Flow checkpoints.",
|
||||
)
|
||||
add_argument(
|
||||
"--sr-tiles-batch-size",
|
||||
type=int,
|
||||
help="Kandinsky6 SR: number of tiles denoised together (default: 1).",
|
||||
)
|
||||
add_argument(
|
||||
"--sr-tile-min-overlap",
|
||||
type=float,
|
||||
help="Kandinsky6 SR: minimum tile overlap fraction in [0, 1) "
|
||||
"(default: 0.2).",
|
||||
)
|
||||
add_argument(
|
||||
"--sr-target-resolution",
|
||||
type=str,
|
||||
help="Kandinsky6 SR delivery resolution: hd, fullhd, 2k or WxH; the "
|
||||
"result is only ever downscaled to it (default: keep the SR size).",
|
||||
)
|
||||
add_argument(
|
||||
"--sr-target-resize-mode",
|
||||
type=str,
|
||||
choices=["fit", "exact"],
|
||||
help="Kandinsky6 SR: 'fit' keeps the aspect ratio inside the target "
|
||||
"resolution, 'exact' resizes to it exactly (default: fit).",
|
||||
)
|
||||
add_argument(
|
||||
"--action-mode",
|
||||
type=str,
|
||||
|
||||
@@ -105,15 +105,14 @@ def _patch_cache_dit_similarity():
|
||||
tp_sp_group = getattr(self, "_sglang_tp_sp_group", None)
|
||||
target_group = tp_sp_group or sp_group or tp_group
|
||||
|
||||
# Averaging over a one-rank group returns the input, so skip the
|
||||
# collective rather than pay for a round trip that cannot change it.
|
||||
# CFG/DP peers are independent; never fall back to the global group
|
||||
if target_group is None or dist.get_world_size(target_group) == 1:
|
||||
return _original_similarity(
|
||||
self,
|
||||
t1,
|
||||
t2,
|
||||
threshold=threshold,
|
||||
parallelized=parallelized,
|
||||
parallelized=False,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
@@ -436,7 +435,29 @@ def _build_custom_block_adapter(
|
||||
) -> Optional[BlockAdapter]:
|
||||
"""Build a manual BlockAdapter for a model absent from cache-dit's registry,
|
||||
or None if the class is unknown."""
|
||||
spec = _CUSTOM_BLOCK_ADAPTER_SPECS.get(transformer.__class__.__name__)
|
||||
# FSDP adds a runtime subclass; retain the native model's adapter contract.
|
||||
model_name = next(
|
||||
(
|
||||
cls.__name__
|
||||
for cls in type(transformer).__mro__
|
||||
if cls.__name__ == "Kandinsky6Transformer3DModel"
|
||||
or cls.__name__ in _CUSTOM_BLOCK_ADAPTER_SPECS
|
||||
),
|
||||
None,
|
||||
)
|
||||
if model_name == "Kandinsky6Transformer3DModel":
|
||||
# the second stream is evolving audio, not static text conditioning
|
||||
return BlockAdapter(
|
||||
transformer=transformer,
|
||||
blocks=transformer.visual_transformer_blocks,
|
||||
forward_pattern=(
|
||||
ForwardPattern.Pattern_0
|
||||
if transformer.config.is_multimodal
|
||||
else ForwardPattern.Pattern_2
|
||||
),
|
||||
has_separate_cfg=has_separate_cfg,
|
||||
)
|
||||
spec = _CUSTOM_BLOCK_ADAPTER_SPECS.get(model_name)
|
||||
if spec is None:
|
||||
return None
|
||||
blocks = getattr(transformer, spec.blocks_attr, None)
|
||||
@@ -608,7 +629,9 @@ def enable_cache_on_transformer(
|
||||
)
|
||||
|
||||
parallelism_config = _build_parallelism_config(sp_group, tp_group)
|
||||
if parallelism_config is not None:
|
||||
if parallelism_config is not None or (
|
||||
dist.is_initialized() and dist.get_world_size() > 1
|
||||
):
|
||||
_patch_cache_dit_similarity()
|
||||
|
||||
_mark_transformer_parallelized(transformer, parallelism_config, sp_group, tp_group)
|
||||
|
||||
@@ -128,10 +128,22 @@ class IpcA2AState:
|
||||
def init(self, group):
|
||||
import ctypes
|
||||
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.platforms.cuda import (
|
||||
device_id_to_physical_device_id,
|
||||
)
|
||||
|
||||
self.rank = dist.get_rank(group=group)
|
||||
self.group = group
|
||||
dev = torch.cuda.current_device()
|
||||
peer_dev = _peer_cuda_device(group, self.rank, dev)
|
||||
# Peer access alone also admits PCIe pairs, where this NVLink transport
|
||||
# can stall in its GPU-side flag wait. Query the same pair on both ranks.
|
||||
physical_devices = sorted(
|
||||
device_id_to_physical_device_id(d) for d in (dev, peer_dev)
|
||||
)
|
||||
if not current_platform.is_full_nvlink(physical_devices):
|
||||
raise _Unsupported("requires an NVLink-connected GPU pair")
|
||||
try:
|
||||
has_peer_access = torch.cuda.can_device_access_peer(dev, peer_dev)
|
||||
except RuntimeError as e:
|
||||
|
||||
@@ -398,7 +398,11 @@ class DiffGenerator:
|
||||
save_outputs(
|
||||
output_batch.output,
|
||||
requests[0].data_type,
|
||||
requests[0].fps,
|
||||
(
|
||||
output_batch.fps
|
||||
if output_batch.fps is not None
|
||||
else requests[0].fps
|
||||
),
|
||||
requests[0].save_output,
|
||||
lambda idx: output_requests[idx].output_file_path(),
|
||||
audio=output_batch.audio,
|
||||
|
||||
@@ -526,14 +526,15 @@ async def create_video(
|
||||
# Parse model-specific multipart metadata before creating request-owned
|
||||
# directories or saving uploads, so malformed JSON leaves no resources.
|
||||
if is_multipart:
|
||||
if not prompt:
|
||||
sampling_params_cls = resolve_sampling_params_cls(server_args)
|
||||
if not prompt and not sampling_params_cls.prompt_optional:
|
||||
raise HTTPException(status_code=400, detail="prompt is required")
|
||||
raw_form = await request.form()
|
||||
extra_from_form = _multipart_video_extras(
|
||||
raw_form,
|
||||
extra_body=extra_body,
|
||||
extra_params=extra_params,
|
||||
sampling_params_cls=resolve_sampling_params_cls(server_args),
|
||||
sampling_params_cls=sampling_params_cls,
|
||||
)
|
||||
|
||||
# Resolve input upload directory (may be a temp dir when saving is disabled)
|
||||
@@ -613,7 +614,11 @@ async def create_video(
|
||||
num_frames_val = form_value("num_frames", num_frames)
|
||||
|
||||
req = VideoGenerationsRequest(
|
||||
prompt=prompt,
|
||||
# ``prompt`` is a required str field on VideoGenerationsRequest; it is only
|
||||
# ``None`` here for a prompt-optional pipeline (the gate above already
|
||||
# rejected a missing prompt for every other one), so an empty string is the
|
||||
# correct substitute, not a real (ignored) prompt value.
|
||||
prompt=prompt or "",
|
||||
enhance_prompt=form_value("enhance_prompt", enhance_prompt) or False,
|
||||
input_reference=input_path,
|
||||
video_path=form_value("video_path", video_input_path),
|
||||
@@ -710,6 +715,8 @@ async def create_video(
|
||||
detail=f"Failed to process image source: {str(e)}",
|
||||
)
|
||||
payload["input_reference"] = input_path
|
||||
if resolve_sampling_params_cls(server_args).prompt_optional:
|
||||
payload.setdefault("prompt", "")
|
||||
req = VideoGenerationsRequest(**payload)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid request body: {e}")
|
||||
|
||||
@@ -1245,13 +1245,13 @@ class USPAttention(nn.Module):
|
||||
if get_ring_parallel_world_size() > 1:
|
||||
if (
|
||||
meta_only_pad
|
||||
and q.shape[0] == 1
|
||||
and self.backend == AttentionBackendEnum.FA
|
||||
and not self.causal
|
||||
):
|
||||
return self._forward_ring_tail_pad(q, k, v, attn_mask_meta)
|
||||
raise NotImplementedError(
|
||||
"USPAttention masked path supports ring parallelism only "
|
||||
"for batch-1 tail-pad metadata on the FA backend."
|
||||
"for non-causal tail-pad metadata on the FA backend."
|
||||
)
|
||||
if attn_mask is not None and attn_mask.dim() != 2:
|
||||
raise NotImplementedError(
|
||||
@@ -1505,15 +1505,15 @@ class USPAttention(nn.Module):
|
||||
attn_impl=self.attn_impl,
|
||||
real_seq_len=int(attn_mask_meta["pad_start"]),
|
||||
ring_ws=get_ring_parallel_world_size(),
|
||||
)
|
||||
).reshape_as(q)
|
||||
# Match the Ulysses tail path: masked query rows read as zeros. This
|
||||
# rank's chunk covers global rows [rank*chunk, (rank+1)*chunk).
|
||||
pad_from = (
|
||||
int(attn_mask_meta["pad_start"]) - get_ring_parallel_rank() * out.shape[0]
|
||||
int(attn_mask_meta["pad_start"]) - get_ring_parallel_rank() * out.shape[1]
|
||||
)
|
||||
if pad_from < out.shape[0]:
|
||||
out[max(pad_from, 0) :].zero_()
|
||||
return _usp_output_all_to_all(out.unsqueeze(0), head_dim=2)
|
||||
if pad_from < out.shape[1]:
|
||||
out[:, max(pad_from, 0) :].zero_()
|
||||
return _usp_output_all_to_all(out, head_dim=2)
|
||||
|
||||
@staticmethod
|
||||
def _gather_sharded_sequence(
|
||||
|
||||
@@ -351,7 +351,7 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
"""An in-place merge is about to write the base storage; if the
|
||||
snapshot is a zero-copy view into it, materialize the clone now."""
|
||||
if self._base_is_view:
|
||||
self.cpu_weight = self.cpu_weight.clone()
|
||||
self.cpu_weight = self.cpu_weight.to("cpu", copy=True)
|
||||
self._base_is_view = False
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -495,11 +495,10 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
if self.disable_lora:
|
||||
return
|
||||
|
||||
# Zero-copy views unmerge by subtracting B@A; do not clone the base.
|
||||
if not self._base_is_view:
|
||||
self._ensure_base_snapshot_owned()
|
||||
if self.merged:
|
||||
self.unmerge_lora_weights()
|
||||
# Only layers actually merged need a snapshot of the mutable backing store.
|
||||
self._ensure_base_snapshot_owned()
|
||||
|
||||
lora_list = self._active_lora_list()
|
||||
if not lora_list:
|
||||
@@ -612,67 +611,14 @@ class BaseLayerWithLoRA(nn.Module):
|
||||
)
|
||||
self.base_layer.weight = nn.Parameter(new_weight_data)
|
||||
del old_weight
|
||||
elif self._base_is_view:
|
||||
self._unmerge_by_inverse()
|
||||
else:
|
||||
current_device = self.base_layer.weight.data.device
|
||||
cpu_weight_on_device = self.cpu_weight.to(current_device, non_blocking=True)
|
||||
if self.base_layer.weight.data.is_inference():
|
||||
self.base_layer.weight.data = self._as_mutable_tensor(
|
||||
cpu_weight_on_device
|
||||
)
|
||||
else:
|
||||
self.base_layer.weight.data.copy_(cpu_weight_on_device)
|
||||
if (
|
||||
cpu_weight_on_device.data_ptr()
|
||||
!= self.base_layer.weight.data.data_ptr()
|
||||
):
|
||||
del cpu_weight_on_device
|
||||
# Update the offload backing store too, not a transient GPU placeholder.
|
||||
# A copy keeps inference-tensor rebinding from aliasing the snapshot.
|
||||
restored = self.cpu_weight.clone()
|
||||
write_dense_weight(self.base_layer, self._as_mutable_tensor(restored))
|
||||
|
||||
self.merged = False
|
||||
|
||||
@torch.no_grad()
|
||||
def _unmerge_by_inverse(self) -> None:
|
||||
lora_list = self._active_lora_list()
|
||||
if not lora_list:
|
||||
raise ValueError(
|
||||
"LoRA weights not set; cannot unmerge a zero-copy view by inverse."
|
||||
)
|
||||
src, current_device = self._materialized_weight_src()
|
||||
data = self._as_mutable_tensor(src.to(get_local_torch_device()))
|
||||
target_dtype = data.dtype
|
||||
if (
|
||||
self._should_merge_in_fp32(lora_list)
|
||||
and data.is_floating_point()
|
||||
and data.dtype != torch.float32
|
||||
):
|
||||
data = data.to(torch.float32)
|
||||
inverted = [
|
||||
(
|
||||
lora_A,
|
||||
lora_B,
|
||||
lora_path,
|
||||
-lora_strength,
|
||||
lora_rank,
|
||||
lora_alpha,
|
||||
output_offset,
|
||||
)
|
||||
for (
|
||||
lora_A,
|
||||
lora_B,
|
||||
lora_path,
|
||||
lora_strength,
|
||||
lora_rank,
|
||||
lora_alpha,
|
||||
output_offset,
|
||||
) in reversed(lora_list)
|
||||
]
|
||||
self._merge_lora_into_data(data, inverted)
|
||||
write_dense_weight(
|
||||
self.base_layer,
|
||||
self._as_mutable_tensor(data.to(dtype=target_dtype)),
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def commit_merged_as_base(self) -> None:
|
||||
"""Promote the currently merged weights to the permanent base.
|
||||
|
||||
@@ -823,12 +823,12 @@ def _ring_merge_attention(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Online-softmax combine of one more ring step's partial attention.
|
||||
|
||||
`step_out` is `[T, H, D]`; `step_lse` is FlashAttention's varlen LSE
|
||||
layout `[H, T]`. Both are already self-normalized over their own KV
|
||||
`step_out` is `[T, H, D]` or `[B, T, H, D]`; `step_lse` is
|
||||
`[H, T]` or `[B, H, T]`. Both are already self-normalized over their own KV
|
||||
chunk, so combining chunks is the standard two-term logsumexp merge
|
||||
(exact up to float rounding); done in fp32 for stability.
|
||||
"""
|
||||
step_lse = step_lse.transpose(0, 1).unsqueeze(-1).to(torch.float32)
|
||||
step_lse = step_lse.transpose(-2, -1).unsqueeze(-1).to(torch.float32)
|
||||
step_out = step_out.to(torch.float32)
|
||||
if out_acc is None:
|
||||
return step_out, step_lse
|
||||
@@ -850,7 +850,8 @@ def _ring_attention_varlen(
|
||||
) -> torch.Tensor:
|
||||
"""Ring-rotated varlen attention over one rank's local packed chunk.
|
||||
|
||||
`q, k, v` are this rank's full local ring chunk (`ring_chunk_len` rows,
|
||||
`q, k, v` are `[T, H, D]` or `[B, T, H, D]`, with equal lengths per batch.
|
||||
They contain this rank's full local ring chunk (`ring_chunk_len` rows,
|
||||
real rows followed by however many of this chunk's rows are padding).
|
||||
KV is P2P-rotated around the ring one hop per step (send this step's
|
||||
buffer to the next rank, receive the following step's buffer from the
|
||||
@@ -864,7 +865,7 @@ def _ring_attention_varlen(
|
||||
"""
|
||||
ring_pg = get_sp_group().ring_group
|
||||
assert ring_pg is not None, "Ring process group is not initialized."
|
||||
ring_chunk_len = q.shape[0]
|
||||
ring_chunk_len = q.shape[-3]
|
||||
_, ring_rank = get_ring_ctx()
|
||||
|
||||
# `isend`/`irecv` (unlike collectives) address peers by global rank even
|
||||
@@ -913,11 +914,14 @@ def _ring_attention_varlen(
|
||||
max(real_seq_len - src_rank * ring_chunk_len, 0), ring_chunk_len
|
||||
)
|
||||
if remote_used > 0:
|
||||
step_out, step_lse = attn_impl.forward_ring_kv_chunk(
|
||||
q,
|
||||
kv_bufs[cur][0, :remote_used],
|
||||
kv_bufs[cur][1, :remote_used],
|
||||
)
|
||||
key = kv_bufs[cur][0, ..., :remote_used, :, :]
|
||||
value = kv_bufs[cur][1, ..., :remote_used, :, :]
|
||||
if q.ndim == 3:
|
||||
step_out, step_lse = attn_impl.forward_ring_kv_chunk(q, key, value)
|
||||
else:
|
||||
step_out, step_lse, *_ = attn_impl.forward(
|
||||
q, key, value, attn_metadata=None, return_softmax_lse=True
|
||||
)
|
||||
out_acc, lse_acc = _ring_merge_attention(
|
||||
out_acc, lse_acc, step_out, step_lse
|
||||
)
|
||||
|
||||
@@ -146,8 +146,8 @@ class ComponentLoader(ABC):
|
||||
# the list of possible name of the component in model_index.json, e.g., scheduler
|
||||
component_names: list[str] = []
|
||||
|
||||
# diffusers or transformers
|
||||
expected_library: str = ""
|
||||
# diffusers or transformers (a tuple lists several accepted spellings)
|
||||
expected_library: str | tuple[str, ...] = ""
|
||||
|
||||
_loaders_registered = False
|
||||
|
||||
@@ -681,7 +681,12 @@ class ComponentLoader(ABC):
|
||||
if loader_cls is not None:
|
||||
expected_library = loader_cls.expected_library
|
||||
# Assert that the library matches what's expected for this component type
|
||||
assert transformers_or_diffusers == expected_library, (
|
||||
allowed = (
|
||||
(expected_library,)
|
||||
if isinstance(expected_library, str)
|
||||
else tuple(expected_library)
|
||||
)
|
||||
assert transformers_or_diffusers in allowed, (
|
||||
f"{loader_type} must be loaded from {expected_library}, got {transformers_or_diffusers}"
|
||||
)
|
||||
loader = loader_cls()
|
||||
|
||||
+80
@@ -0,0 +1,80 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Loader of the ``latent_upscaler`` component of an official Kandinsky 6 SR repo."""
|
||||
|
||||
import torch
|
||||
from safetensors.torch import load_file as safetensors_load_file
|
||||
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
|
||||
ComponentLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import _list_safetensors_files
|
||||
from sglang.multimodal_gen.runtime.models.registry import ModelRegistry
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.hf_diffusers_utils import (
|
||||
get_diffusers_component_config,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LatentUpscalerLoader(ComponentLoader):
|
||||
"""Builds the bank from ``config.json`` and loads its weights strictly.
|
||||
|
||||
Every failure (unparsable model config, missing or unexpected keys) surfaces as an
|
||||
error: this component has no native diffusers fallback, so the base class must not
|
||||
swallow the customized-load error and try ``AutoModel``.
|
||||
"""
|
||||
|
||||
component_names = ["latent_upscaler"]
|
||||
# The official ``save_pretrained`` writes "kandinsky6", the Hub repo "diffusers".
|
||||
expected_library = ("diffusers", "kandinsky6")
|
||||
|
||||
def should_raise_customized_load_error(
|
||||
self, server_args: ServerArgs, component_name: str
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def load_customized(
|
||||
self, component_model_path: str, server_args: ServerArgs, component_name: str
|
||||
):
|
||||
config = get_diffusers_component_config(component_path=component_model_path)
|
||||
class_name = config.pop("_class_name", None)
|
||||
if class_name is None:
|
||||
raise ValueError(f"{component_model_path}/config.json has no _class_name")
|
||||
bank_cls, _ = ModelRegistry.resolve_model_cls(class_name)
|
||||
bank = self._build_bank(bank_cls, config, component_model_path)
|
||||
|
||||
state_dict: dict[str, torch.Tensor] = {}
|
||||
for path in _list_safetensors_files(component_model_path):
|
||||
state_dict.update(safetensors_load_file(path))
|
||||
if not state_dict:
|
||||
raise FileNotFoundError(f"no safetensors files in {component_model_path}")
|
||||
bank.load_state_dict(state_dict, strict=True)
|
||||
|
||||
target_device = self.target_device(
|
||||
server_args.should_start_component_on_cpu(component_name)
|
||||
)
|
||||
# The reference runs the LU in bf16 (weights and non-persistent buffers).
|
||||
return (
|
||||
bank.eval()
|
||||
.requires_grad_(False)
|
||||
.to(device=target_device, dtype=torch.bfloat16)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_bank(bank_cls, config: dict, component_model_path: str):
|
||||
for key in ("models", "scaling_factor"):
|
||||
if key not in config:
|
||||
raise ValueError(f"{component_model_path}/config.json lacks {key!r}")
|
||||
try:
|
||||
return bank_cls(
|
||||
models=config["models"],
|
||||
scaling_factor=config["scaling_factor"],
|
||||
scales=config.get("scales", (2, 4)),
|
||||
)
|
||||
except (ValueError, KeyError, TypeError) as error:
|
||||
raise ValueError(
|
||||
f"invalid latent upscaler config in {component_model_path}: "
|
||||
f"{type(error).__name__}: {error}"
|
||||
) from error
|
||||
@@ -1435,6 +1435,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
output=result.output,
|
||||
audio=getattr(result, "audio", None),
|
||||
audio_sample_rate=getattr(result, "audio_sample_rate", None),
|
||||
fps=getattr(result, "fps", None),
|
||||
metrics=result.metrics,
|
||||
usage=getattr(result, "usage", None),
|
||||
trajectory_timesteps=getattr(result, "trajectory_timesteps", None),
|
||||
@@ -1450,6 +1451,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
) -> OutputBatch:
|
||||
"""Merge per-output batches produced by grouped execution."""
|
||||
merged = OutputBatch()
|
||||
merged.fps = output_batches[0].fps
|
||||
parts = _ExpandedOutputParts()
|
||||
|
||||
for output_batch in output_batches:
|
||||
|
||||
+1
@@ -53,6 +53,7 @@ VAE_COMPONENT_NAMES = frozenset(
|
||||
"audio_vae",
|
||||
"vocoder",
|
||||
"spatial_upsampler",
|
||||
"latent_upscaler",
|
||||
"condition_image_encoder",
|
||||
"delight_vae",
|
||||
"diffusion_decoder",
|
||||
|
||||
@@ -1122,6 +1122,7 @@ class Scheduler(SchedulerWarmupMixin, SchedulerPostTrainingMixin, SchedulerDisag
|
||||
output_batch.audio, start, end, total_items
|
||||
),
|
||||
audio_sample_rate=output_batch.audio_sample_rate,
|
||||
fps=output_batch.fps,
|
||||
action_pred=self._slice_batched_value(
|
||||
output_batch.action_pred, start, end, total_items
|
||||
),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,430 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Text-free Kandinsky 6 SR DiT, reusing the joint model's TP/SP building blocks.
|
||||
|
||||
Latents are [B, T, H, W, C]. Residuals, norms and modulation remain fp32;
|
||||
linear inputs use the parameter dtype. A pi-Flow DX head emits n_grid velocities
|
||||
per latent channel. VAE, upscaler and tile scheduling remain replicated."""
|
||||
|
||||
from collections.abc import Iterable, Iterator
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
ATTRIBUTE_OVERRIDE_WHITELIST,
|
||||
Kandinsky6SRArchConfig,
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.fsdp import is_module_list_entry_in
|
||||
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_ring_ctx,
|
||||
get_ulysses_ctx,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.sp_shard_utils import (
|
||||
gather_seq,
|
||||
shard_like,
|
||||
shard_seq,
|
||||
tail_attn_meta,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.layernorm import LayerNormScaleShift
|
||||
from sglang.multimodal_gen.runtime.layers.quantization.configs.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.base import BaseDiT
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
|
||||
Kandinsky6Attention,
|
||||
Kandinsky6FeedForward,
|
||||
Kandinsky6Modulation,
|
||||
Kandinsky6OutLayer,
|
||||
Kandinsky6RoPE3D,
|
||||
Kandinsky6TimeEmbeddings,
|
||||
Kandinsky6VisualEmbeddings,
|
||||
_build_rotary_freqs,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.srt.utils import add_prefix
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_ARCH_CONFIG_DEFAULTS = Kandinsky6SRDitConfig().arch_config
|
||||
_BLOCK_CONTAINERS = ("visual_transformer_blocks",)
|
||||
_TIME_EMBEDDING_NAMES = ("time_embeddings", "motion_embeddings", "lq_noise_embeddings")
|
||||
|
||||
|
||||
def _is_sr_transformer_block(name: str, module: object) -> bool:
|
||||
return is_module_list_entry_in(name, _BLOCK_CONTAINERS)
|
||||
|
||||
|
||||
def _modulate(
|
||||
norm: LayerNormScaleShift, x: torch.Tensor, shift: torch.Tensor, scale: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""fp32 ``LN(x) * (scale + 1) + shift`` (no cast back: the caller picks the dtype)."""
|
||||
return norm(x.float(), shift=shift, scale=scale)
|
||||
|
||||
|
||||
class Kandinsky6SRRoPE3D(Kandinsky6RoPE3D):
|
||||
"""Keep fp32 RoPE angle tables across residency-manager dtype casts."""
|
||||
|
||||
def _apply(self, fn, recurse=True):
|
||||
device = fn(torch.zeros(1)).device
|
||||
for name, buffer in self._buffers.items():
|
||||
if buffer is not None:
|
||||
self._buffers[name] = buffer.to(device=device)
|
||||
return self
|
||||
|
||||
|
||||
class Kandinsky6SRDecoderBlock(nn.Module):
|
||||
"""Modulated self-attention and FFN with fp32 residuals and parameter-dtype linears."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_dim: int,
|
||||
time_dim: int,
|
||||
ff_dim: int,
|
||||
head_dim: int,
|
||||
supported_attention_backends: set[AttentionBackendEnum] | None = None,
|
||||
prefix: str = "",
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.visual_modulation = Kandinsky6Modulation(time_dim, model_dim, 6)
|
||||
self.self_attention_norm = LayerNormScaleShift(
|
||||
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
|
||||
)
|
||||
self.self_attention = Kandinsky6Attention(
|
||||
model_dim,
|
||||
head_dim,
|
||||
supported_attention_backends=supported_attention_backends,
|
||||
prefix=add_prefix("self_attention", prefix),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.feed_forward_norm = LayerNormScaleShift(
|
||||
model_dim, eps=1e-5, elementwise_affine=False, dtype=torch.float32
|
||||
)
|
||||
self.feed_forward = Kandinsky6FeedForward(
|
||||
model_dim,
|
||||
ff_dim,
|
||||
prefix=add_prefix("feed_forward", prefix),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
visual_embed: torch.Tensor,
|
||||
time_embed: torch.Tensor,
|
||||
rope: torch.Tensor,
|
||||
compute_dtype: torch.dtype,
|
||||
attn_mask_meta: dict | None = None,
|
||||
) -> torch.Tensor:
|
||||
modulation = self.visual_modulation(time_embed).unsqueeze(dim=1)
|
||||
self_attn_params, ff_params = torch.chunk(modulation, 2, dim=-1)
|
||||
|
||||
shift, scale, gate = torch.chunk(self_attn_params, 3, dim=-1)
|
||||
out = _modulate(self.self_attention_norm, visual_embed, shift, scale)
|
||||
out = self.self_attention(
|
||||
out.to(compute_dtype), rotary_emb=rope, attn_mask_meta=attn_mask_meta
|
||||
)
|
||||
visual_embed = visual_embed.float() + gate.float() * out.float()
|
||||
|
||||
shift, scale, gate = torch.chunk(ff_params, 3, dim=-1)
|
||||
out = _modulate(self.feed_forward_norm, visual_embed, shift, scale)
|
||||
out = self.feed_forward(out.to(compute_dtype))
|
||||
return visual_embed + gate.float() * out.float()
|
||||
|
||||
|
||||
class Kandinsky6SROutLayer(Kandinsky6OutLayer):
|
||||
"""K6 ``OutLayer`` for an fp32 residual stream (norm in fp32, linear in weight dtype)."""
|
||||
|
||||
def forward(
|
||||
self, visual_embed: torch.Tensor, time_embed: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
shift, scale = torch.chunk(
|
||||
self.modulation(time_embed).unsqueeze(dim=1), 2, dim=-1
|
||||
)
|
||||
x = (
|
||||
self.norm(visual_embed.float()) * (scale.float()[:, None, None] + 1.0)
|
||||
+ shift.float()[:, None, None]
|
||||
)
|
||||
x, _ = self.out_layer(x.to(self.out_layer.weight.dtype))
|
||||
|
||||
batch_size, duration, height, width, _ = x.shape
|
||||
pt, ph, pw = self.patch_size
|
||||
return (
|
||||
x.view(batch_size, duration, height, width, -1, pt, ph, pw)
|
||||
.permute(0, 1, 5, 2, 6, 3, 7, 4)
|
||||
.flatten(1, 2)
|
||||
.flatten(2, 3)
|
||||
.flatten(3, 4)
|
||||
)
|
||||
|
||||
|
||||
class Kandinsky6SRTransformer3DModel(BaseDiT, LayerwiseOffloadableModuleMixin):
|
||||
"""Text-free Kandinsky 6 SR DiT (optionally with a DX / pi-Flow head)."""
|
||||
|
||||
_fsdp_shard_conditions = [_is_sr_transformer_block]
|
||||
_compile_conditions = [_is_sr_transformer_block]
|
||||
param_names_mapping = _ARCH_CONFIG_DEFAULTS.param_names_mapping
|
||||
reverse_param_names_mapping: dict = {}
|
||||
lora_param_names_mapping: dict = {}
|
||||
# Dense attention only: the sparse (NABLA) path of the reference is not ported.
|
||||
_supported_attention_backends = {
|
||||
AttentionBackendEnum.FA,
|
||||
AttentionBackendEnum.TORCH_SDPA,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _validate_tp_config(*, arch: Kandinsky6SRArchConfig, tp_size: int) -> None:
|
||||
if tp_size <= 0:
|
||||
raise ValueError("Kandinsky6SR TP size must be positive.")
|
||||
for name, value in (
|
||||
("num_attention_heads", arch.num_attention_heads),
|
||||
("ff_dim", arch.ff_dim),
|
||||
):
|
||||
if value % tp_size:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR {name}={value} must be divisible by TP size {tp_size}."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_sequence_parallel_config(
|
||||
*, arch: Kandinsky6SRArchConfig, tp_size: int, ulysses_size: int, ring_size: int
|
||||
) -> None:
|
||||
if ulysses_size <= 0:
|
||||
raise ValueError("Kandinsky6SR Ulysses size must be positive.")
|
||||
if ring_size <= 0:
|
||||
raise ValueError("Kandinsky6SR ring size must be positive.")
|
||||
if ulysses_size == 1 and ring_size == 1:
|
||||
return
|
||||
# Ring Attention rotates whole K/V shards between ranks rather than
|
||||
# splitting heads, so only Ulysses constrains head divisibility.
|
||||
local_heads = arch.num_attention_heads // tp_size
|
||||
if local_heads % ulysses_size:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR TP-local attention heads {local_heads} must be "
|
||||
f"divisible by Ulysses size {ulysses_size} (total heads="
|
||||
f"{arch.num_attention_heads}, TP={tp_size})."
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Kandinsky6SRDitConfig,
|
||||
hf_config: dict[str, Any],
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
) -> None:
|
||||
super().__init__(config=config, hf_config=hf_config)
|
||||
arch: Kandinsky6SRArchConfig = self.config
|
||||
self.quant_config = quant_config
|
||||
head_dim = sum(arch.axes_dims)
|
||||
|
||||
tp_size = get_tp_world_size()
|
||||
ulysses_size, _ = get_ulysses_ctx()
|
||||
ring_size, _ = get_ring_ctx()
|
||||
self._validate_tp_config(arch=arch, tp_size=tp_size)
|
||||
self._validate_sequence_parallel_config(
|
||||
arch=arch, tp_size=tp_size, ulysses_size=ulysses_size, ring_size=ring_size
|
||||
)
|
||||
|
||||
self.in_visual_dim = arch.in_visual_dim
|
||||
self.base_out_visual_dim = arch.base_out_visual_dim
|
||||
self.n_grid = arch.n_grid
|
||||
self.model_dim = arch.model_dim
|
||||
self.patch_size = arch.patch_size
|
||||
self.use_motion_score = arch.use_motion_score
|
||||
self.use_lq_noise_cond = arch.use_lq_noise_cond
|
||||
# The input width is fixed by the *trained* configuration ...
|
||||
self.visual_embed_dim = (
|
||||
2 * arch.in_visual_dim + 1
|
||||
if arch.trained_wide_input
|
||||
else arch.in_visual_dim
|
||||
)
|
||||
self._build_layers(arch, head_dim=head_dim, quant_config=quant_config)
|
||||
|
||||
self.hidden_size = arch.hidden_size
|
||||
self.num_attention_heads = arch.num_attention_heads
|
||||
self.num_channels_latents = arch.num_channels_latents
|
||||
self.layer_names = list(_BLOCK_CONTAINERS)
|
||||
# ... while inference behaviour comes from the post-load overrides.
|
||||
self._apply_attribute_overrides(arch)
|
||||
self._require_all_checkpoint_keys()
|
||||
self.__post_init__()
|
||||
|
||||
def _build_layers(
|
||||
self,
|
||||
arch: Kandinsky6SRArchConfig,
|
||||
*,
|
||||
head_dim: int,
|
||||
quant_config: QuantizationConfig | None,
|
||||
) -> None:
|
||||
self.time_embeddings = Kandinsky6TimeEmbeddings(arch.model_dim, arch.time_dim)
|
||||
if arch.use_motion_score:
|
||||
self.motion_embeddings = Kandinsky6TimeEmbeddings(
|
||||
arch.model_dim, arch.time_dim
|
||||
)
|
||||
if arch.use_lq_noise_cond:
|
||||
# Present in the checkpoint; never used at inference (kept for loading).
|
||||
self.lq_noise_embeddings = Kandinsky6TimeEmbeddings(
|
||||
arch.model_dim, arch.time_dim
|
||||
)
|
||||
# Constant that replaces the pooled empty-caption embedding of the text DiT.
|
||||
self.pooled_bias = nn.Parameter(torch.zeros(arch.time_dim))
|
||||
self.visual_embeddings = Kandinsky6VisualEmbeddings(
|
||||
self.visual_embed_dim,
|
||||
arch.model_dim,
|
||||
arch.patch_size,
|
||||
prefix=add_prefix("visual_embeddings", self.prefix),
|
||||
)
|
||||
self.visual_rope_embeddings = Kandinsky6SRRoPE3D(arch.axes_dims)
|
||||
self.visual_transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
Kandinsky6SRDecoderBlock(
|
||||
arch.model_dim,
|
||||
arch.time_dim,
|
||||
arch.ff_dim,
|
||||
head_dim,
|
||||
self._supported_attention_backends,
|
||||
prefix=add_prefix(f"visual_transformer_blocks.{i}", self.prefix),
|
||||
quant_config=quant_config,
|
||||
)
|
||||
for i in range(arch.num_visual_blocks)
|
||||
]
|
||||
)
|
||||
self.out_layer = Kandinsky6SROutLayer(
|
||||
arch.model_dim,
|
||||
arch.time_dim,
|
||||
arch.head_width,
|
||||
arch.patch_size,
|
||||
)
|
||||
|
||||
def _apply_attribute_overrides(self, arch: Kandinsky6SRArchConfig) -> None:
|
||||
self.instruct_type = arch.instruct_type
|
||||
self.visual_cond = arch.visual_cond
|
||||
self.attention_params = arch.attention_params
|
||||
for name in ATTRIBUTE_OVERRIDE_WHITELIST:
|
||||
if name in arch.attribute_overrides:
|
||||
setattr(self, name, arch.attribute_overrides[name])
|
||||
self._check_input_width_matches_overrides()
|
||||
sparse_type = arch.requested_sparse_attention()
|
||||
if sparse_type is not None:
|
||||
logger.warning(
|
||||
"Kandinsky6SR: attention_params request %r sparse attention; this port "
|
||||
"runs dense attention, so outputs differ from the sparse reference.",
|
||||
sparse_type,
|
||||
)
|
||||
|
||||
def _check_input_width_matches_overrides(self) -> None:
|
||||
run_wide = self.visual_cond or self.instruct_type in (
|
||||
"channel",
|
||||
"hybrid",
|
||||
"hybrid_anchor",
|
||||
)
|
||||
run_width = 2 * self.in_visual_dim + 1 if run_wide else self.in_visual_dim
|
||||
if run_width != self.visual_embed_dim:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR: the trained input layer takes {self.visual_embed_dim} "
|
||||
f"channels but instruct_type={self.instruct_type!r} with "
|
||||
f"visual_cond={self.visual_cond} feeds {run_width}; set "
|
||||
"attribute_overrides.visual_cond=true to run a wide-input checkpoint "
|
||||
"under instruct_type='noise'."
|
||||
)
|
||||
|
||||
def _require_all_checkpoint_keys(self) -> None:
|
||||
"""A missing tensor must fail loudly instead of being zero-filled by the loader."""
|
||||
for param in self.parameters():
|
||||
param.missing_param_init = "error"
|
||||
|
||||
def preprocess_loaded_state_dict(
|
||||
self, weight_iterator: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Pass tensors through, then raise if the checkpoint held unexpected keys."""
|
||||
map_name = get_param_names_mapping(self.param_names_mapping)
|
||||
expected = set(self.state_dict())
|
||||
unexpected: list[str] = []
|
||||
for name, tensor in weight_iterator:
|
||||
if map_name(name)[0] not in expected:
|
||||
unexpected.append(name)
|
||||
yield name, tensor
|
||||
if unexpected:
|
||||
shown = ", ".join(unexpected[:10])
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR checkpoint has {len(unexpected)} unexpected keys "
|
||||
f"(first: {shown}); the transformer config does not match the weights."
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
visual_rope_pos: (
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor] | list[torch.Tensor]
|
||||
),
|
||||
scale_factor: tuple[float, float, float] = (1.0, 1.0, 1.0),
|
||||
motion_score: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Predict [B, T, H, W, head_width] velocities from channel-last latents.
|
||||
|
||||
timesteps are scaled by 1000. DX channels are ordered grid-major;
|
||||
the scheduler reshapes them into n_grid velocity predictions."""
|
||||
self._check_latent_shape(hidden_states)
|
||||
compute_dtype = self.visual_embeddings.in_layer.weight.dtype
|
||||
time_embed = self.time_embeddings(timestep)
|
||||
if motion_score is not None and self.use_motion_score:
|
||||
time_embed = time_embed + self.motion_embeddings(motion_score)
|
||||
time_embed = time_embed + self.pooled_bias
|
||||
|
||||
visual_embed = self.visual_embeddings(hidden_states.to(compute_dtype))
|
||||
visual_shape = visual_embed.shape[:-1]
|
||||
visual_rope = self.visual_rope_embeddings(
|
||||
visual_shape, visual_rope_pos, scale_factor
|
||||
)
|
||||
visual_embed = visual_embed.flatten(1, 3)
|
||||
visual_rope = visual_rope.flatten(1, 3)
|
||||
visual_embed, shard = shard_seq(visual_embed)
|
||||
visual_rope = shard_like(visual_rope, shard, pad_mode="repeat_last")
|
||||
attn_meta = tail_attn_meta(shard, visual_embed.shape[0], visual_embed.device)
|
||||
for block in self.visual_transformer_blocks:
|
||||
visual_embed = block(
|
||||
visual_embed,
|
||||
time_embed,
|
||||
visual_rope,
|
||||
compute_dtype,
|
||||
attn_mask_meta=attn_meta,
|
||||
)
|
||||
visual_embed = gather_seq(visual_embed, shard.orig_len)
|
||||
return self.out_layer(visual_embed.reshape(*visual_shape, -1), time_embed)
|
||||
|
||||
def _check_latent_shape(self, hidden_states: torch.Tensor) -> None:
|
||||
if hidden_states.ndim != 5 or hidden_states.shape[-1] != self.visual_embed_dim:
|
||||
raise ValueError(
|
||||
"Kandinsky6SR expects a [B, T, H, W, "
|
||||
f"{self.visual_embed_dim}] latent, got {tuple(hidden_states.shape)}"
|
||||
)
|
||||
height, width = hidden_states.shape[2:4]
|
||||
if height % self.patch_size[1] or width % self.patch_size[2]:
|
||||
raise ValueError(
|
||||
f"latent {height}x{width} is not divisible by patch {self.patch_size}"
|
||||
)
|
||||
|
||||
def post_load_weights(self) -> None:
|
||||
"""Rebuild RoPE / time-embedding tables that a meta-device init left on meta."""
|
||||
device = next(self.parameters()).device
|
||||
rope = self.visual_rope_embeddings
|
||||
if any(buf.is_meta for buf in rope.buffers()):
|
||||
self.visual_rope_embeddings = Kandinsky6SRRoPE3D(
|
||||
rope.axes_dims, rope.max_pos, rope.max_period
|
||||
).to(device)
|
||||
for name in _TIME_EMBEDDING_NAMES:
|
||||
embeddings = self._modules.get(name)
|
||||
if embeddings is not None and embeddings.freqs.is_meta:
|
||||
embeddings.freqs = _build_rotary_freqs(
|
||||
embeddings.model_dim // 2, embeddings.max_period
|
||||
).to(device=device)
|
||||
|
||||
|
||||
EntryClass = Kandinsky6SRTransformer3DModel
|
||||
@@ -183,6 +183,9 @@ class CLIPTextModel(TextEncoder):
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
for name, loaded_weight in weights:
|
||||
# standalone CLIP text-transformer checkpoints omit the model wrapper
|
||||
if name.startswith(("embeddings.", "encoder.", "final_layer_norm.")):
|
||||
name = f"text_model.{name}"
|
||||
name = _srt_clip_param_name(name)
|
||||
# Handle q_proj, k_proj, v_proj -> qkv_proj mapping
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
# Copyright 2025 The Kandinsky Team and The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Diffusers scheduler for distilled Kandinsky 6 PiFlow checkpoints."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from diffusers.configuration_utils import register_to_config
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
FlowMatchEulerDiscreteSchedulerOutput,
|
||||
)
|
||||
|
||||
|
||||
class DXPolicy:
|
||||
def __init__( # noqa: PLR0913
|
||||
self,
|
||||
denoising_output: torch.Tensor,
|
||||
x_t_src: torch.Tensor,
|
||||
sigma_t_src: torch.Tensor,
|
||||
segment_size: float | torch.Tensor = 1.0,
|
||||
shift: float = 1.0,
|
||||
mode: str = "grid",
|
||||
eps: float = 1e-4,
|
||||
) -> None:
|
||||
self.x_t_src = x_t_src
|
||||
self.ndim = x_t_src.dim()
|
||||
self.shift = shift
|
||||
self.eps = eps
|
||||
if mode not in ("grid", "polynomial"):
|
||||
raise ValueError(f"Unknown mode: {mode}")
|
||||
self.mode = mode
|
||||
|
||||
self.sigma_t_src = sigma_t_src.reshape(
|
||||
*sigma_t_src.size(), *((self.ndim - sigma_t_src.dim()) * [1])
|
||||
)
|
||||
self.raw_t_src = self._unwarp_t(self.sigma_t_src)
|
||||
segment = segment_size
|
||||
if isinstance(segment, torch.Tensor) and segment.dim() < self.raw_t_src.dim():
|
||||
segment = segment.reshape(
|
||||
*segment.size(), *((self.raw_t_src.dim() - segment.dim()) * [1])
|
||||
)
|
||||
self.raw_t_dst = (self.raw_t_src - segment).clamp(min=0)
|
||||
self.segment_size = (self.raw_t_src - self.raw_t_dst).clamp(min=eps)
|
||||
self.denoising_output_x_0 = self._u_to_x_0(
|
||||
denoising_output, self.x_t_src, self.sigma_t_src
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _interpolate(x: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
|
||||
n = x.size(1)
|
||||
if n < 2: # noqa: PLR2004
|
||||
return x.squeeze(1)
|
||||
t = t.clamp(min=0, max=1) * (n - 1)
|
||||
t0 = t.floor().to(torch.long).clamp(min=0, max=n - 2)
|
||||
t1 = t0 + 1
|
||||
indices = torch.stack([t0, t1], dim=1)
|
||||
values = torch.gather(x, dim=1, index=indices.expand(-1, -1, *x.shape[2:]))
|
||||
return (t1 - t) * values[:, 0] + (t - t0) * values[:, 1]
|
||||
|
||||
def _unwarp_t(self, sigma_t: torch.Tensor) -> torch.Tensor:
|
||||
return sigma_t / (self.shift + (1 - self.shift) * sigma_t)
|
||||
|
||||
@staticmethod
|
||||
def _u_to_x_0(
|
||||
denoising_output: torch.Tensor,
|
||||
x_t: torch.Tensor,
|
||||
sigma_t: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return x_t.unsqueeze(1) - sigma_t.unsqueeze(1) * denoising_output
|
||||
|
||||
def pi(self, x_t: torch.Tensor, sigma_t: torch.Tensor) -> torch.Tensor:
|
||||
sigma_t = sigma_t.reshape(*sigma_t.size(), *((self.ndim - sigma_t.dim()) * [1]))
|
||||
raw_t = self._unwarp_t(sigma_t)
|
||||
if self.mode == "grid":
|
||||
x_0 = self._interpolate(
|
||||
self.denoising_output_x_0,
|
||||
(raw_t - self.raw_t_dst) / self.segment_size,
|
||||
)
|
||||
else:
|
||||
p_order = self.denoising_output_x_0.size(1)
|
||||
diff_t = self.raw_t_src - raw_t
|
||||
basis = torch.stack([diff_t**i for i in range(p_order)], dim=1)
|
||||
x_0 = torch.sum(basis * self.denoising_output_x_0, dim=1)
|
||||
return (x_t - x_0) / sigma_t.clamp(min=self.eps)
|
||||
|
||||
|
||||
def shift_timesteps(t: torch.Tensor, shift: float) -> torch.Tensor:
|
||||
return shift * t / (1 + (shift - 1) * t)
|
||||
|
||||
|
||||
def policy_rollout_fm( # noqa: PLR0913
|
||||
x_t_start: torch.Tensor,
|
||||
sigma_t_start: torch.Tensor,
|
||||
raw_t_start: torch.Tensor,
|
||||
raw_t_end: torch.Tensor,
|
||||
total_substeps: int,
|
||||
policy: DXPolicy,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
num_batches = x_t_start.size(0)
|
||||
ndim = x_t_start.dim()
|
||||
shape = (num_batches, *((ndim - 1) * [1]))
|
||||
raw_t_start = raw_t_start.reshape(shape)
|
||||
raw_t_end = raw_t_end.reshape(shape)
|
||||
sigma_t = sigma_t_start.reshape(shape)
|
||||
|
||||
delta_raw_t = raw_t_start - raw_t_end
|
||||
num_substeps = (delta_raw_t * total_substeps).round().to(torch.long).clamp(min=1)
|
||||
substep_size = delta_raw_t / num_substeps
|
||||
max_num_substeps = num_substeps.max()
|
||||
|
||||
raw_t = raw_t_start
|
||||
x_t = x_t_start
|
||||
for substep_id in range(max_num_substeps.item()):
|
||||
velocity = policy.pi(x_t, sigma_t)
|
||||
raw_t_minus = (raw_t - substep_size).clamp(min=0)
|
||||
sigma_t_minus = shift_timesteps(raw_t_minus, policy.shift)
|
||||
x_t_minus = x_t + velocity * (sigma_t_minus - sigma_t)
|
||||
|
||||
active_mask = num_substeps > substep_id
|
||||
x_t = torch.where(active_mask, x_t_minus, x_t)
|
||||
sigma_t = torch.where(active_mask, sigma_t_minus, sigma_t)
|
||||
raw_t = torch.where(active_mask, raw_t_minus, raw_t)
|
||||
|
||||
return x_t, sigma_t, sigma_t.flatten() * 1_000
|
||||
|
||||
|
||||
class PiflowScheduler(FlowMatchEulerDiscreteScheduler):
|
||||
is_piflow = True
|
||||
|
||||
@register_to_config
|
||||
def __init__(
|
||||
self,
|
||||
num_train_timesteps: int = 1000,
|
||||
shift: float = 5.0,
|
||||
n_grid: int = 10,
|
||||
nfe: int | None = None,
|
||||
eps: float = 1e-6,
|
||||
final_step_size_scale: float = 0.5,
|
||||
num_policy_substeps: int = 128,
|
||||
) -> None:
|
||||
if n_grid < 2:
|
||||
raise ValueError(f"PiflowScheduler requires n_grid >= 2, got {n_grid}")
|
||||
if eps <= 0:
|
||||
raise ValueError(f"PiflowScheduler requires eps > 0, got {eps}")
|
||||
if not 0 < final_step_size_scale <= 1:
|
||||
raise ValueError("PiflowScheduler requires 0 < final_step_size_scale <= 1")
|
||||
if num_policy_substeps < 1:
|
||||
raise ValueError("PiflowScheduler requires num_policy_substeps >= 1")
|
||||
super().__init__(
|
||||
num_train_timesteps=num_train_timesteps,
|
||||
shift=shift,
|
||||
)
|
||||
self.n_grid = int(n_grid)
|
||||
self.eps = float(eps)
|
||||
self.final_step_size_scale = float(final_step_size_scale)
|
||||
self.num_policy_substeps = int(num_policy_substeps)
|
||||
self._piflow_raw_timesteps = torch.empty(0)
|
||||
|
||||
def set_timesteps(
|
||||
self,
|
||||
num_inference_steps: int | None = None,
|
||||
device: str | torch.device | None = None,
|
||||
sigmas: list[float] | None = None,
|
||||
mu: float | None = None,
|
||||
timesteps: list[float] | None = None,
|
||||
) -> None:
|
||||
if sigmas is not None or mu is not None or timesteps is not None:
|
||||
raise ValueError(
|
||||
"PiflowScheduler only supports its configured distilled timestep schedule"
|
||||
)
|
||||
if num_inference_steps is None or num_inference_steps < 1:
|
||||
raise ValueError(
|
||||
f"num_inference_steps must be positive, got {num_inference_steps}"
|
||||
)
|
||||
one_minus_final = 1.0 - self.final_step_size_scale
|
||||
segment = 1.0 / (num_inference_steps - one_minus_final)
|
||||
raw = (
|
||||
1.0
|
||||
- torch.arange(num_inference_steps, dtype=torch.float32, device=device)
|
||||
* segment
|
||||
)
|
||||
sigmas = shift_timesteps(raw, float(self.config.shift))
|
||||
self.num_inference_steps = int(num_inference_steps)
|
||||
self._piflow_raw_timesteps = raw
|
||||
self.timesteps = sigmas * self.config.num_train_timesteps
|
||||
self.sigmas = torch.cat([sigmas, sigmas.new_zeros(1)])
|
||||
self._step_index = None
|
||||
self._begin_index = None
|
||||
|
||||
def _to_grid(
|
||||
self, model_output: torch.Tensor, sample: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
if (
|
||||
model_output.ndim != sample.ndim
|
||||
or model_output.shape[:-1] != sample.shape[:-1]
|
||||
):
|
||||
raise ValueError(
|
||||
"Piflow model output must match sample shape except for the output channels: "
|
||||
f"got {tuple(model_output.shape)} for sample {tuple(sample.shape)}"
|
||||
)
|
||||
if model_output.shape[-1] % self.n_grid != 0:
|
||||
raise ValueError(
|
||||
f"Piflow model output channels {model_output.shape[-1]} are not divisible by n_grid={self.n_grid}"
|
||||
)
|
||||
output_dim = model_output.shape[-1] // self.n_grid
|
||||
if output_dim != sample.shape[-1]:
|
||||
raise ValueError(
|
||||
"Piflow model output channels do not match the sample: "
|
||||
f"expected {sample.shape[-1] * self.n_grid}, got {model_output.shape[-1]}"
|
||||
)
|
||||
return model_output.reshape(
|
||||
*model_output.shape[:-1], self.n_grid, output_dim
|
||||
).movedim(-2, 1)
|
||||
|
||||
def _policy_step(
|
||||
self, model_output: torch.Tensor, sample: torch.Tensor, step_index: int
|
||||
) -> torch.Tensor:
|
||||
model_output = self._to_grid(model_output, sample)
|
||||
raw_src = self._piflow_raw_timesteps[step_index].to(device=sample.device)
|
||||
raw_dst = (
|
||||
self._piflow_raw_timesteps[step_index + 1]
|
||||
if step_index + 1 < self._piflow_raw_timesteps.numel()
|
||||
else self._piflow_raw_timesteps.new_full((), self.eps)
|
||||
).to(device=sample.device)
|
||||
sigma_src = self.sigmas[step_index].to(device=sample.device)
|
||||
token_shape = (sample.shape[0], *((sample.ndim - 1) * [1]))
|
||||
sigma = sigma_src.expand(sample.shape[0]).reshape(token_shape)
|
||||
segment = (raw_src - raw_dst).expand(sample.shape[0])
|
||||
policy = DXPolicy(
|
||||
model_output,
|
||||
sample,
|
||||
sigma,
|
||||
segment,
|
||||
shift=float(self.config.shift),
|
||||
mode="grid",
|
||||
eps=self.eps,
|
||||
)
|
||||
updated, _, _ = policy_rollout_fm(
|
||||
sample,
|
||||
sigma,
|
||||
raw_src.expand(sample.shape[0]),
|
||||
raw_dst.expand(sample.shape[0]),
|
||||
self.num_policy_substeps,
|
||||
policy,
|
||||
)
|
||||
return updated.to(dtype=sample.dtype)
|
||||
|
||||
def _step_index_for(self, timestep: torch.Tensor | float) -> int:
|
||||
if self.step_index is None:
|
||||
if self.begin_index is not None:
|
||||
self._step_index = self.begin_index
|
||||
else:
|
||||
schedule_timesteps = self.timesteps
|
||||
if not isinstance(schedule_timesteps, torch.Tensor):
|
||||
schedule_timesteps = torch.as_tensor(
|
||||
schedule_timesteps, dtype=torch.float32
|
||||
)
|
||||
timestep = torch.as_tensor(
|
||||
timestep,
|
||||
device=schedule_timesteps.device,
|
||||
dtype=schedule_timesteps.dtype,
|
||||
)
|
||||
indices = torch.nonzero(schedule_timesteps == timestep).flatten()
|
||||
if not indices.numel():
|
||||
raise ValueError(
|
||||
f"timestep {timestep.item()} is not in the Piflow schedule"
|
||||
)
|
||||
position = 1 if indices.numel() > 1 else 0
|
||||
self._step_index = int(indices[position].item())
|
||||
if self.step_index is None or self.step_index >= self.num_inference_steps:
|
||||
raise RuntimeError(
|
||||
"PiflowScheduler.step called after the schedule was exhausted"
|
||||
)
|
||||
return int(self.step_index)
|
||||
|
||||
def step(
|
||||
self,
|
||||
model_output: torch.FloatTensor,
|
||||
timestep: float | torch.FloatTensor,
|
||||
sample: torch.FloatTensor,
|
||||
return_dict: bool = True,
|
||||
) -> FlowMatchEulerDiscreteSchedulerOutput | tuple:
|
||||
if isinstance(timestep, int) or isinstance(
|
||||
timestep, (torch.IntTensor, torch.LongTensor)
|
||||
):
|
||||
raise ValueError(
|
||||
"Passing integer indices as timesteps to PiflowScheduler.step() is not supported; "
|
||||
"pass a value from scheduler.timesteps instead"
|
||||
)
|
||||
step_index = self._step_index_for(timestep)
|
||||
# PiFlow's policy rollout performs its update in float32. Keep that
|
||||
# precision across outer steps, matching the native sampler.
|
||||
updated = self._policy_step(model_output, sample.to(torch.float32), step_index)
|
||||
self._step_index += 1
|
||||
if return_dict:
|
||||
return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=updated)
|
||||
return (updated,)
|
||||
|
||||
|
||||
EntryClass = PiflowScheduler
|
||||
+336
@@ -0,0 +1,336 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Kandinsky 6 SR x2/x4 upsampler bank for scaled [B, C, T, H, W] KVAE latents.
|
||||
|
||||
Norms are modulated by the source latent; temporal conv padding repeats edge frames.
|
||||
_models entries follow checkpoint scale order and retain checkpoint module names."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import Tensor, nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.upsamplers.kandinsky6_sr import (
|
||||
Kandinsky6SRLatentUpscalerEntryConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ReplicateTimeConv3d(nn.Conv3d):
|
||||
"""Conv3d with explicit edge-repeated temporal padding and zero spatial padding."""
|
||||
|
||||
def __init__(self, in_channels: int, out_channels: int, kernel_size: int) -> None:
|
||||
pad = kernel_size // 2
|
||||
super().__init__(in_channels, out_channels, kernel_size, padding=(0, pad, pad))
|
||||
self.temporal_pad = pad
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
x = F.pad(
|
||||
x, (0, 0, 0, 0, self.temporal_pad, self.temporal_pad), mode="replicate"
|
||||
)
|
||||
return super().forward(x)
|
||||
|
||||
|
||||
class RMSNorm(nn.Module):
|
||||
"""Channel RMS norm of ``[B, C, T, H, W]`` features with a learnable per-channel gain."""
|
||||
|
||||
def __init__(self, dim: int) -> None:
|
||||
super().__init__()
|
||||
self.scale = dim**0.5
|
||||
self.gamma = nn.Parameter(torch.ones(dim, 1, 1, 1))
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
# Upcast to float32 around the normalize (matches the Diffusers reference's
|
||||
# Kandinsky6SRLatentUpscalerRMSNorm): without it, bf16 activations drift from the
|
||||
# reference across the many ResidualBlock/X2Branch calls that route through this norm.
|
||||
normalized = F.normalize(x.float(), dim=1).to(x.dtype)
|
||||
return normalized * self.scale * self.gamma
|
||||
|
||||
|
||||
class ModulatedRMSNorm(nn.Module):
|
||||
"""``RMSNorm(x) * conv_y(zq) + conv_b(zq)`` with 1x1x1 convs of the conditioning latent ``zq``."""
|
||||
|
||||
def __init__(self, dim: int, zq_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.norm = RMSNorm(dim)
|
||||
self.conv_y = nn.Conv3d(zq_dim, dim, kernel_size=1)
|
||||
self.conv_b = nn.Conv3d(zq_dim, dim, kernel_size=1)
|
||||
|
||||
def forward(self, x: Tensor, zq: Tensor) -> Tensor:
|
||||
if zq.shape[2:] != x.shape[2:]:
|
||||
zq = F.interpolate(zq, size=x.shape[2:], mode="nearest")
|
||||
return self.norm(x) * self.conv_y(zq) + self.conv_b(zq)
|
||||
|
||||
|
||||
class ResidualBlock(nn.Module):
|
||||
"""Pre-activation block ``norm1 -> SiLU -> conv1 -> norm2 -> SiLU -> conv2`` plus a (1x1x1 if
|
||||
narrowing) skip."""
|
||||
|
||||
def __init__(
|
||||
self, in_channels: int, out_channels: int, mid_channels: int, zq_dim: int
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.norm1 = ModulatedRMSNorm(in_channels, zq_dim)
|
||||
self.conv1 = ReplicateTimeConv3d(in_channels, mid_channels, kernel_size=3)
|
||||
self.norm2 = ModulatedRMSNorm(mid_channels, zq_dim)
|
||||
self.conv2 = ReplicateTimeConv3d(mid_channels, out_channels, kernel_size=3)
|
||||
self.shortcut: nn.Module = (
|
||||
nn.Identity()
|
||||
if in_channels == out_channels
|
||||
else ReplicateTimeConv3d(in_channels, out_channels, kernel_size=1)
|
||||
)
|
||||
|
||||
def forward(self, x: Tensor, zq: Tensor) -> Tensor:
|
||||
h = F.silu(self.norm1(x, zq))
|
||||
h = self.conv1(h)
|
||||
h = F.silu(self.norm2(h, zq))
|
||||
h = self.conv2(h)
|
||||
return self.shortcut(x) + h
|
||||
|
||||
|
||||
def nearest_2x(x: Tensor) -> Tensor:
|
||||
"""Nearest-neighbour 2x resize of H and W of ``[B, C, T, H, W]`` (T folded into the batch)."""
|
||||
b, c, t, h, w = x.shape
|
||||
x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w)
|
||||
x = F.interpolate(x, scale_factor=2, mode="nearest")
|
||||
return x.reshape(b, t, c, 2 * h, 2 * w).permute(0, 2, 1, 3, 4)
|
||||
|
||||
|
||||
class PXSUpsample(nn.Module):
|
||||
"""2x spatial upsample ``linear(up + spatial_conv(up))`` with ``up = nearest_2x(x)``."""
|
||||
|
||||
def __init__(self, channels: int) -> None:
|
||||
super().__init__()
|
||||
self.spatial_conv = nn.Conv3d(
|
||||
channels, channels, kernel_size=(1, 3, 3), padding=(0, 1, 1)
|
||||
)
|
||||
self.linear = nn.Conv3d(channels, channels, kernel_size=1)
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
up = nearest_2x(x)
|
||||
return self.linear(up + self.spatial_conv(up))
|
||||
|
||||
|
||||
class X2Finisher(nn.Module):
|
||||
"""The ``PXSUpsample`` conv pair applied on the unchanged grid (no resize)."""
|
||||
|
||||
def __init__(self, channels: int) -> None:
|
||||
super().__init__()
|
||||
self.spatial_conv = nn.Conv3d(
|
||||
channels, channels, kernel_size=(1, 3, 3), padding=(0, 1, 1)
|
||||
)
|
||||
self.linear = nn.Conv3d(channels, channels, kernel_size=1)
|
||||
|
||||
def forward(self, x: Tensor) -> Tensor:
|
||||
return self.linear(x + self.spatial_conv(x))
|
||||
|
||||
|
||||
def _stem(in_channels: int, out_channels: int) -> nn.Sequential:
|
||||
return nn.Sequential(ReplicateTimeConv3d(in_channels, out_channels, kernel_size=3))
|
||||
|
||||
|
||||
def _mid_output_head(channels: int, out_channels: int) -> nn.Sequential:
|
||||
"""Unused training head whose checkpoint layout remains positional."""
|
||||
return nn.Sequential(
|
||||
RMSNorm(channels),
|
||||
nn.SiLU(),
|
||||
ReplicateTimeConv3d(channels, out_channels, kernel_size=3),
|
||||
)
|
||||
|
||||
|
||||
class Kandinsky6SRLatentUpscalerOutputHead(nn.Module):
|
||||
"""Named ``norm -> activation -> conv`` projection used by current checkpoints."""
|
||||
|
||||
def __init__(self, channels: int, out_channels: int, zq_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.norm = ModulatedRMSNorm(channels, zq_dim)
|
||||
self.activation = nn.SiLU()
|
||||
self.conv = ReplicateTimeConv3d(channels, out_channels, kernel_size=3)
|
||||
|
||||
def forward(self, x: Tensor, zq: Tensor) -> Tensor:
|
||||
return self.conv(self.activation(self.norm(x, zq)))
|
||||
|
||||
|
||||
def _residual_stack(
|
||||
count: int, in_channels: int, channels: int, expand_ratio: int, zq_dim: int
|
||||
) -> nn.Sequential:
|
||||
"""``count`` blocks at width ``channels``; the first one narrows from ``in_channels``."""
|
||||
return nn.Sequential(
|
||||
*(
|
||||
ResidualBlock(
|
||||
in_channels if index == 0 else channels,
|
||||
channels,
|
||||
channels * expand_ratio,
|
||||
zq_dim,
|
||||
)
|
||||
for index in range(count)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _run_blocks(blocks: nn.Sequential, x: Tensor, zq: Tensor) -> Tensor:
|
||||
for block in blocks:
|
||||
x = block(x, zq)
|
||||
return x
|
||||
|
||||
|
||||
def _second_stage(
|
||||
upsample: nn.Module,
|
||||
blocks: nn.Sequential,
|
||||
head: Kandinsky6SRLatentUpscalerOutputHead,
|
||||
x: Tensor,
|
||||
zq: Tensor,
|
||||
) -> Tensor:
|
||||
"""``upsample -> blocks -> head``, with the head's first (norm) layer conditioned on ``zq``."""
|
||||
x = _run_blocks(blocks, upsample(x), zq)
|
||||
return head(x, zq)
|
||||
|
||||
|
||||
class X2Branch(nn.Module):
|
||||
"""Weights exclusive to the x2 path."""
|
||||
|
||||
def __init__(self, config: Kandinsky6SRLatentUpscalerEntryConfig) -> None:
|
||||
super().__init__()
|
||||
c = config.in_channels
|
||||
w1, w2, w3 = config.stage_channels
|
||||
er = config.expand_ratio
|
||||
self.adapter = _residual_stack(config.x2_adapter_blocks, w1, w1, er, c)
|
||||
self.finisher = X2Finisher(w1)
|
||||
self.mid_blocks = _residual_stack(config.num_mid_blocks, w1, w2, er, c)
|
||||
self.upsample = PXSUpsample(w2)
|
||||
self.blocks = _residual_stack(config.num_post_blocks, w2, w3, er, c)
|
||||
self.output_proj = Kandinsky6SRLatentUpscalerOutputHead(w3, c, zq_dim=c)
|
||||
|
||||
def forward(self, x: Tensor, zq: Tensor) -> Tensor:
|
||||
x = _run_blocks(self.adapter, x, zq)
|
||||
x = self.finisher(x)
|
||||
x = _run_blocks(self.mid_blocks, x, zq)
|
||||
return _second_stage(self.upsample, self.blocks, self.output_proj, x, zq)
|
||||
|
||||
|
||||
class Kandinsky6SRLatentUpscaler(nn.Module):
|
||||
"""One bank entry: ``[B, C, T, h, w]`` -> ``[B, C, T, s*h, s*w]`` for ``s`` = 4, or 2 with
|
||||
``enable_x2_entry``."""
|
||||
|
||||
def __init__(self, config: Kandinsky6SRLatentUpscalerEntryConfig) -> None:
|
||||
super().__init__()
|
||||
c = config.in_channels
|
||||
w1, w2, w3 = config.stage_channels
|
||||
er = config.expand_ratio
|
||||
self.input_proj = _stem(c, w1)
|
||||
# The 2x deep-supervision head of training: part of the checkpoint, never run at inference.
|
||||
self.mid_output_head = _mid_output_head(w2, c)
|
||||
self.output_proj = Kandinsky6SRLatentUpscalerOutputHead(w3, c, zq_dim=c)
|
||||
self.upsample_1 = PXSUpsample(w1)
|
||||
self.upsample_2 = PXSUpsample(w2)
|
||||
self.pre_blocks = _residual_stack(config.num_pre_blocks, w1, w1, er, c)
|
||||
self.mid_blocks = _residual_stack(config.num_mid_blocks, w1, w2, er, c)
|
||||
self.post_blocks = _residual_stack(config.num_post_blocks, w2, w3, er, c)
|
||||
self.mid_input_proj: nn.Sequential | None = None
|
||||
self.x2_branch: X2Branch | None = None
|
||||
if config.enable_x2_entry:
|
||||
self.mid_input_proj = _stem(c, config.hidden_channels)
|
||||
self.x2_branch = X2Branch(config)
|
||||
|
||||
def forward(self, z: Tensor, scale: int) -> Tensor:
|
||||
zq = z
|
||||
if scale == 2:
|
||||
if self.x2_branch is None or self.mid_input_proj is None:
|
||||
raise ValueError(
|
||||
"this latent-upscaler entry has no x2 path (enable_x2_entry=false)"
|
||||
)
|
||||
return self.x2_branch(self.mid_input_proj(z), zq)
|
||||
if scale != 4:
|
||||
raise ValueError(
|
||||
f"latent-upscaler entries upsample by 2 or 4, got scale={scale!r}"
|
||||
)
|
||||
x = _run_blocks(self.pre_blocks, self.input_proj(z), zq)
|
||||
x = _run_blocks(self.mid_blocks, self.upsample_1(x), zq)
|
||||
return _second_stage(self.upsample_2, self.post_blocks, self.output_proj, x, zq)
|
||||
|
||||
|
||||
class Kandinsky6SRLatentUpscalerBank(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
"""Select the checkpoint bank entry to upscale scaled KVAE latents in H/W."""
|
||||
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
models: Sequence[Mapping[str, Any]],
|
||||
scaling_factor: float = 1.0,
|
||||
scales: Sequence[int] = (2, 4),
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if not models:
|
||||
raise ValueError("latent upscaler config declares no `models` entries")
|
||||
entries = [
|
||||
Kandinsky6SRLatentUpscalerEntryConfig.from_dict(spec, index)
|
||||
for index, spec in enumerate(models)
|
||||
]
|
||||
entries_by_scale = {entry.target_scale: entry for entry in entries}
|
||||
if len(entries_by_scale) != len(entries):
|
||||
raise ValueError("latent upscaler `models` has duplicate target scales")
|
||||
self._scales = tuple(int(scale) for scale in scales)
|
||||
if len(set(self._scales)) != len(self._scales) or set(self._scales) != set(
|
||||
entries_by_scale
|
||||
):
|
||||
raise ValueError(
|
||||
"latent upscaler `scales` must match the target scales in `models`: "
|
||||
f"{self._scales} vs {tuple(entries_by_scale)}"
|
||||
)
|
||||
self.scaling_factor = float(scaling_factor)
|
||||
self._models = nn.ModuleList(
|
||||
[
|
||||
Kandinsky6SRLatentUpscaler(entries_by_scale[scale])
|
||||
for scale in self._scales
|
||||
]
|
||||
)
|
||||
|
||||
self.layer_names = []
|
||||
for index, entry in enumerate(self._models):
|
||||
self.layer_names += [
|
||||
f"_models.{index}.pre_blocks",
|
||||
f"_models.{index}.mid_blocks",
|
||||
f"_models.{index}.post_blocks",
|
||||
]
|
||||
if entry.x2_branch is not None:
|
||||
self.layer_names += [
|
||||
f"_models.{index}.x2_branch.adapter",
|
||||
f"_models.{index}.x2_branch.mid_blocks",
|
||||
f"_models.{index}.x2_branch.blocks",
|
||||
]
|
||||
|
||||
@property
|
||||
def scales(self) -> tuple[int, ...]:
|
||||
"""The upscale factors this bank serves, ascending."""
|
||||
return tuple(sorted(self._scales))
|
||||
|
||||
def for_scale(self, scale: int) -> nn.Module | None:
|
||||
"""Return the bank entry matching ``scale``, or ``None`` if the bank does not serve it."""
|
||||
scale = int(scale)
|
||||
return (
|
||||
self._models[self._scales.index(scale)] if scale in self._scales else None
|
||||
)
|
||||
|
||||
def upscale(self, latent: Tensor, *, scale: int) -> Tensor:
|
||||
"""Run the entry for ``scale`` on a *scaled* latent ``[B, C, T, h, w]``."""
|
||||
entry = self.for_scale(scale)
|
||||
if entry is None:
|
||||
raise ValueError(
|
||||
f"the latent-upscaler bank has no x{scale} entry (scales: {self.scales})"
|
||||
)
|
||||
return entry(latent, scale)
|
||||
|
||||
def forward(self, z: Tensor, *, scale: int) -> Tensor:
|
||||
return self.upscale(z, scale=scale)
|
||||
|
||||
|
||||
EntryClass = Kandinsky6SRLatentUpscalerBank
|
||||
+12
@@ -1,5 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
|
||||
"""Alias-free-resampling DSP primitives shared by the BigVGAN-lineage vocoders.
|
||||
|
||||
Generic, architecture-agnostic building blocks (Kaiser-windowed-sinc low-pass
|
||||
filtering, 2x up/down resampling, and the up-activate-down ``Activation1d``
|
||||
wrapper) -- not specific to any one model. Previously duplicated
|
||||
byte-for-byte between ``kandinsky6_audio_vae/alias_free.py`` and
|
||||
``minimax_h3_audio_vae/alias_free.py``; both ``BigVGANV2``/``BigVGAN``
|
||||
implementations now import ``Activation1d`` from here instead, matching this
|
||||
package's existing cross-model-shared-VAE-code convention (see ``common.py``
|
||||
in this same directory, reused by ``dac.py``, ``ltx_2_audio.py``,
|
||||
``wanvae.py``, ``hunyuanvae.py``, and others).
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
@@ -25,6 +25,7 @@ from sglang.multimodal_gen.runtime.cache.conditioning import (
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_decode_parallel_group_coordinator,
|
||||
get_decode_parallel_world_size,
|
||||
get_sp_group,
|
||||
get_sp_parallel_rank,
|
||||
get_sp_world_size,
|
||||
model_parallel_is_initialized,
|
||||
@@ -286,7 +287,8 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
"""
|
||||
Parallel version of tiled_decode that distributes both temporal and spatial computation across GPUs
|
||||
"""
|
||||
world_size, rank = get_sp_world_size(), get_sp_parallel_rank()
|
||||
sp_group = get_sp_group()
|
||||
world_size, rank = sp_group.world_size, sp_group.rank_in_group
|
||||
_, _, T, H, W = z.shape
|
||||
|
||||
tile_latent_min_height = (
|
||||
@@ -361,7 +363,7 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
torch.zeros(1, device=results.device, dtype=torch.int64)
|
||||
for _ in range(world_size)
|
||||
]
|
||||
dist.all_gather(all_sizes, local_size)
|
||||
dist.all_gather(all_sizes, local_size, group=sp_group.device_group)
|
||||
max_size = max(size.item() for size in all_sizes)
|
||||
|
||||
padded_results = torch.zeros(
|
||||
@@ -375,8 +377,12 @@ class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
.repeat(world_size, *[1] * len(padded_results.shape))
|
||||
.contiguous()
|
||||
)
|
||||
all_gather_single(gathered_results, padded_results)
|
||||
dist.all_gather_object(gathered_dim_metadata, local_dim_metadata)
|
||||
all_gather_single(
|
||||
gathered_results.view(-1), padded_results, group=sp_group.device_group
|
||||
)
|
||||
dist.all_gather_object(
|
||||
gathered_dim_metadata, local_dim_metadata, group=sp_group.cpu_group
|
||||
)
|
||||
gathered_dim_metadata = cast(list[list[torch.Size]], gathered_dim_metadata)
|
||||
|
||||
data: list = [
|
||||
|
||||
@@ -35,6 +35,7 @@ from sglang.multimodal_gen.runtime.layers.parallel_conv import (
|
||||
disable_spatial_parallel_decode,
|
||||
gather_and_trim_height,
|
||||
gather_variable_height,
|
||||
spatial_parallel_decode_disabled,
|
||||
split_height_for_parallel_decode,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.common import (
|
||||
@@ -125,7 +126,11 @@ class HunyuanVAEAttention(nn.Module):
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, hidden_states: torch.Tensor, attention_mask: torch.Tensor | None = None
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
attention_mask: torch.Tensor | None = None,
|
||||
*,
|
||||
num_frames: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
residual = hidden_states
|
||||
|
||||
@@ -145,10 +150,30 @@ class HunyuanVAEAttention(nn.Module):
|
||||
key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
|
||||
value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
|
||||
|
||||
# Perform scaled dot-product attention
|
||||
hidden_states = F.scaled_dot_product_attention(
|
||||
query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
|
||||
)
|
||||
if num_frames is None:
|
||||
hidden_states = F.scaled_dot_product_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask=attention_mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=False,
|
||||
)
|
||||
else:
|
||||
# each frame sees all spatial tokens in itself and earlier frames
|
||||
# slicing the key prefix avoids a quadratic whole-video mask
|
||||
tokens_per_frame = sequence_length // num_frames
|
||||
hidden_states = torch.cat(
|
||||
[
|
||||
F.scaled_dot_product_attention(
|
||||
query[:, :, start : start + tokens_per_frame],
|
||||
key[:, :, : start + tokens_per_frame],
|
||||
value[:, :, : start + tokens_per_frame],
|
||||
)
|
||||
for start in range(0, sequence_length, tokens_per_frame)
|
||||
],
|
||||
dim=2,
|
||||
)
|
||||
|
||||
# Reshape back
|
||||
hidden_states = hidden_states.transpose(1, 2).reshape(
|
||||
@@ -278,6 +303,8 @@ class HunyuanVideoUpsampleCausal3D(nn.Module):
|
||||
else:
|
||||
hidden_states = first_frame
|
||||
|
||||
# release interpolation buffers before allocating convolution workspace
|
||||
del first_frame, other_frames
|
||||
hidden_states = self.conv(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
@@ -422,19 +449,25 @@ class HunyuanVideoMidBlock3D(nn.Module):
|
||||
self, attn: HunyuanVAEAttention, hidden_states: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
heights = None
|
||||
if self.spatial_parallel:
|
||||
spatial_parallel = (
|
||||
self.spatial_parallel and not spatial_parallel_decode_disabled()
|
||||
)
|
||||
if spatial_parallel:
|
||||
hidden_states, heights = gather_variable_height(hidden_states)
|
||||
|
||||
batch_size, num_channels, num_frames, height, width = hidden_states.shape
|
||||
hidden_states = hidden_states.permute(0, 2, 3, 4, 1).flatten(1, 3)
|
||||
attention_mask = prepare_causal_attention_mask(
|
||||
num_frames,
|
||||
height * width,
|
||||
hidden_states.dtype,
|
||||
hidden_states.device,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
hidden_states = attn(hidden_states, attention_mask=attention_mask)
|
||||
if spatial_parallel:
|
||||
hidden_states = attn(hidden_states, num_frames=num_frames)
|
||||
else:
|
||||
attention_mask = prepare_causal_attention_mask(
|
||||
num_frames,
|
||||
height * width,
|
||||
hidden_states.dtype,
|
||||
hidden_states.device,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
hidden_states = attn(hidden_states, attention_mask=attention_mask)
|
||||
hidden_states = hidden_states.unflatten(1, (num_frames, height, width)).permute(
|
||||
0, 4, 1, 2, 3
|
||||
)
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Kandinsky 6 mel VAE and waveform vocoder.
|
||||
|
||||
The vae.*, vocoder.* and mel_converter.* names match the official checkpoint,
|
||||
allowing the generic VAELoader to load the full tree without key remapping."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
|
||||
Kandinsky6AudioVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.bigvgan import (
|
||||
BigVGANV2,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.mel_converter import (
|
||||
MelConverter,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.mmaudio_vae import (
|
||||
MMAudioVAE,
|
||||
)
|
||||
|
||||
|
||||
class Kandinsky6AudioVAE(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
"""MMAudio mel codec and BigVGAN vocoder with checkpoint-compatible nesting.
|
||||
|
||||
decode: [B, embed_dim, A] -> [B, num_mels, T]
|
||||
vocode: [B, num_mels, T] -> [B, 1, samples]
|
||||
wrapped_decode and wrapped_encode compose the codec and waveform stages."""
|
||||
|
||||
# official checkpoint class name
|
||||
_aliases = ["MMAudioVAE"]
|
||||
|
||||
# Decode-only small model relative to the DiT; no benefit from DiT-group
|
||||
# offload tuning knobs. Mirrors MiniMaxH3AudioVAE's choice.
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
|
||||
def __init__(self, config: Kandinsky6AudioVAEConfig) -> None:
|
||||
super().__init__()
|
||||
arch = config.arch_config
|
||||
self.scaling_factor = float(arch.scaling_factor)
|
||||
self.mean_value = float(arch.mean_value)
|
||||
# audio latent stride; frame-count alignment is owned by the pipeline config
|
||||
self.downsample_factor = 1024
|
||||
|
||||
self.mel_converter = MelConverter(
|
||||
sampling_rate=44100,
|
||||
n_fft=2048,
|
||||
num_mels=128,
|
||||
hop_size=512,
|
||||
win_size=2048,
|
||||
fmin=0,
|
||||
fmax=22050,
|
||||
)
|
||||
# checkpoint weights include the encoder regardless of need_vae_encoder
|
||||
self.vae = MMAudioVAE(mode=arch.mode, need_encoder=True)
|
||||
# normalize random initialization before loading; checkpoint weights are
|
||||
# already normalized and must never be renormalized
|
||||
self.vae.remove_weight_norm()
|
||||
self.vocoder = BigVGANV2(dict(arch.vocoder_config))
|
||||
# strip PyTorch weight_norm before loading plain .weight checkpoint keys;
|
||||
# this differs from MMAudio's RMS renormalization above
|
||||
self.vocoder.remove_weight_norm()
|
||||
|
||||
# resolution containers are not called; hooks belong on their blocks
|
||||
self.layer_names = (
|
||||
[
|
||||
f"{path}.{index}.{group}"
|
||||
for path, levels in (
|
||||
("vae.encoder.down", self.vae.encoder.down),
|
||||
("vae.decoder.up", self.vae.decoder.up),
|
||||
)
|
||||
for index in range(len(levels))
|
||||
for group in ("block", "attn")
|
||||
]
|
||||
+ [f"vocoder.ups.{index}" for index in range(len(self.vocoder.ups))]
|
||||
+ ["vocoder.resblocks"]
|
||||
)
|
||||
|
||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
||||
"""latents: [B, embed_dim, A] (1D-conv channel-first) -> mel: [B, num_mels, T]."""
|
||||
return self.vae.decode(latents, unnormalize_output=True)
|
||||
|
||||
def vocode(self, mel: torch.Tensor) -> torch.Tensor:
|
||||
"""mel: [B, num_mels, T] -> waveform: [B, 1, samples]."""
|
||||
# mel denormalization returns fp32; cast to the loaded vocoder's dtype
|
||||
vocoder_dtype = next(self.vocoder.parameters()).dtype
|
||||
return self.vocoder(mel.to(dtype=vocoder_dtype))
|
||||
|
||||
def wrapped_decode(self, latents: torch.Tensor) -> torch.Tensor:
|
||||
"""latents -> waveform in one call, matching the diffusers
|
||||
reference's MMAudioVAE.wrapped_decode."""
|
||||
return self.vocode(self.decode(latents))
|
||||
|
||||
def encode_audio(self, waveform: torch.Tensor):
|
||||
"""waveform -> mel -> VAE posterior. See MelConverter's module
|
||||
docstring: not exercised by / verified against the current
|
||||
(decode-only) inference path."""
|
||||
mel = self.mel_converter(waveform)
|
||||
return self.vae.encode(mel)
|
||||
|
||||
def wrapped_encode(self, waveform: torch.Tensor) -> torch.Tensor:
|
||||
"""waveform -> mean audio latent in one call, matching the
|
||||
diffusers reference's MMAudioVAE.wrapped_encode."""
|
||||
return self.encode_audio(waveform).mean
|
||||
|
||||
def remove_weight_norm(self) -> Kandinsky6AudioVAE:
|
||||
"""Idempotent MMAudio weight normalization, normally applied before loading."""
|
||||
self.vae.remove_weight_norm()
|
||||
return self
|
||||
|
||||
|
||||
EntryClass = Kandinsky6AudioVAE
|
||||
|
||||
__all__ = ["Kandinsky6AudioVAE"]
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
This directory contains code adapted from the following MIT-licensed projects:
|
||||
|
||||
- NVIDIA BigVGAN, bigvgan.py
|
||||
https://github.com/NVIDIA/BigVGAN
|
||||
Copyright (c) 2024 NVIDIA CORPORATION.
|
||||
|
||||
- MMAudio, mmaudio_vae.py
|
||||
https://github.com/hkchengrex/MMAudio
|
||||
Copyright (c) 2024 Sony Research Inc.
|
||||
|
||||
The following license applies to those portions of the code:
|
||||
|
||||
MIT License
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Audio codec building blocks; the registry entry lives in ../kandinsky6_audio.py."""
|
||||
|
||||
from .bigvgan import BigVGANV2
|
||||
from .mel_converter import MelConverter
|
||||
from .mmaudio_vae import DiagonalGaussianDistribution, MMAudioVAE
|
||||
|
||||
__all__ = ["BigVGANV2", "DiagonalGaussianDistribution", "MMAudioVAE", "MelConverter"]
|
||||
@@ -0,0 +1,350 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright (c) 2024 NVIDIA CORPORATION.
|
||||
# Adapted from NVIDIA BigVGAN-v2 (MIT license) via FastVideo's
|
||||
# fastvideo/models/audio/bigvgan.py. The CUDA activation kernel is
|
||||
# intentionally excluded; this module uses the portable PyTorch path for
|
||||
# deterministic loading and parity across platforms.
|
||||
# Original license terms are retained in THIRD_PARTY_LICENSES.
|
||||
#
|
||||
# This top-level generator class (BigVGANV2) and its Snake/SnakeBeta/
|
||||
# AMPBlock1/AMPBlock2 building blocks are freshly ported here rather than
|
||||
# reused from this codebase's sibling ``minimax_h3_audio_vae/bigvgan.py`` --
|
||||
# that module's ``BigVGAN``/``AMPBlock1`` only support resblock="1" and
|
||||
# activation="snakebeta" (MiniMax's own DAC-vocoder config never needs more),
|
||||
# while this implementation supports either resblock variant or activation. The
|
||||
# alias-free resampling primitives (LowPassFilter1d/UpSample1d/DownSample1d/
|
||||
# Activation1d) ARE architecture-agnostic DSP building blocks shared with
|
||||
# that same sibling module, though -- both import ``Activation1d`` from
|
||||
# ``runtime/models/vaes/alias_free.py`` (sibling of this package, alongside
|
||||
# ``common.py``) instead of each keeping its own copy.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn import Conv1d, ConvTranspose1d
|
||||
from torch.nn.utils.parametrizations import weight_norm
|
||||
from torch.nn.utils.parametrize import remove_parametrizations
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.vaes.alias_free import Activation1d
|
||||
|
||||
|
||||
class AttrDict(dict):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.__dict__ = self
|
||||
|
||||
|
||||
def get_padding(kernel_size: int, dilation: int = 1) -> int:
|
||||
return int((kernel_size * dilation - dilation) / 2)
|
||||
|
||||
|
||||
def init_weights(module: nn.Module, mean: float = 0.0, std: float = 0.01) -> None:
|
||||
if "Conv" in module.__class__.__name__:
|
||||
module.weight.data.normal_(mean, std)
|
||||
|
||||
|
||||
class Snake(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_features: int,
|
||||
alpha: float = 1.0,
|
||||
alpha_trainable: bool = True,
|
||||
alpha_logscale: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.in_features = in_features
|
||||
self.alpha_logscale = alpha_logscale
|
||||
initial = (
|
||||
torch.zeros(in_features) * alpha
|
||||
if alpha_logscale
|
||||
else torch.ones(in_features) * alpha
|
||||
)
|
||||
self.alpha = nn.Parameter(initial, requires_grad=alpha_trainable)
|
||||
self.no_div_by_zero = 1e-9
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
|
||||
if self.alpha_logscale:
|
||||
alpha = torch.exp(alpha)
|
||||
return x + (1.0 / (alpha + self.no_div_by_zero)) * torch.pow(
|
||||
torch.sin(x * alpha), 2
|
||||
)
|
||||
|
||||
|
||||
class SnakeBeta(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_features: int,
|
||||
alpha: float = 1.0,
|
||||
alpha_trainable: bool = True,
|
||||
alpha_logscale: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.in_features = in_features
|
||||
self.alpha_logscale = alpha_logscale
|
||||
initial = (
|
||||
torch.zeros(in_features) * alpha
|
||||
if alpha_logscale
|
||||
else torch.ones(in_features) * alpha
|
||||
)
|
||||
self.alpha = nn.Parameter(initial.clone(), requires_grad=alpha_trainable)
|
||||
self.beta = nn.Parameter(initial.clone(), requires_grad=alpha_trainable)
|
||||
self.no_div_by_zero = 1e-9
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
alpha = self.alpha.unsqueeze(0).unsqueeze(-1)
|
||||
beta = self.beta.unsqueeze(0).unsqueeze(-1)
|
||||
if self.alpha_logscale:
|
||||
alpha = torch.exp(alpha)
|
||||
beta = torch.exp(beta)
|
||||
return x + (1.0 / (beta + self.no_div_by_zero)) * torch.pow(
|
||||
torch.sin(x * alpha), 2
|
||||
)
|
||||
|
||||
|
||||
def _activation(name: str, channels: int, logscale: bool) -> Activation1d:
|
||||
if name == "snake":
|
||||
activation = Snake(channels, alpha_logscale=logscale)
|
||||
elif name == "snakebeta":
|
||||
activation = SnakeBeta(channels, alpha_logscale=logscale)
|
||||
else:
|
||||
raise ValueError(f"Unsupported BigVGAN activation: {name}")
|
||||
return Activation1d(activation)
|
||||
|
||||
|
||||
class AMPBlock1(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AttrDict,
|
||||
channels: int,
|
||||
kernel_size: int = 3,
|
||||
dilation: tuple[int, ...] = (1, 3, 5),
|
||||
activation: str = "snake",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.convs1 = nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
stride=1,
|
||||
dilation=rate,
|
||||
padding=get_padding(kernel_size, rate),
|
||||
)
|
||||
)
|
||||
for rate in dilation
|
||||
]
|
||||
)
|
||||
self.convs1.apply(init_weights)
|
||||
self.convs2 = nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
stride=1,
|
||||
dilation=1,
|
||||
padding=get_padding(kernel_size, 1),
|
||||
)
|
||||
)
|
||||
for _ in dilation
|
||||
]
|
||||
)
|
||||
self.convs2.apply(init_weights)
|
||||
self.activations = nn.ModuleList(
|
||||
[
|
||||
_activation(activation, channels, config.snake_logscale)
|
||||
for _ in range(len(self.convs1) + len(self.convs2))
|
||||
]
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
acts1, acts2 = self.activations[::2], self.activations[1::2]
|
||||
for conv1, conv2, act1, act2 in zip(
|
||||
self.convs1, self.convs2, acts1, acts2, strict=True
|
||||
):
|
||||
residual = conv2(act2(conv1(act1(x))))
|
||||
x = residual + x
|
||||
return x
|
||||
|
||||
def remove_weight_norm(self) -> None:
|
||||
for layer in self.convs1:
|
||||
remove_parametrizations(layer, "weight")
|
||||
for layer in self.convs2:
|
||||
remove_parametrizations(layer, "weight")
|
||||
|
||||
|
||||
class AMPBlock2(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: AttrDict,
|
||||
channels: int,
|
||||
kernel_size: int = 3,
|
||||
dilation: tuple[int, ...] = (1, 3, 5),
|
||||
activation: str = "snake",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.convs = nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
Conv1d(
|
||||
channels,
|
||||
channels,
|
||||
kernel_size,
|
||||
stride=1,
|
||||
dilation=rate,
|
||||
padding=get_padding(kernel_size, rate),
|
||||
)
|
||||
)
|
||||
for rate in dilation
|
||||
]
|
||||
)
|
||||
self.convs.apply(init_weights)
|
||||
self.activations = nn.ModuleList(
|
||||
[
|
||||
_activation(activation, channels, config.snake_logscale)
|
||||
for _ in self.convs
|
||||
]
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
for conv, activation in zip(self.convs, self.activations, strict=True):
|
||||
x = conv(activation(x)) + x
|
||||
return x
|
||||
|
||||
def remove_weight_norm(self) -> None:
|
||||
for layer in self.convs:
|
||||
remove_parametrizations(layer, "weight")
|
||||
|
||||
|
||||
class BigVGANV2(nn.Module):
|
||||
"""BigVGAN-v2 using checkpoint vocoder_config and the portable PyTorch path."""
|
||||
|
||||
def __init__(self, config: dict[str, Any]) -> None:
|
||||
super().__init__()
|
||||
config = dict(config)
|
||||
config.pop("_class_name", None)
|
||||
config.pop("architectures", None)
|
||||
weight_norm_removed = bool(config.pop("weight_norm_removed", False))
|
||||
self.config = AttrDict(config)
|
||||
if self.config.get("use_cuda_kernel", False):
|
||||
raise ValueError(
|
||||
"Kandinsky6 BigVGANV2 supports only the portable PyTorch path"
|
||||
)
|
||||
self.config["use_cuda_kernel"] = False
|
||||
self.num_kernels = len(self.config.resblock_kernel_sizes)
|
||||
self.num_upsamples = len(self.config.upsample_rates)
|
||||
self.conv_pre = weight_norm(
|
||||
Conv1d(
|
||||
self.config.num_mels,
|
||||
self.config.upsample_initial_channel,
|
||||
7,
|
||||
1,
|
||||
padding=3,
|
||||
)
|
||||
)
|
||||
if self.config.resblock == "1":
|
||||
block_class = AMPBlock1
|
||||
elif self.config.resblock == "2":
|
||||
block_class = AMPBlock2
|
||||
else:
|
||||
raise ValueError(f"Unsupported BigVGAN resblock: {self.config.resblock}")
|
||||
|
||||
self.ups = nn.ModuleList()
|
||||
for index, (rate, kernel) in enumerate(
|
||||
zip(
|
||||
self.config.upsample_rates,
|
||||
self.config.upsample_kernel_sizes,
|
||||
strict=True,
|
||||
)
|
||||
):
|
||||
self.ups.append(
|
||||
nn.ModuleList(
|
||||
[
|
||||
weight_norm(
|
||||
ConvTranspose1d(
|
||||
self.config.upsample_initial_channel // (2**index),
|
||||
self.config.upsample_initial_channel
|
||||
// (2 ** (index + 1)),
|
||||
kernel,
|
||||
rate,
|
||||
padding=(kernel - rate) // 2,
|
||||
)
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
self.resblocks = nn.ModuleList()
|
||||
for index in range(len(self.ups)):
|
||||
channels = self.config.upsample_initial_channel // (2 ** (index + 1))
|
||||
for kernel, dilation in zip(
|
||||
self.config.resblock_kernel_sizes,
|
||||
self.config.resblock_dilation_sizes,
|
||||
strict=True,
|
||||
):
|
||||
self.resblocks.append(
|
||||
block_class(
|
||||
self.config,
|
||||
channels,
|
||||
kernel,
|
||||
tuple(dilation),
|
||||
activation=self.config.activation,
|
||||
)
|
||||
)
|
||||
|
||||
channels = self.config.upsample_initial_channel // (2 ** len(self.ups))
|
||||
self.activation_post = _activation(
|
||||
self.config.activation, channels, self.config.snake_logscale
|
||||
)
|
||||
self.use_bias_at_final = self.config.get("use_bias_at_final", True)
|
||||
self.conv_post = weight_norm(
|
||||
Conv1d(channels, 1, 7, 1, padding=3, bias=self.use_bias_at_final)
|
||||
)
|
||||
for upsampler in self.ups:
|
||||
upsampler.apply(init_weights)
|
||||
self.conv_post.apply(init_weights)
|
||||
self.use_tanh_at_final = self.config.get("use_tanh_at_final", True)
|
||||
if weight_norm_removed:
|
||||
self.remove_weight_norm()
|
||||
|
||||
def forward(self, mel: torch.Tensor) -> torch.Tensor:
|
||||
hidden = self.conv_pre(mel)
|
||||
for index in range(self.num_upsamples):
|
||||
for upsampler in self.ups[index]:
|
||||
hidden = upsampler(hidden)
|
||||
accumulated = None
|
||||
for kernel in range(self.num_kernels):
|
||||
block_output = self.resblocks[index * self.num_kernels + kernel](hidden)
|
||||
if accumulated is None:
|
||||
accumulated = block_output
|
||||
else:
|
||||
# Preserve BigVGAN's published sequential accumulation
|
||||
# order. Tree reduction drifts through later nonlinear
|
||||
# upsampling stages with the full checkpoint.
|
||||
accumulated += block_output
|
||||
assert accumulated is not None
|
||||
hidden = accumulated / self.num_kernels
|
||||
hidden = self.conv_post(self.activation_post(hidden))
|
||||
if self.use_tanh_at_final:
|
||||
return torch.tanh(hidden)
|
||||
return torch.clamp(hidden, min=-1.0, max=1.0)
|
||||
|
||||
def remove_weight_norm(self) -> None:
|
||||
try:
|
||||
for upsamplers in self.ups:
|
||||
for upsampler in upsamplers:
|
||||
remove_parametrizations(upsampler, "weight")
|
||||
for block in self.resblocks:
|
||||
block.remove_weight_norm()
|
||||
remove_parametrizations(self.conv_pre, "weight")
|
||||
remove_parametrizations(self.conv_post, "weight")
|
||||
except ValueError:
|
||||
# Idempotent for pipeline setup and converted checkpoints.
|
||||
return
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Adapted from FastVideo's fastvideo/models/audio/kandinsky6_audio_vae.py.
|
||||
"""Checkpoint-compatible waveform-to-log-mel frontend.
|
||||
|
||||
Included for state-dict completeness; generation only decodes audio latents,
|
||||
so waveform encoding has not been validated end-to-end."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class MelConverter(nn.Module):
|
||||
"""Waveform -> log-mel-spectrogram STFT frontend. See module docstring:
|
||||
not exercised by the current (decode-only) inference path."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
sampling_rate: int = 44100,
|
||||
n_fft: int = 2048,
|
||||
num_mels: int = 128,
|
||||
hop_size: int = 512,
|
||||
win_size: int = 2048,
|
||||
fmin: int = 0,
|
||||
fmax: int | None = 22050,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.n_fft = n_fft
|
||||
self.hop_size = hop_size
|
||||
self.win_size = win_size
|
||||
self.register_buffer(
|
||||
"hann_window", torch.hann_window(win_size), persistent=True
|
||||
)
|
||||
try:
|
||||
from librosa.filters import mel as librosa_mel_fn
|
||||
|
||||
mel_basis = torch.from_numpy(
|
||||
librosa_mel_fn(
|
||||
sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax
|
||||
)
|
||||
).float()
|
||||
except Exception:
|
||||
# generation does not use this buffer; the checkpoint supplies its real values
|
||||
mel_basis = torch.zeros(num_mels, n_fft // 2 + 1)
|
||||
self.register_buffer("mel_basis", mel_basis, persistent=True)
|
||||
|
||||
def forward(self, waveform: torch.Tensor, center: bool = False) -> torch.Tensor:
|
||||
waveform = waveform.clamp(min=-1.0, max=1.0)
|
||||
pad = (self.n_fft - self.hop_size) // 2
|
||||
padded = F.pad(waveform.unsqueeze(1), (pad, pad), mode="reflect").squeeze(1)
|
||||
spec = torch.stft(
|
||||
padded,
|
||||
self.n_fft,
|
||||
hop_length=self.hop_size,
|
||||
win_length=self.win_size,
|
||||
window=self.hann_window,
|
||||
center=center,
|
||||
pad_mode="reflect",
|
||||
normalized=False,
|
||||
onesided=True,
|
||||
return_complex=True,
|
||||
)
|
||||
spec = torch.view_as_real(spec)
|
||||
magnitude = torch.sqrt(spec.pow(2).sum(-1) + 1e-9).float()
|
||||
mel = torch.matmul(self.mel_basis, magnitude)
|
||||
return torch.log(torch.clamp(mel, min=1e-5))
|
||||
@@ -0,0 +1,854 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Adapted from MMAudio (https://github.com/hkchengrex/MMAudio), MIT License,
|
||||
# Copyright (c) 2024 Sony Research Inc., via FastVideo's
|
||||
# fastvideo/models/audio/mmaudio_vae.py. This is the mel-spectrogram
|
||||
# <-> latent 1D-conv VAE codec nested at ``Kandinsky6AudioVAE.vae``
|
||||
# (see kandinsky6_audio.py in the parent directory).
|
||||
# Original license terms are retained in THIRD_PARTY_LICENSES.
|
||||
#
|
||||
# Ported verbatim except: FastVideo's ``MMAudioVAE.load_weights()`` (and its
|
||||
# ``fastvideo.models.loader.weight_utils.default_weight_loader`` import) is
|
||||
# dropped -- sglang's generic VAELoader never calls a per-submodule
|
||||
# ``load_weights()`` hook, it does a single plain ``nn.Module.load_state_dict``
|
||||
# over the whole ``Kandinsky6AudioVAE`` tree, so that method would be dead code
|
||||
# here.
|
||||
"""Native 1D audio VAE used by MMAudio (mel-spectrogram <-> latent codec)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DATA_MEAN_80D = [
|
||||
-1.6058,
|
||||
-1.3676,
|
||||
-1.2520,
|
||||
-1.2453,
|
||||
-1.2078,
|
||||
-1.2224,
|
||||
-1.2419,
|
||||
-1.2439,
|
||||
-1.2922,
|
||||
-1.2927,
|
||||
-1.3170,
|
||||
-1.3543,
|
||||
-1.3401,
|
||||
-1.3836,
|
||||
-1.3907,
|
||||
-1.3912,
|
||||
-1.4313,
|
||||
-1.4152,
|
||||
-1.4527,
|
||||
-1.4728,
|
||||
-1.4568,
|
||||
-1.5101,
|
||||
-1.5051,
|
||||
-1.5172,
|
||||
-1.5623,
|
||||
-1.5373,
|
||||
-1.5746,
|
||||
-1.5687,
|
||||
-1.6032,
|
||||
-1.6131,
|
||||
-1.6081,
|
||||
-1.6331,
|
||||
-1.6489,
|
||||
-1.6489,
|
||||
-1.6700,
|
||||
-1.6738,
|
||||
-1.6953,
|
||||
-1.6969,
|
||||
-1.7048,
|
||||
-1.7280,
|
||||
-1.7361,
|
||||
-1.7495,
|
||||
-1.7658,
|
||||
-1.7814,
|
||||
-1.7889,
|
||||
-1.8064,
|
||||
-1.8221,
|
||||
-1.8377,
|
||||
-1.8417,
|
||||
-1.8643,
|
||||
-1.8857,
|
||||
-1.8929,
|
||||
-1.9173,
|
||||
-1.9379,
|
||||
-1.9531,
|
||||
-1.9673,
|
||||
-1.9824,
|
||||
-2.0042,
|
||||
-2.0215,
|
||||
-2.0436,
|
||||
-2.0766,
|
||||
-2.1064,
|
||||
-2.1418,
|
||||
-2.1855,
|
||||
-2.2319,
|
||||
-2.2767,
|
||||
-2.3161,
|
||||
-2.3572,
|
||||
-2.3954,
|
||||
-2.4282,
|
||||
-2.4659,
|
||||
-2.5072,
|
||||
-2.5552,
|
||||
-2.6074,
|
||||
-2.6584,
|
||||
-2.7107,
|
||||
-2.7634,
|
||||
-2.8266,
|
||||
-2.8981,
|
||||
-2.9673,
|
||||
]
|
||||
|
||||
DATA_STD_80D = [
|
||||
1.0291,
|
||||
1.0411,
|
||||
1.0043,
|
||||
0.9820,
|
||||
0.9677,
|
||||
0.9543,
|
||||
0.9450,
|
||||
0.9392,
|
||||
0.9343,
|
||||
0.9297,
|
||||
0.9276,
|
||||
0.9263,
|
||||
0.9242,
|
||||
0.9254,
|
||||
0.9232,
|
||||
0.9281,
|
||||
0.9263,
|
||||
0.9315,
|
||||
0.9274,
|
||||
0.9247,
|
||||
0.9277,
|
||||
0.9199,
|
||||
0.9188,
|
||||
0.9194,
|
||||
0.9160,
|
||||
0.9161,
|
||||
0.9146,
|
||||
0.9161,
|
||||
0.9100,
|
||||
0.9095,
|
||||
0.9145,
|
||||
0.9076,
|
||||
0.9066,
|
||||
0.9095,
|
||||
0.9032,
|
||||
0.9043,
|
||||
0.9038,
|
||||
0.9011,
|
||||
0.9019,
|
||||
0.9010,
|
||||
0.8984,
|
||||
0.8983,
|
||||
0.8986,
|
||||
0.8961,
|
||||
0.8962,
|
||||
0.8978,
|
||||
0.8962,
|
||||
0.8973,
|
||||
0.8993,
|
||||
0.8976,
|
||||
0.8995,
|
||||
0.9016,
|
||||
0.8982,
|
||||
0.8972,
|
||||
0.8974,
|
||||
0.8949,
|
||||
0.8940,
|
||||
0.8947,
|
||||
0.8936,
|
||||
0.8939,
|
||||
0.8951,
|
||||
0.8956,
|
||||
0.9017,
|
||||
0.9167,
|
||||
0.9436,
|
||||
0.9690,
|
||||
1.0003,
|
||||
1.0225,
|
||||
1.0381,
|
||||
1.0491,
|
||||
1.0545,
|
||||
1.0604,
|
||||
1.0761,
|
||||
1.0929,
|
||||
1.1089,
|
||||
1.1196,
|
||||
1.1176,
|
||||
1.1156,
|
||||
1.1117,
|
||||
1.1070,
|
||||
]
|
||||
|
||||
DATA_MEAN_128D = [
|
||||
-3.3462,
|
||||
-2.6723,
|
||||
-2.4893,
|
||||
-2.3143,
|
||||
-2.2664,
|
||||
-2.3317,
|
||||
-2.1802,
|
||||
-2.4006,
|
||||
-2.2357,
|
||||
-2.4597,
|
||||
-2.3717,
|
||||
-2.4690,
|
||||
-2.5142,
|
||||
-2.4919,
|
||||
-2.6610,
|
||||
-2.5047,
|
||||
-2.7483,
|
||||
-2.5926,
|
||||
-2.7462,
|
||||
-2.7033,
|
||||
-2.7386,
|
||||
-2.8112,
|
||||
-2.7502,
|
||||
-2.9594,
|
||||
-2.7473,
|
||||
-3.0035,
|
||||
-2.8891,
|
||||
-2.9922,
|
||||
-2.9856,
|
||||
-3.0157,
|
||||
-3.1191,
|
||||
-2.9893,
|
||||
-3.1718,
|
||||
-3.0745,
|
||||
-3.1879,
|
||||
-3.2310,
|
||||
-3.1424,
|
||||
-3.2296,
|
||||
-3.2791,
|
||||
-3.2782,
|
||||
-3.2756,
|
||||
-3.3134,
|
||||
-3.3509,
|
||||
-3.3750,
|
||||
-3.3951,
|
||||
-3.3698,
|
||||
-3.4505,
|
||||
-3.4509,
|
||||
-3.5089,
|
||||
-3.4647,
|
||||
-3.5536,
|
||||
-3.5788,
|
||||
-3.5867,
|
||||
-3.6036,
|
||||
-3.6400,
|
||||
-3.6747,
|
||||
-3.7072,
|
||||
-3.7279,
|
||||
-3.7283,
|
||||
-3.7795,
|
||||
-3.8259,
|
||||
-3.8447,
|
||||
-3.8663,
|
||||
-3.9182,
|
||||
-3.9605,
|
||||
-3.9861,
|
||||
-4.0105,
|
||||
-4.0373,
|
||||
-4.0762,
|
||||
-4.1121,
|
||||
-4.1488,
|
||||
-4.1874,
|
||||
-4.2461,
|
||||
-4.3170,
|
||||
-4.3639,
|
||||
-4.4452,
|
||||
-4.5282,
|
||||
-4.6297,
|
||||
-4.7019,
|
||||
-4.7960,
|
||||
-4.8700,
|
||||
-4.9507,
|
||||
-5.0303,
|
||||
-5.0866,
|
||||
-5.1634,
|
||||
-5.2342,
|
||||
-5.3242,
|
||||
-5.4053,
|
||||
-5.4927,
|
||||
-5.5712,
|
||||
-5.6464,
|
||||
-5.7052,
|
||||
-5.7619,
|
||||
-5.8410,
|
||||
-5.9188,
|
||||
-6.0103,
|
||||
-6.0955,
|
||||
-6.1673,
|
||||
-6.2362,
|
||||
-6.3120,
|
||||
-6.3926,
|
||||
-6.4797,
|
||||
-6.5565,
|
||||
-6.6511,
|
||||
-6.8130,
|
||||
-6.9961,
|
||||
-7.1275,
|
||||
-7.2457,
|
||||
-7.3576,
|
||||
-7.4663,
|
||||
-7.6136,
|
||||
-7.7469,
|
||||
-7.8815,
|
||||
-8.0132,
|
||||
-8.1515,
|
||||
-8.3071,
|
||||
-8.4722,
|
||||
-8.7418,
|
||||
-9.3975,
|
||||
-9.6628,
|
||||
-9.7671,
|
||||
-9.8863,
|
||||
-9.9992,
|
||||
-10.0860,
|
||||
-10.1709,
|
||||
-10.5418,
|
||||
-11.2795,
|
||||
-11.3861,
|
||||
]
|
||||
|
||||
DATA_STD_128D = [
|
||||
2.3804,
|
||||
2.4368,
|
||||
2.3772,
|
||||
2.3145,
|
||||
2.2803,
|
||||
2.2510,
|
||||
2.2316,
|
||||
2.2083,
|
||||
2.1996,
|
||||
2.1835,
|
||||
2.1769,
|
||||
2.1659,
|
||||
2.1631,
|
||||
2.1618,
|
||||
2.1540,
|
||||
2.1606,
|
||||
2.1571,
|
||||
2.1567,
|
||||
2.1612,
|
||||
2.1579,
|
||||
2.1679,
|
||||
2.1683,
|
||||
2.1634,
|
||||
2.1557,
|
||||
2.1668,
|
||||
2.1518,
|
||||
2.1415,
|
||||
2.1449,
|
||||
2.1406,
|
||||
2.1350,
|
||||
2.1313,
|
||||
2.1415,
|
||||
2.1281,
|
||||
2.1352,
|
||||
2.1219,
|
||||
2.1182,
|
||||
2.1327,
|
||||
2.1195,
|
||||
2.1137,
|
||||
2.1080,
|
||||
2.1179,
|
||||
2.1036,
|
||||
2.1087,
|
||||
2.1036,
|
||||
2.1015,
|
||||
2.1068,
|
||||
2.0975,
|
||||
2.0991,
|
||||
2.0902,
|
||||
2.1015,
|
||||
2.0857,
|
||||
2.0920,
|
||||
2.0893,
|
||||
2.0897,
|
||||
2.0910,
|
||||
2.0881,
|
||||
2.0925,
|
||||
2.0873,
|
||||
2.0960,
|
||||
2.0900,
|
||||
2.0957,
|
||||
2.0958,
|
||||
2.0978,
|
||||
2.0936,
|
||||
2.0886,
|
||||
2.0905,
|
||||
2.0845,
|
||||
2.0855,
|
||||
2.0796,
|
||||
2.0840,
|
||||
2.0813,
|
||||
2.0817,
|
||||
2.0838,
|
||||
2.0840,
|
||||
2.0917,
|
||||
2.1061,
|
||||
2.1431,
|
||||
2.1976,
|
||||
2.2482,
|
||||
2.3055,
|
||||
2.3700,
|
||||
2.4088,
|
||||
2.4372,
|
||||
2.4609,
|
||||
2.4731,
|
||||
2.4847,
|
||||
2.5072,
|
||||
2.5451,
|
||||
2.5772,
|
||||
2.6147,
|
||||
2.6529,
|
||||
2.6596,
|
||||
2.6645,
|
||||
2.6726,
|
||||
2.6803,
|
||||
2.6812,
|
||||
2.6899,
|
||||
2.6916,
|
||||
2.6931,
|
||||
2.6998,
|
||||
2.7062,
|
||||
2.7262,
|
||||
2.7222,
|
||||
2.7158,
|
||||
2.7041,
|
||||
2.7485,
|
||||
2.7491,
|
||||
2.7451,
|
||||
2.7485,
|
||||
2.7233,
|
||||
2.7297,
|
||||
2.7233,
|
||||
2.7145,
|
||||
2.6958,
|
||||
2.6788,
|
||||
2.6439,
|
||||
2.6007,
|
||||
2.4786,
|
||||
2.2469,
|
||||
2.1877,
|
||||
2.1392,
|
||||
2.0717,
|
||||
2.0107,
|
||||
1.9676,
|
||||
1.9140,
|
||||
1.7102,
|
||||
0.9101,
|
||||
0.7164,
|
||||
]
|
||||
|
||||
|
||||
_NORMALIZATION_EPSILON = 1e-4
|
||||
_SILU_DIVISOR = 0.596
|
||||
_RESIDUAL_BLEND = 0.3
|
||||
_RESIDUAL_DIVISOR = math.hypot(1.0 - _RESIDUAL_BLEND, _RESIDUAL_BLEND)
|
||||
|
||||
|
||||
def _rms_normalize(x: torch.Tensor, dim: int | tuple[int, ...]) -> torch.Tensor:
|
||||
dims = (dim,) if isinstance(dim, int) else dim
|
||||
element_count = math.prod(x.shape[axis] for axis in dims)
|
||||
l2_norm = torch.linalg.vector_norm(x, dim=dims, keepdim=True, dtype=torch.float32)
|
||||
rms = torch.add(_NORMALIZATION_EPSILON, l2_norm, alpha=element_count**-0.5)
|
||||
return x / rms.to(x.dtype)
|
||||
|
||||
|
||||
def _conv1d(in_channels: int, out_channels: int, kernel_size: int) -> nn.Conv1d:
|
||||
return nn.Conv1d(
|
||||
in_channels, out_channels, kernel_size, padding=kernel_size // 2, bias=False
|
||||
)
|
||||
|
||||
|
||||
def _conv1d_with_gain(
|
||||
conv: nn.Conv1d, x: torch.Tensor, gain: torch.Tensor | float
|
||||
) -> torch.Tensor:
|
||||
return F.conv1d(
|
||||
x,
|
||||
conv.weight * gain,
|
||||
stride=conv.stride,
|
||||
padding=conv.padding,
|
||||
dilation=conv.dilation,
|
||||
groups=conv.groups,
|
||||
)
|
||||
|
||||
|
||||
class DiagonalGaussianDistribution:
|
||||
def __init__(self, parameters: torch.Tensor, deterministic: bool = False) -> None:
|
||||
self.parameters = parameters
|
||||
self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)
|
||||
self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
|
||||
self.deterministic = deterministic
|
||||
self.std = torch.exp(0.5 * self.logvar)
|
||||
self.var = torch.exp(self.logvar)
|
||||
if deterministic:
|
||||
self.var = self.std = torch.zeros_like(self.mean)
|
||||
|
||||
def sample(self, generator: torch.Generator | None = None) -> torch.Tensor:
|
||||
noise = torch.empty_like(self.mean).normal_(generator=generator)
|
||||
return self.mean + self.std * noise
|
||||
|
||||
def mode(self) -> torch.Tensor:
|
||||
return self.mean
|
||||
|
||||
|
||||
class ResnetBlock1D(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
in_dim: int,
|
||||
out_dim: int | None = None,
|
||||
conv_shortcut: bool = False,
|
||||
kernel_size: int = 3,
|
||||
use_norm: bool = True,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.in_dim = in_dim
|
||||
self.out_dim = in_dim if out_dim is None else out_dim
|
||||
self.use_conv_shortcut = conv_shortcut
|
||||
self.use_norm = use_norm
|
||||
self.conv1 = _conv1d(in_dim, self.out_dim, kernel_size)
|
||||
self.conv2 = _conv1d(self.out_dim, self.out_dim, kernel_size)
|
||||
if self.in_dim != self.out_dim:
|
||||
if conv_shortcut:
|
||||
self.conv_shortcut = _conv1d(in_dim, self.out_dim, kernel_size)
|
||||
else:
|
||||
self.nin_shortcut = _conv1d(in_dim, self.out_dim, 1)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.use_norm:
|
||||
x = _rms_normalize(x, dim=1)
|
||||
hidden = self.conv1(F.silu(x) / _SILU_DIVISOR)
|
||||
hidden = self.conv2(F.silu(hidden) / _SILU_DIVISOR)
|
||||
if self.in_dim != self.out_dim:
|
||||
shortcut = (
|
||||
self.conv_shortcut if self.use_conv_shortcut else self.nin_shortcut
|
||||
)
|
||||
x = shortcut(x)
|
||||
return torch.lerp(x, hidden, _RESIDUAL_BLEND) / _RESIDUAL_DIVISOR
|
||||
|
||||
|
||||
class AttnBlock1D(nn.Module):
|
||||
def __init__(self, in_channels: int, num_heads: int = 1) -> None:
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.num_heads = num_heads
|
||||
self.qkv = _conv1d(in_channels, in_channels * 3, 1)
|
||||
self.proj_out = _conv1d(in_channels, in_channels, 1)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
qkv = self.qkv(x).reshape(x.shape[0], self.num_heads, -1, 3, x.shape[-1])
|
||||
query, key, value = _rms_normalize(qkv, dim=2).unbind(3)
|
||||
query = rearrange(query, "b h c l -> b h l c")
|
||||
key = rearrange(key, "b h c l -> b h l c")
|
||||
value = rearrange(value, "b h c l -> b h l c")
|
||||
hidden = F.scaled_dot_product_attention(query, key, value)
|
||||
hidden = rearrange(hidden, "b h l c -> b (h c) l")
|
||||
return torch.lerp(x, self.proj_out(hidden), _RESIDUAL_BLEND) / _RESIDUAL_DIVISOR
|
||||
|
||||
|
||||
class Upsample1D(nn.Module):
|
||||
def __init__(self, in_channels: int, with_conv: bool) -> None:
|
||||
super().__init__()
|
||||
self.with_conv = with_conv
|
||||
if with_conv:
|
||||
self.conv = _conv1d(in_channels, in_channels, 3)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = F.interpolate(x, scale_factor=2.0, mode="nearest-exact")
|
||||
return self.conv(x) if self.with_conv else x
|
||||
|
||||
|
||||
class Downsample1D(nn.Module):
|
||||
def __init__(self, in_channels: int, with_conv: bool) -> None:
|
||||
super().__init__()
|
||||
self.with_conv = with_conv
|
||||
if with_conv:
|
||||
self.conv1 = _conv1d(in_channels, in_channels, 1)
|
||||
self.conv2 = _conv1d(in_channels, in_channels, 1)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.with_conv:
|
||||
x = self.conv1(x)
|
||||
x = F.avg_pool1d(x, kernel_size=2, stride=2)
|
||||
return self.conv2(x) if self.with_conv else x
|
||||
|
||||
|
||||
class Encoder1D(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dim: int,
|
||||
ch_mult: tuple[int, ...],
|
||||
num_res_blocks: int,
|
||||
attn_layers: list[int],
|
||||
down_layers: list[int],
|
||||
in_dim: int,
|
||||
embed_dim: int,
|
||||
resamp_with_conv: bool = True,
|
||||
double_z: bool = True,
|
||||
kernel_size: int = 3,
|
||||
clip_act: float = 256.0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.num_layers = len(ch_mult)
|
||||
self.num_res_blocks = num_res_blocks
|
||||
self.in_channels = in_dim
|
||||
self.clip_act = clip_act
|
||||
self.down_layers = down_layers
|
||||
self.attn_layers = attn_layers
|
||||
self.conv_in = _conv1d(in_dim, dim, kernel_size)
|
||||
|
||||
in_ch_mult = (1,) + ch_mult
|
||||
self.in_ch_mult = in_ch_mult
|
||||
self.down = nn.ModuleList()
|
||||
for level in range(self.num_layers):
|
||||
block = nn.ModuleList()
|
||||
attn = nn.ModuleList()
|
||||
block_in = dim * in_ch_mult[level]
|
||||
block_out = dim * ch_mult[level]
|
||||
for _ in range(num_res_blocks):
|
||||
block.append(
|
||||
ResnetBlock1D(
|
||||
in_dim=block_in,
|
||||
out_dim=block_out,
|
||||
kernel_size=kernel_size,
|
||||
use_norm=True,
|
||||
)
|
||||
)
|
||||
block_in = block_out
|
||||
if level in attn_layers:
|
||||
attn.append(AttnBlock1D(block_in))
|
||||
down = nn.Module()
|
||||
down.block = block
|
||||
down.attn = attn
|
||||
if level in down_layers:
|
||||
down.downsample = Downsample1D(block_in, resamp_with_conv)
|
||||
self.down.append(down)
|
||||
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = ResnetBlock1D(
|
||||
in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True
|
||||
)
|
||||
self.mid.attn_1 = AttnBlock1D(block_in)
|
||||
self.mid.block_2 = ResnetBlock1D(
|
||||
in_dim=block_in, out_dim=block_in, kernel_size=kernel_size, use_norm=True
|
||||
)
|
||||
output_dim = 2 * embed_dim if double_z else embed_dim
|
||||
self.conv_out = _conv1d(block_in, output_dim, kernel_size)
|
||||
self.learnable_gain = nn.Parameter(torch.zeros([]))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
states = [self.conv_in(x)]
|
||||
for level in range(self.num_layers):
|
||||
for block_index in range(self.num_res_blocks):
|
||||
hidden = self.down[level].block[block_index](states[-1])
|
||||
if len(self.down[level].attn) > 0:
|
||||
hidden = self.down[level].attn[block_index](hidden)
|
||||
states.append(hidden.clamp(-self.clip_act, self.clip_act))
|
||||
if level in self.down_layers:
|
||||
states.append(self.down[level].downsample(states[-1]))
|
||||
hidden = self.mid.block_1(states[-1])
|
||||
hidden = self.mid.attn_1(hidden)
|
||||
hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act)
|
||||
return _conv1d_with_gain(
|
||||
self.conv_out, F.silu(hidden) / _SILU_DIVISOR, self.learnable_gain + 1
|
||||
)
|
||||
|
||||
|
||||
class Decoder1D(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
dim: int,
|
||||
out_dim: int,
|
||||
ch_mult: tuple[int, ...],
|
||||
num_res_blocks: int,
|
||||
attn_layers: list[int],
|
||||
down_layers: list[int],
|
||||
in_dim: int,
|
||||
embed_dim: int,
|
||||
kernel_size: int = 3,
|
||||
resamp_with_conv: bool = True,
|
||||
clip_act: float = 256.0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.ch = dim
|
||||
self.num_layers = len(ch_mult)
|
||||
self.num_res_blocks = num_res_blocks
|
||||
self.in_channels = in_dim
|
||||
self.clip_act = clip_act
|
||||
self.down_layers = [level + 1 for level in down_layers]
|
||||
block_in = dim * ch_mult[-1]
|
||||
self.conv_in = _conv1d(embed_dim, block_in, kernel_size)
|
||||
self.mid = nn.Module()
|
||||
self.mid.block_1 = ResnetBlock1D(
|
||||
in_dim=block_in, out_dim=block_in, use_norm=True
|
||||
)
|
||||
self.mid.attn_1 = AttnBlock1D(block_in)
|
||||
self.mid.block_2 = ResnetBlock1D(
|
||||
in_dim=block_in, out_dim=block_in, use_norm=True
|
||||
)
|
||||
|
||||
self.up = nn.ModuleList()
|
||||
for level in reversed(range(self.num_layers)):
|
||||
block = nn.ModuleList()
|
||||
attn = nn.ModuleList()
|
||||
block_out = dim * ch_mult[level]
|
||||
for _ in range(num_res_blocks + 1):
|
||||
block.append(
|
||||
ResnetBlock1D(in_dim=block_in, out_dim=block_out, use_norm=True)
|
||||
)
|
||||
block_in = block_out
|
||||
if level in attn_layers:
|
||||
attn.append(AttnBlock1D(block_in))
|
||||
up = nn.Module()
|
||||
up.block = block
|
||||
up.attn = attn
|
||||
if level in self.down_layers:
|
||||
up.upsample = Upsample1D(block_in, resamp_with_conv)
|
||||
self.up.insert(0, up)
|
||||
|
||||
self.conv_out = _conv1d(block_in, out_dim, kernel_size)
|
||||
self.learnable_gain = nn.Parameter(torch.zeros([]))
|
||||
|
||||
def forward(self, z: torch.Tensor) -> torch.Tensor:
|
||||
hidden = self.conv_in(z)
|
||||
hidden = self.mid.block_1(hidden)
|
||||
hidden = self.mid.attn_1(hidden)
|
||||
hidden = self.mid.block_2(hidden).clamp(-self.clip_act, self.clip_act)
|
||||
for level in reversed(range(self.num_layers)):
|
||||
for block_index in range(self.num_res_blocks + 1):
|
||||
hidden = self.up[level].block[block_index](hidden)
|
||||
if len(self.up[level].attn) > 0:
|
||||
hidden = self.up[level].attn[block_index](hidden)
|
||||
hidden = hidden.clamp(-self.clip_act, self.clip_act)
|
||||
if level in self.down_layers:
|
||||
hidden = self.up[level].upsample(hidden)
|
||||
return _conv1d_with_gain(
|
||||
self.conv_out, F.silu(hidden) / _SILU_DIVISOR, self.learnable_gain + 1
|
||||
)
|
||||
|
||||
|
||||
class MMAudioVAE(nn.Module):
|
||||
"""MMAudio mel-spectrogram VAE for 16 kHz or 44.1 kHz audio."""
|
||||
|
||||
def __init__(
|
||||
self, mode: str | dict[str, Any] = "44k", need_encoder: bool = False
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if isinstance(mode, dict):
|
||||
config = mode
|
||||
mode = config.get("mode", "44k")
|
||||
need_encoder = config.get("need_encoder", need_encoder)
|
||||
if mode == "16k":
|
||||
data_dim, embed_dim, hidden_dim = 80, 20, 384
|
||||
data_mean, data_std = DATA_MEAN_80D, DATA_STD_80D
|
||||
elif mode == "44k":
|
||||
data_dim, embed_dim, hidden_dim = 128, 40, 512
|
||||
data_mean, data_std = DATA_MEAN_128D, DATA_STD_128D
|
||||
else:
|
||||
raise ValueError(f"Unknown MMAudio VAE mode: {mode}")
|
||||
|
||||
self.mode = mode
|
||||
self.embed_dim = embed_dim
|
||||
self._weights_normalized = False
|
||||
self.register_buffer(
|
||||
"data_mean", torch.tensor(data_mean, dtype=torch.float32).view(1, -1, 1)
|
||||
)
|
||||
self.register_buffer(
|
||||
"data_std", torch.tensor(data_std, dtype=torch.float32).view(1, -1, 1)
|
||||
)
|
||||
if need_encoder:
|
||||
self.encoder = Encoder1D(
|
||||
dim=hidden_dim,
|
||||
ch_mult=(1, 2, 4),
|
||||
num_res_blocks=2,
|
||||
attn_layers=[3],
|
||||
down_layers=[0],
|
||||
in_dim=data_dim,
|
||||
embed_dim=embed_dim,
|
||||
)
|
||||
self.decoder = Decoder1D(
|
||||
dim=hidden_dim,
|
||||
ch_mult=(1, 2, 4),
|
||||
num_res_blocks=2,
|
||||
attn_layers=[3],
|
||||
down_layers=[0],
|
||||
in_dim=data_dim,
|
||||
out_dim=data_dim,
|
||||
embed_dim=embed_dim,
|
||||
)
|
||||
|
||||
def encode(
|
||||
self, mel: torch.Tensor, normalize_input: bool = True
|
||||
) -> DiagonalGaussianDistribution:
|
||||
self._require_normalized_weights()
|
||||
if not hasattr(self, "encoder"):
|
||||
raise RuntimeError("This MMAudio VAE was loaded decoder-only")
|
||||
if normalize_input:
|
||||
mel = self.normalize(mel)
|
||||
return DiagonalGaussianDistribution(self.encoder(mel))
|
||||
|
||||
def decode(
|
||||
self, latent: torch.Tensor, unnormalize_output: bool = True
|
||||
) -> torch.Tensor:
|
||||
self._require_normalized_weights()
|
||||
mel = self.decoder(latent)
|
||||
return self.unnormalize(mel) if unnormalize_output else mel
|
||||
|
||||
def forward(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
return self.decode(latent)
|
||||
|
||||
def normalize(self, mel: torch.Tensor) -> torch.Tensor:
|
||||
return (mel - self.data_mean) / self.data_std
|
||||
|
||||
def unnormalize(self, mel: torch.Tensor) -> torch.Tensor:
|
||||
return mel * self.data_std + self.data_mean
|
||||
|
||||
def _require_normalized_weights(self) -> None:
|
||||
if not self._weights_normalized:
|
||||
raise RuntimeError("call remove_weight_norm() before inference")
|
||||
|
||||
@torch.no_grad()
|
||||
def remove_weight_norm(self) -> MMAudioVAE:
|
||||
# Idempotent: the diffusers reference calls this exactly once, in
|
||||
# ``MMAudioVAE.__init__``, BEFORE ``from_pretrained`` overwrites the
|
||||
# freshly-constructed (random) Conv1d weights with a checkpoint's
|
||||
# real, already-final values -- so a real checkpoint's own weights
|
||||
# are never renormalized. Mirroring that timing here (see
|
||||
# ``Kandinsky6AudioVAE.__init__``) means this method only ever runs
|
||||
# against pre-load weights too; the guard below additionally makes a
|
||||
# second, redundant call (e.g. the wrapper's own public
|
||||
# ``remove_weight_norm()``) a true no-op instead of re-mutating an
|
||||
# already-normalized (possibly checkpoint-loaded) weight a second
|
||||
# time.
|
||||
if self._weights_normalized:
|
||||
return self
|
||||
for name, module in self.named_modules():
|
||||
if isinstance(module, nn.Conv1d):
|
||||
weight = _rms_normalize(module.weight.to(torch.float32), dim=(1, 2))
|
||||
weight = weight / math.sqrt(weight[0].numel())
|
||||
module.weight.copy_(weight.to(module.weight.dtype))
|
||||
logger.debug("Removed weight norm from %s", name)
|
||||
self._weights_normalized = True
|
||||
return self
|
||||
@@ -0,0 +1,656 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Causal Kandinsky 6 SR KVAE: spatial x16 and temporal x4.
|
||||
|
||||
Encode/decode segments preserve causal-convolution caches; segmentation affects
|
||||
numerics. Pixels normalize as x/128-1, encode returns an unscaled posterior mean,
|
||||
and decode returns normalized pixels. encoder.* and decoder.* match the checkpoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from diffusers.models.autoencoders.vae import DecoderOutput
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
Kandinsky6SRVAEArchConfig,
|
||||
Kandinsky6SRVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_decode_parallel_rank,
|
||||
get_decode_parallel_world_size,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.parallel_conv import (
|
||||
SpatialParallelConv3d,
|
||||
disable_spatial_parallel_decode,
|
||||
gather_and_trim_height,
|
||||
split_height_for_parallel_decode,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.common import (
|
||||
can_install_spatial_shard_parallel_decode,
|
||||
should_run_spatial_shard_parallel_decode,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.wanvae import WanRMS_norm
|
||||
|
||||
# Pixel frames per encode segment after the first (which has one more: the causal first frame).
|
||||
_SEGMENT_FRAMES = 16
|
||||
# Inputs above this many elements are convolved in temporal chunks: at 4x SR decode tiles
|
||||
# (~1536x832) the full-resolution 256-channel convs of the last decoder level see ~5.6e9
|
||||
# elements per segment, beyond 32-bit indexing, and chunking also keeps the transient conv
|
||||
# workspace to a fraction of the activation.
|
||||
_MAX_CONV_NUMEL = 2 * 10**9
|
||||
|
||||
|
||||
class _SegmentCache:
|
||||
"""Causal state carried from one temporal segment to the next during a single encode / decode call."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.first = True
|
||||
self.padding: dict[nn.Module, torch.Tensor] = {}
|
||||
|
||||
|
||||
def _silu(x: torch.Tensor) -> torch.Tensor:
|
||||
# The output norms use this form and the resblocks use F.silu; the two are not bitwise
|
||||
# equal in bf16, so this distinction (inherited from the Diffusers / FastVideo ports) is
|
||||
# deliberate, not a simplification opportunity.
|
||||
return x * torch.sigmoid(x)
|
||||
|
||||
|
||||
class _K6RMSNorm(WanRMS_norm):
|
||||
"""``WanRMS_norm`` with an explicit float32 upcast around the normalize, matching the
|
||||
Diffusers reference's ``Kandinsky6SRRMSNorm`` (which upcasts unconditionally, not only for
|
||||
fp16/bf16/fp8 inputs): at bf16/fp16 activations, normalizing without upcasting measurably
|
||||
drifts from the reference over many resnet blocks."""
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
normed = F.normalize(x.float(), dim=1).to(x.dtype)
|
||||
return normed * self.scale * self.gamma + self.bias
|
||||
|
||||
|
||||
class _ChunkedConv3d(nn.Conv3d):
|
||||
"""``nn.Conv3d`` that convolves inputs larger than ``_MAX_CONV_NUMEL`` in overlapping
|
||||
temporal chunks (plain ``F.conv3d`` silently corrupts most elements once the input exceeds
|
||||
~2**31 elements: it appears to do 32-bit index arithmetic internally)."""
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
num_parts = math.ceil(x.numel() / _MAX_CONV_NUMEL)
|
||||
if num_parts <= 1:
|
||||
return super().forward(x)
|
||||
k, frames = self.kernel_size[0], x.size(2)
|
||||
if self.padding[0] != 0:
|
||||
raise ValueError(
|
||||
f"temporal chunking needs an unpadded time axis, got kernel {self.kernel_size}, "
|
||||
f"padding {self.padding}"
|
||||
)
|
||||
if self.stride[0] != 1:
|
||||
# A strided conv can't reuse the stride-1 path's "prefix each chunk with the
|
||||
# previous chunk's tail" trick below (a strided conv's output doesn't align with
|
||||
# input chunk boundaries), so slide one kernel-width window at a time instead, only
|
||||
# raising if even a single window would overflow.
|
||||
stride = self.stride[0]
|
||||
window_numel = x.shape[0] * x.shape[1] * k * x.shape[3] * x.shape[4]
|
||||
if window_numel >= _MAX_CONV_NUMEL:
|
||||
raise ValueError(
|
||||
f"frames are too big for Conv3d even one stride-{stride} window at a time "
|
||||
f"({window_numel} elements per window)"
|
||||
)
|
||||
# Bare super() doesn't resolve inside a list comprehension's own implicit scope,
|
||||
# hence the loop.
|
||||
window_outputs = []
|
||||
for i in range(0, frames - k + 1, stride):
|
||||
window_outputs.append(super().forward(x[:, :, i : i + k]))
|
||||
return torch.cat(window_outputs, dim=2)
|
||||
step = math.ceil(frames / num_parts)
|
||||
last = frames - step * (math.ceil(frames / step) - 1)
|
||||
if k > 1 and min(step, last) < k:
|
||||
# A chunk would be shorter than the kernel: convolve one output frame at a time instead.
|
||||
windows = [(i, i + k) for i in range(frames - k + 1)]
|
||||
else:
|
||||
# Each input chunk [start, end) is prefixed with the previous chunk's last k - 1 frames.
|
||||
windows = [
|
||||
(max(start - k + 1, 0), min(start + step, frames))
|
||||
for start in range(0, frames, step)
|
||||
]
|
||||
out: torch.Tensor | None = None
|
||||
for lo, hi in windows:
|
||||
y = super().forward(x[:, :, lo:hi])
|
||||
if out is None:
|
||||
out = y.new_empty((*y.shape[:2], frames - k + 1, *y.shape[3:]))
|
||||
out[:, :, lo : lo + y.size(2)] = y
|
||||
assert out is not None
|
||||
return out
|
||||
|
||||
|
||||
class _SpatialChunkedConv3d(SpatialParallelConv3d, _ChunkedConv3d):
|
||||
# exchange halos once before temporal chunking; uneven height shards may need
|
||||
# different chunk counts, so collectives must stay outside that loop
|
||||
@classmethod
|
||||
def from_conv(cls, conv: _ChunkedConv3d, height_padding=None):
|
||||
with torch.device("meta"):
|
||||
parallel = cls(
|
||||
conv.in_channels,
|
||||
conv.out_channels,
|
||||
conv.kernel_size,
|
||||
stride=conv.stride,
|
||||
padding=conv.padding,
|
||||
dilation=conv.dilation,
|
||||
groups=conv.groups,
|
||||
bias=conv.bias is not None,
|
||||
height_padding=height_padding,
|
||||
)
|
||||
parallel.weight, parallel.bias = conv.weight, conv.bias
|
||||
return parallel
|
||||
|
||||
def _direct_forward(self, x):
|
||||
x = F.pad(x, (0, 0, self.height_pad_top, self.height_pad_bottom, 0, 0))
|
||||
return _ChunkedConv3d.forward(self, x)
|
||||
|
||||
|
||||
class _CausalConv3d(nn.Module):
|
||||
"""Conv3d that is causal in time: the first segment is left-padded with copies of its
|
||||
first frame, later segments with the tail of the previous one; height / width are zero
|
||||
padded."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
chan_in: int,
|
||||
chan_out: int,
|
||||
kernel_size: int | tuple[int, int, int],
|
||||
stride: tuple[int, int, int] = (1, 1, 1),
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if isinstance(kernel_size, int):
|
||||
kernel_size = (kernel_size,) * 3
|
||||
time_kernel, height_kernel, width_kernel = kernel_size
|
||||
self.height_pad = height_kernel // 2
|
||||
self.width_pad = width_kernel // 2
|
||||
self.time_pad = time_kernel - 1
|
||||
self.time_kernel = time_kernel
|
||||
self.time_stride = stride[0]
|
||||
self.conv = _ChunkedConv3d(chan_in, chan_out, kernel_size, stride=stride)
|
||||
|
||||
def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
|
||||
s, k = self.time_stride, self.time_kernel
|
||||
batch, _, frames, height, width = x.shape
|
||||
x = F.pad(
|
||||
x, (self.width_pad, self.width_pad, self.height_pad, self.height_pad, 0, 0)
|
||||
)
|
||||
if cache.first:
|
||||
first_frame = x[:, :, :1]
|
||||
padding = first_frame.expand(-1, -1, self.time_pad, -1, -1)
|
||||
else:
|
||||
padding = cache.padding[self]
|
||||
|
||||
# conv(cat([padding, x])) without materializing the concatenation of the whole segment:
|
||||
# only the outputs whose window overlaps ``padding`` are computed on a small
|
||||
# concatenated head.
|
||||
out_frames = frames if s == 1 else (frames + 1) // 2
|
||||
output = x.new_empty((batch, self.conv.out_channels, out_frames, height, width))
|
||||
head_out = math.ceil(padding.size(2) / s)
|
||||
head_in = head_out * s - padding.size(2)
|
||||
if head_out > 0:
|
||||
output[:, :, :head_out] = self.conv(
|
||||
torch.cat([padding, x[:, :, : head_in + k - s]], dim=2)
|
||||
)
|
||||
if head_out < out_frames:
|
||||
output[:, :, head_out:] = self.conv(x[:, :, head_in:])
|
||||
|
||||
# Keep exactly the input frames the next segment's first output window needs.
|
||||
pad_offset = head_in + s * math.trunc((frames - head_in - k) / s) + s
|
||||
if pad_offset < 0: # segment shorter than the kernel
|
||||
cache.padding[self] = torch.cat([padding[:, :, pad_offset:], x], dim=2)
|
||||
else:
|
||||
cache.padding[self] = x[:, :, pad_offset:].clone()
|
||||
return output
|
||||
|
||||
|
||||
class _ResnetBlock3D(nn.Module):
|
||||
"""norm -> SiLU -> causal conv, twice, plus a 1x1 shortcut when the width changes. Decoder
|
||||
blocks use the latent-conditioned ``_SpatialNorm3D`` (``zq_channels`` set)."""
|
||||
|
||||
def __init__(
|
||||
self, in_channels: int, out_channels: int, zq_channels: int | None = None
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if zq_channels is None:
|
||||
self.norm1: nn.Module = _K6RMSNorm(in_channels, images=False)
|
||||
self.norm2: nn.Module = _K6RMSNorm(out_channels, images=False)
|
||||
else:
|
||||
self.norm1 = _SpatialNorm3D(in_channels, zq_channels)
|
||||
self.norm2 = _SpatialNorm3D(out_channels, zq_channels)
|
||||
self.conv1 = _CausalConv3d(in_channels, out_channels, kernel_size=3)
|
||||
self.conv2 = _CausalConv3d(out_channels, out_channels, kernel_size=3)
|
||||
if in_channels != out_channels:
|
||||
self.nin_shortcut = _ChunkedConv3d(in_channels, out_channels, kernel_size=1)
|
||||
|
||||
def _norm(
|
||||
self,
|
||||
norm: nn.Module,
|
||||
h: torch.Tensor,
|
||||
zq: torch.Tensor | None,
|
||||
cache: _SegmentCache,
|
||||
) -> torch.Tensor:
|
||||
return norm(h) if zq is None else norm(h, zq, cache)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, cache: _SegmentCache, zq: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
h = F.silu(self._norm(self.norm1, x, zq, cache), inplace=True)
|
||||
h = self.conv1(h, cache)
|
||||
h = F.silu(self._norm(self.norm2, h, zq, cache), inplace=True)
|
||||
h = self.conv2(h, cache)
|
||||
if hasattr(self, "nin_shortcut"):
|
||||
x = self.nin_shortcut(x)
|
||||
return x + h
|
||||
|
||||
|
||||
def _chunked_interpolate_nearest(
|
||||
x: torch.Tensor, size: tuple[int, int, int], channels: int = 32
|
||||
) -> torch.Tensor:
|
||||
"""Nearest interpolation in channel chunks, bounding memory and int32 indexing."""
|
||||
if x.shape[1] <= channels:
|
||||
return F.interpolate(x, size=size, mode="nearest")
|
||||
return torch.cat(
|
||||
[
|
||||
F.interpolate(chunk, size=size, mode="nearest")
|
||||
for chunk in torch.split(x, channels, dim=1)
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
|
||||
|
||||
class _SpatialNorm3D(nn.Module):
|
||||
"""RMS norm modulated by the latent: ``norm(f) * conv_y(zq) + conv_b(zq)`` with ``zq``
|
||||
nearest-resized to ``f``."""
|
||||
|
||||
def __init__(self, f_channels: int, zq_channels: int) -> None:
|
||||
super().__init__()
|
||||
self.norm_layer = _K6RMSNorm(f_channels, images=False)
|
||||
self.conv_y = _ChunkedConv3d(zq_channels, f_channels, kernel_size=1)
|
||||
self.conv_b = _ChunkedConv3d(zq_channels, f_channels, kernel_size=1)
|
||||
|
||||
def forward(
|
||||
self, f: torch.Tensor, zq: torch.Tensor, cache: _SegmentCache
|
||||
) -> torch.Tensor:
|
||||
frames, height, width = f.shape[-3:]
|
||||
if cache.first:
|
||||
# The causal first latent frame maps to exactly one pixel frame; the rest cover the
|
||||
# remaining frames.
|
||||
zq_first = _chunked_interpolate_nearest(zq[:, :, :1], (1, height, width))
|
||||
if zq.size(2) > 1:
|
||||
zq_rest = _chunked_interpolate_nearest(
|
||||
zq[:, :, 1:], (frames - 1, height, width)
|
||||
)
|
||||
zq = torch.cat([zq_first, zq_rest], dim=2)
|
||||
else:
|
||||
zq = zq_first
|
||||
else:
|
||||
zq = _chunked_interpolate_nearest(zq, (frames, height, width))
|
||||
norm_f = self.norm_layer(f)
|
||||
norm_f.mul_(self.conv_y(zq))
|
||||
norm_f.add_(self.conv_b(zq))
|
||||
return norm_f
|
||||
|
||||
|
||||
class _PXSDownsample(nn.Module):
|
||||
"""x2 spatial (strided conv + channel-averaged pixel-unshuffle) and optional x2 causal
|
||||
temporal (two causal convs + average pooling) downsample; doubles the channels."""
|
||||
|
||||
def __init__(self, in_channels: int, compress_time: bool) -> None:
|
||||
super().__init__()
|
||||
out_channels = 2 * in_channels
|
||||
self.spatial_conv = _ChunkedConv3d(
|
||||
in_channels,
|
||||
out_channels,
|
||||
kernel_size=(1, 3, 3),
|
||||
stride=(1, 2, 2),
|
||||
padding=(0, 1, 1),
|
||||
)
|
||||
self.compress_time = compress_time
|
||||
if compress_time:
|
||||
self.temporal_conv = nn.Sequential(
|
||||
_CausalConv3d(out_channels, out_channels, kernel_size=(2, 1, 1)),
|
||||
_CausalConv3d(
|
||||
out_channels, out_channels, kernel_size=(2, 1, 1), stride=(2, 1, 1)
|
||||
),
|
||||
)
|
||||
self.linear = _ChunkedConv3d(out_channels, out_channels, kernel_size=1)
|
||||
|
||||
def _spatial(self, x: torch.Tensor) -> torch.Tensor:
|
||||
b, c, t, h, w = x.shape
|
||||
pxs = F.pixel_unshuffle(x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w), 2)
|
||||
pxs = pxs.view(b * t, 2 * c, 2, h // 2, w // 2).mean(dim=2)
|
||||
pxs = pxs.reshape(b, t, 2 * c, h // 2, w // 2).permute(0, 2, 1, 3, 4)
|
||||
return self.spatial_conv(x) + pxs
|
||||
|
||||
def _temporal(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
|
||||
b, c, t, h, w = x.shape
|
||||
pooled = x.permute(0, 3, 4, 1, 2).reshape(b * h * w, c, t)
|
||||
if cache.first: # the causal first frame is kept as is
|
||||
first, rest = pooled[..., :1], pooled[..., 1:]
|
||||
pooled = (
|
||||
torch.cat([first, F.avg_pool1d(rest, kernel_size=2, stride=2)], dim=-1)
|
||||
if t > 1
|
||||
else first
|
||||
)
|
||||
else:
|
||||
pooled = F.avg_pool1d(pooled, kernel_size=2, stride=2)
|
||||
pooled = pooled.reshape(b, h, w, c, -1).permute(0, 3, 4, 1, 2)
|
||||
conv = self.temporal_conv[1](self.temporal_conv[0](x, cache), cache)
|
||||
return conv + pooled
|
||||
|
||||
def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
|
||||
out = self._spatial(x)
|
||||
if self.compress_time:
|
||||
out = self._temporal(out, cache)
|
||||
return self.linear(out)
|
||||
|
||||
|
||||
class _PXSUpsample(nn.Module):
|
||||
"""Optional x2 causal temporal (frame repeat + causal conv residual) then x2 spatial
|
||||
(nearest + conv residual) upsample."""
|
||||
|
||||
def __init__(self, in_channels: int, compress_time: bool) -> None:
|
||||
super().__init__()
|
||||
self.spatial_conv = _ChunkedConv3d(
|
||||
in_channels, in_channels, kernel_size=(1, 3, 3), padding=(0, 1, 1)
|
||||
)
|
||||
self.compress_time = compress_time
|
||||
if compress_time:
|
||||
self.temporal_conv = _CausalConv3d(
|
||||
in_channels, in_channels, kernel_size=(3, 1, 1)
|
||||
)
|
||||
self.linear = _ChunkedConv3d(in_channels, in_channels, kernel_size=1)
|
||||
|
||||
def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
|
||||
if self.compress_time:
|
||||
x = x.repeat_interleave(2, dim=2)
|
||||
if cache.first: # the causal first frame is not duplicated
|
||||
x = x[:, :, 1:]
|
||||
x = self.temporal_conv(x, cache) + x
|
||||
frames, height, width = x.shape[-3:]
|
||||
# Chunked along channels: see _chunked_interpolate_nearest's docstring for the overflow
|
||||
# this guards against (the old vendored code did not chunk this specific call).
|
||||
x = _chunked_interpolate_nearest(x, (frames, height * 2, width * 2))
|
||||
x.add_(self.spatial_conv(x))
|
||||
return self.linear(x)
|
||||
|
||||
|
||||
def _level_module(**children: nn.Module) -> nn.Module:
|
||||
module = nn.Module()
|
||||
for name, child in children.items():
|
||||
setattr(module, name, child)
|
||||
return module
|
||||
|
||||
|
||||
class _Encoder3D(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
ch: int,
|
||||
ch_mult: list[float],
|
||||
num_res_blocks: int,
|
||||
in_channels: int,
|
||||
z_channels: int,
|
||||
temporal_compress_times: int,
|
||||
temporal_compress_start_level: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.num_res_blocks = num_res_blocks
|
||||
time_levels = range(
|
||||
temporal_compress_start_level,
|
||||
temporal_compress_start_level + int(math.log2(temporal_compress_times)),
|
||||
)
|
||||
self.conv_in = _CausalConv3d(in_channels, round(ch * ch_mult[0]), kernel_size=3)
|
||||
self.down = nn.ModuleList()
|
||||
block_in = round(ch * ch_mult[0])
|
||||
for level, mult in enumerate(ch_mult):
|
||||
block_out = round(ch * mult)
|
||||
blocks = nn.ModuleList()
|
||||
for _ in range(num_res_blocks):
|
||||
blocks.append(_ResnetBlock3D(block_in, block_out))
|
||||
block_in = block_out
|
||||
down = _level_module(block=blocks)
|
||||
if level != len(ch_mult) - 1:
|
||||
down.downsample = _PXSDownsample(
|
||||
block_in, compress_time=level in time_levels
|
||||
)
|
||||
block_in *= 2
|
||||
self.down.append(down)
|
||||
self.mid = _level_module(
|
||||
block_1=_ResnetBlock3D(block_in, block_in),
|
||||
block_2=_ResnetBlock3D(block_in, block_in),
|
||||
)
|
||||
self.norm_out = _K6RMSNorm(block_in, images=False)
|
||||
self.conv_out = _CausalConv3d(block_in, 2 * z_channels, kernel_size=3)
|
||||
|
||||
def forward(self, x: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
|
||||
h = self.conv_in(x, cache)
|
||||
for down in self.down:
|
||||
for block in down.block:
|
||||
h = block(h, cache)
|
||||
if hasattr(down, "downsample"):
|
||||
h = down.downsample(h, cache)
|
||||
h = self.mid.block_2(self.mid.block_1(h, cache), cache)
|
||||
return self.conv_out(_silu(self.norm_out(h)), cache)
|
||||
|
||||
|
||||
class _Decoder3D(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
ch: int,
|
||||
ch_mult: list[float],
|
||||
num_res_blocks: int,
|
||||
out_ch: int,
|
||||
z_channels: int,
|
||||
temporal_compress_times: int,
|
||||
temporal_compress_start_level: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
num_levels = len(ch_mult)
|
||||
# Mirror of the encoder's temporally compressing levels.
|
||||
time_levels = range(
|
||||
num_levels
|
||||
- temporal_compress_start_level
|
||||
- int(math.log2(temporal_compress_times)),
|
||||
num_levels - temporal_compress_start_level,
|
||||
)
|
||||
block_in = round(ch * ch_mult[-1])
|
||||
self.conv_in = _CausalConv3d(z_channels, block_in, kernel_size=3)
|
||||
self.mid = _level_module(
|
||||
block_1=_ResnetBlock3D(block_in, block_in, z_channels),
|
||||
block_2=_ResnetBlock3D(block_in, block_in, z_channels),
|
||||
)
|
||||
up_levels: list[nn.Module] = []
|
||||
for level in reversed(range(num_levels)):
|
||||
block_out = round(ch * ch_mult[level])
|
||||
blocks = nn.ModuleList()
|
||||
for _ in range(num_res_blocks + 1):
|
||||
blocks.append(_ResnetBlock3D(block_in, block_out, z_channels))
|
||||
block_in = block_out
|
||||
up = _level_module(block=blocks)
|
||||
if level != 0:
|
||||
up.upsample = _PXSUpsample(block_in, compress_time=level in time_levels)
|
||||
up_levels.insert(0, up)
|
||||
self.up = nn.ModuleList(up_levels)
|
||||
self.norm_out = _SpatialNorm3D(block_in, z_channels)
|
||||
self.conv_out = _CausalConv3d(block_in, out_ch, kernel_size=3)
|
||||
|
||||
def forward(self, z: torch.Tensor, cache: _SegmentCache) -> torch.Tensor:
|
||||
h = self.conv_in(z, cache)
|
||||
h = self.mid.block_2(self.mid.block_1(h, cache, z), cache, z)
|
||||
for up in reversed(self.up):
|
||||
for block in up.block:
|
||||
h = block(h, cache, z)
|
||||
if hasattr(up, "upsample"):
|
||||
h = up.upsample(h, cache)
|
||||
return self.conv_out(_silu(self.norm_out(h, z, cache)), cache)
|
||||
|
||||
|
||||
# The structural knobs every Encoder3D / Decoder3D constructor kwarg comes from; everything
|
||||
# else in a checkpoint's encoder_config / decoder_config (``checkpoint_list``, ``resolution``,
|
||||
# the placeholder ``"None"`` channel field, ...) is ignored, the same way the old vendored
|
||||
# module silently dropped unknown keys via its own ``**ignore_kwargs`` / unused params.
|
||||
_ARCH_KEYS = (
|
||||
"ch",
|
||||
"ch_mult",
|
||||
"num_res_blocks",
|
||||
"z_channels",
|
||||
"temporal_compress_times",
|
||||
)
|
||||
|
||||
|
||||
def _arch_kwargs(conf: dict[str, Any], extra: str, which: str) -> dict[str, Any]:
|
||||
# norm_type is the one legacy knob this port still checks: unlike padding_mode /
|
||||
# downsample_version / fix_pxs / double_z (which the new Encoder3D / Decoder3D no longer
|
||||
# branch on at all -- they always run the "zeros" / v2-pixel-unshuffle / fixed-stride /
|
||||
# double-z behavior those knobs used to select), a group_norm checkpoint would silently
|
||||
# get rms_norm layers instead of a clear error, which is the one mismatch worth failing
|
||||
# loudly on here.
|
||||
norm_type = conf.get("norm_type", "rms_norm")
|
||||
if norm_type != "rms_norm":
|
||||
raise ValueError(
|
||||
f"Kandinsky6SRVAE supports only {which}_config['norm_type'] == 'rms_norm' "
|
||||
f"(group-norm KVAEs are not implemented by this port), got {norm_type!r}"
|
||||
)
|
||||
missing = [key for key in (*_ARCH_KEYS, extra) if key not in conf]
|
||||
if missing:
|
||||
raise ValueError(f"Kandinsky6SRVAE {which}_config is missing {missing}")
|
||||
kwargs = {key: conf[key] for key in (*_ARCH_KEYS, extra)}
|
||||
kwargs["temporal_compress_start_level"] = conf.get(
|
||||
"temporal_compress_start_level", 0
|
||||
)
|
||||
return kwargs
|
||||
|
||||
|
||||
class Kandinsky6SRVAE(nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
"""Segment-cached causal video KVAE used by the SR pipeline."""
|
||||
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
|
||||
def __init__(self, config: Kandinsky6SRVAEConfig) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
arch = config.arch_config
|
||||
assert isinstance(arch, Kandinsky6SRVAEArchConfig)
|
||||
if not arch.encoder_config or not arch.decoder_config:
|
||||
raise ValueError(
|
||||
"Kandinsky6SRVAE needs `encoder_config` and `decoder_config` (KVAE "
|
||||
"architecture) in the component config.json."
|
||||
)
|
||||
enc = _arch_kwargs(dict(arch.encoder_config), "in_channels", "encoder")
|
||||
dec = _arch_kwargs(dict(arch.decoder_config), "out_ch", "decoder")
|
||||
self.temporal_compression = enc["temporal_compress_times"]
|
||||
self.encoder = _Encoder3D(**enc)
|
||||
self.decoder = _Decoder3D(**dec)
|
||||
self._spatial_parallel_decode_enabled = (
|
||||
can_install_spatial_shard_parallel_decode(config)
|
||||
)
|
||||
if self._spatial_parallel_decode_enabled:
|
||||
for module in self.decoder.modules():
|
||||
if isinstance(module, _CausalConv3d) and module.height_pad:
|
||||
module.conv = _SpatialChunkedConv3d.from_conv(
|
||||
module.conv, (module.height_pad, module.height_pad)
|
||||
)
|
||||
module.height_pad = 0
|
||||
elif isinstance(module, _PXSUpsample):
|
||||
module.spatial_conv = _SpatialChunkedConv3d.from_conv(
|
||||
module.spatial_conv
|
||||
)
|
||||
# encode/decode invoke each block, not the surrounding level container
|
||||
self.layer_names = [
|
||||
f"{path}.{index}.block"
|
||||
for path, levels in (
|
||||
("encoder.down", self.encoder.down),
|
||||
("decoder.up", self.decoder.up),
|
||||
)
|
||||
for index in range(len(levels))
|
||||
]
|
||||
|
||||
@property
|
||||
def scaling_factor(self) -> float:
|
||||
return float(self.config.arch_config.scaling_factor)
|
||||
|
||||
@property
|
||||
def spatial_factor(self) -> int:
|
||||
return int(self.config.arch_config.spatial_factor)
|
||||
|
||||
@property
|
||||
def temporal_factor(self) -> int:
|
||||
return int(self.config.arch_config.temporal_factor)
|
||||
|
||||
@staticmethod
|
||||
def normalize_data(data: torch.Tensor) -> torch.Tensor:
|
||||
return data / 128 - 1.0
|
||||
|
||||
@staticmethod
|
||||
def denormalize_data(data: torch.Tensor) -> torch.Tensor:
|
||||
return (data + 1) * 128
|
||||
|
||||
def encode(self, x: torch.Tensor) -> tuple[torch.Tensor, list[int]]:
|
||||
"""``[B, C, T, H, W]`` normalized pixels -> ``(raw latent mean, pixel-frame segment sizes)``."""
|
||||
split_list = [_SEGMENT_FRAMES + 1]
|
||||
remaining = x.size(2) - split_list[0]
|
||||
while remaining > 0:
|
||||
split_list.append(_SEGMENT_FRAMES)
|
||||
remaining -= _SEGMENT_FRAMES
|
||||
split_list[-1] += remaining
|
||||
cache = _SegmentCache()
|
||||
latents = []
|
||||
for segment in torch.split(x, split_list, dim=2):
|
||||
moments = self.encoder(segment, cache)
|
||||
cache.first = False
|
||||
latents.append(
|
||||
moments.chunk(2, dim=1)[0]
|
||||
) # deterministic posterior: the mean
|
||||
return torch.cat(latents, dim=2), split_list
|
||||
|
||||
def decode(self, z: torch.Tensor) -> DecoderOutput:
|
||||
"""``[B, C, T', h, w]`` raw latent -> ``.sample`` ``[B, 3, T, H, W]`` in the normalized pixel range."""
|
||||
if self._spatial_parallel_decode_enabled:
|
||||
world_size = get_decode_parallel_world_size()
|
||||
if (
|
||||
should_run_spatial_shard_parallel_decode(self.config, z)
|
||||
and z.shape[-2] >= world_size
|
||||
):
|
||||
z, height = split_height_for_parallel_decode(
|
||||
z,
|
||||
expected_height=z.shape[-2] * self.spatial_factor,
|
||||
world_size=world_size,
|
||||
rank=get_decode_parallel_rank(),
|
||||
)
|
||||
return DecoderOutput(
|
||||
sample=gather_and_trim_height(self._decode(z), height)
|
||||
)
|
||||
with disable_spatial_parallel_decode():
|
||||
return DecoderOutput(sample=self._decode(z))
|
||||
return DecoderOutput(sample=self._decode(z))
|
||||
|
||||
def _decode(self, z: torch.Tensor) -> torch.Tensor:
|
||||
segment = _SEGMENT_FRAMES // self.temporal_compression
|
||||
num_frames = z.size(2)
|
||||
if num_frames == 1:
|
||||
split_list = [1]
|
||||
else:
|
||||
split_list = [segment] * ((num_frames - 1) // segment)
|
||||
if (num_frames - 1) % segment:
|
||||
split_list.append((num_frames - 1) % segment)
|
||||
split_list[0] += 1
|
||||
cache = _SegmentCache()
|
||||
samples = []
|
||||
for chunk in torch.split(z, split_list, dim=2):
|
||||
samples.append(self.decoder(chunk, cache))
|
||||
cache.first = False
|
||||
return torch.cat(samples, dim=2)
|
||||
|
||||
|
||||
EntryClass = Kandinsky6SRVAE
|
||||
|
||||
__all__ = ["Kandinsky6SRVAE"]
|
||||
@@ -3,13 +3,19 @@
|
||||
# Licensed under the MIT license.
|
||||
|
||||
# Adapted from https://github.com/jik876/hifi-gan under the MIT license.
|
||||
#
|
||||
# The alias-free resampling primitives (LowPassFilter1d/UpSample1d/
|
||||
# DownSample1d/Activation1d) are architecture-agnostic DSP building blocks
|
||||
# shared with the sibling ``kandinsky6_audio_vae/bigvgan.py`` -- both import
|
||||
# ``Activation1d`` from ``runtime/models/vaes/alias_free.py`` (sibling of
|
||||
# this package, alongside ``common.py``) instead of each keeping its own copy.
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.nn import Conv1d, ConvTranspose1d, Parameter
|
||||
from torch.nn.utils.parametrizations import weight_norm
|
||||
|
||||
from .alias_free import Activation1d
|
||||
from sglang.multimodal_gen.runtime.models.vaes.alias_free import Activation1d
|
||||
|
||||
|
||||
def get_padding(kernel_size, dilation=1):
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Joint video/audio generation with optional image conditioning.
|
||||
|
||||
One stage chain handles both T2VA and IT2VA; the image stage is a no-op for T2VA."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6 import (
|
||||
Kandinsky6TI2VASamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline import LoRAPipeline
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages import InputValidationStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6 import (
|
||||
Kandinsky6AudioDecodingStage,
|
||||
Kandinsky6DecodingStage,
|
||||
Kandinsky6DenoisingStage,
|
||||
Kandinsky6ImageEncodingStage,
|
||||
Kandinsky6LatentPreparationStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
|
||||
TextEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation import (
|
||||
TimestepPreparationStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
|
||||
class Kandinsky6TI2VAPipeline(LoRAPipeline, ComposedPipelineBase):
|
||||
pipeline_name = "Kandinsky6TI2VAPipeline"
|
||||
is_video_pipeline = True
|
||||
pipeline_config_cls = Kandinsky6TI2VAPipelineConfig
|
||||
sampling_params_cls = Kandinsky6TI2VASamplingParams
|
||||
|
||||
# Kandinsky6AudioVAE bundles the mel-VAE decoder *and* the BigVGAN-v2
|
||||
# vocoder in one checkpoint component -- there is no separate "vocoder"
|
||||
# module, unlike LTX-2's separately-loaded audio_vae/vocoder pair.
|
||||
_required_config_modules = [
|
||||
"scheduler",
|
||||
"text_encoder",
|
||||
"text_encoder_2",
|
||||
"tokenizer",
|
||||
"tokenizer_2",
|
||||
"transformer",
|
||||
"vae",
|
||||
"audio_vae",
|
||||
]
|
||||
|
||||
def validate_disagg_role(self, role: RoleType) -> None:
|
||||
# coupled denoising and reference-frame state are not yet disaggregation-aware
|
||||
if role != RoleType.MONOLITHIC:
|
||||
raise ValueError(
|
||||
"Kandinsky6TI2VAPipeline only supports monolithic deployment; "
|
||||
f"disaggregation role {role.value!r} is not supported"
|
||||
)
|
||||
|
||||
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
|
||||
self.add_stage(
|
||||
stage_name="input_validation_stage", stage=InputValidationStage()
|
||||
)
|
||||
|
||||
self.add_stage(
|
||||
stage_name="text_encoding_stage",
|
||||
stage=TextEncodingStage(
|
||||
text_encoders=[
|
||||
self.get_module("text_encoder"),
|
||||
self.get_module("text_encoder_2"),
|
||||
],
|
||||
tokenizers=[
|
||||
self.get_module("tokenizer"),
|
||||
self.get_module("tokenizer_2"),
|
||||
],
|
||||
),
|
||||
)
|
||||
|
||||
self.add_stage(
|
||||
stage_name="timestep_preparation_stage",
|
||||
stage=TimestepPreparationStage(scheduler=self.get_module("scheduler")),
|
||||
)
|
||||
|
||||
self.add_stage(
|
||||
stage_name="latent_preparation_stage",
|
||||
stage=Kandinsky6LatentPreparationStage(),
|
||||
)
|
||||
|
||||
# No-op unless the request supplies a conditioning image; see
|
||||
# Kandinsky6ImageEncodingStage's docstring.
|
||||
self.add_stage(
|
||||
stage_name="image_encoding_stage",
|
||||
stage=Kandinsky6ImageEncodingStage(vae=self.get_module("vae")),
|
||||
)
|
||||
|
||||
self.add_stage(
|
||||
stage_name="denoising_stage",
|
||||
stage=Kandinsky6DenoisingStage(
|
||||
transformer=self.get_module("transformer"),
|
||||
scheduler=self.get_module("scheduler"),
|
||||
pipeline=self,
|
||||
),
|
||||
)
|
||||
|
||||
# Audio decodes before video, matching the diffusers reference and
|
||||
# FastVideo's own stage order -- do not reorder.
|
||||
self.add_stage(
|
||||
stage_name="audio_decoding_stage",
|
||||
stage=Kandinsky6AudioDecodingStage(audio_vae=self.get_module("audio_vae")),
|
||||
)
|
||||
|
||||
self.add_stage(
|
||||
stage_name="decoding_stage",
|
||||
stage=Kandinsky6DecodingStage(vae=self.get_module("vae"), pipeline=self),
|
||||
)
|
||||
|
||||
|
||||
EntryClass = [Kandinsky6TI2VAPipeline]
|
||||
@@ -0,0 +1,103 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Kandinsky 6 video SR: prepare, encode/upscale, denoise, decode and blend.
|
||||
|
||||
Work is grouped by component to avoid weight transfers per tile. The bundle's
|
||||
scheduler drives denoising through the shared DenoisingStage lifecycle.
|
||||
num_inference_steps counts DiT calls; pi-Flow uses its checkpoint-defined nfe."""
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import (
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
|
||||
Kandinsky6SRDecodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
|
||||
Kandinsky6SRDenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import (
|
||||
Kandinsky6SREncodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
|
||||
Kandinsky6SRInputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
|
||||
Kandinsky6SRLatentPrepStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import (
|
||||
Kandinsky6SROutputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
|
||||
class Kandinsky6SRPipeline(ComposedPipelineBase):
|
||||
pipeline_name = "Kandinsky6SRPipeline"
|
||||
is_video_pipeline = True
|
||||
pipeline_config_cls = Kandinsky6SRPipelineConfig
|
||||
sampling_params_cls = Kandinsky6SRSamplingParams
|
||||
|
||||
_required_config_modules = ["transformer", "vae", "scheduler", "latent_upscaler"]
|
||||
# Repos without a latent-upscaler bank run the pixel path only.
|
||||
_optional_config_modules = ("latent_upscaler",)
|
||||
|
||||
def validate_disagg_role(self, role: RoleType) -> None:
|
||||
# The tiled loop keeps tile latents / decoded tiles in one process.
|
||||
if role != RoleType.MONOLITHIC:
|
||||
raise ValueError(
|
||||
"Kandinsky6SRPipeline only supports monolithic deployment; "
|
||||
f"disaggregation role {role.value!r} is not supported"
|
||||
)
|
||||
|
||||
def load_modules(self, server_args: ServerArgs, loaded_modules=None):
|
||||
model_index = self._load_config()
|
||||
if "dit" in model_index and "transformer" not in model_index:
|
||||
raise ValueError(
|
||||
f"{self.model_path} is a k6_video SR bundle (it has a 'dit' component), "
|
||||
"not an official Diffusers Kandinsky6SRPipeline repo. Use "
|
||||
"kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers or "
|
||||
"kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers"
|
||||
)
|
||||
for name in self._optional_config_modules:
|
||||
if model_index.get(name) is None and name in self._required_config_modules:
|
||||
self._required_config_modules.remove(name)
|
||||
model_index.pop(name, None)
|
||||
return super().load_modules(server_args, loaded_modules)
|
||||
|
||||
def create_pipeline_stages(self, server_args: ServerArgs) -> None:
|
||||
latent_upscaler = self.get_module("latent_upscaler")
|
||||
vae = self.get_module("vae")
|
||||
transformer = self.get_module("transformer")
|
||||
scheduler = self.get_module("scheduler")
|
||||
self.add_stage(stage_name="input_stage", stage=Kandinsky6SRInputStage())
|
||||
self.add_stage(
|
||||
stage_name="encode_stage",
|
||||
stage=Kandinsky6SREncodeStage(vae=vae, latent_upscaler=latent_upscaler),
|
||||
)
|
||||
self.add_stage(
|
||||
stage_name="latent_prep_stage",
|
||||
stage=Kandinsky6SRLatentPrepStage(
|
||||
vae=vae,
|
||||
transformer=transformer,
|
||||
latent_upscaler=latent_upscaler,
|
||||
scheduler=scheduler,
|
||||
),
|
||||
)
|
||||
self.add_stage(
|
||||
stage_name="denoising_stage",
|
||||
stage=Kandinsky6SRDenoisingStage(
|
||||
transformer=transformer, scheduler=scheduler, pipeline=self
|
||||
),
|
||||
)
|
||||
self.add_stage(
|
||||
stage_name="decode_stage", stage=Kandinsky6SRDecodeStage(vae=vae)
|
||||
)
|
||||
self.add_stage(stage_name="output_stage", stage=Kandinsky6SROutputStage())
|
||||
|
||||
|
||||
EntryClass = [Kandinsky6SRPipeline]
|
||||
@@ -488,6 +488,12 @@ class OutputBatch:
|
||||
raw_frame_metadata: dict[str, Any] | None = None
|
||||
audio: torch.Tensor | None = None
|
||||
audio_sample_rate: int | None = None
|
||||
# The effective fps the worker actually produced ``output`` at (e.g. a model whose input
|
||||
# stage resamples / resolves the request's nominal fps to a source-derived one, such as
|
||||
# Kandinsky6 SR). ``None`` means the worker did not resolve its own fps; callers that save
|
||||
# ``output`` client-side (return_file_paths_only=False) then fall back to the originating
|
||||
# request's own fps field, as before this field existed.
|
||||
fps: int | None = None
|
||||
action_pred: torch.Tensor | None = None
|
||||
action_mode: str | None = None
|
||||
action_domain_id: int | None = None
|
||||
|
||||
@@ -1035,8 +1035,16 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
|
||||
self._cache_dit_request_overrides = resolve_cache_dit_request_overrides(
|
||||
batch.sampling_params.cache_dit_params
|
||||
)
|
||||
has_separate_cfg = (
|
||||
requested
|
||||
and batch.do_classifier_free_guidance
|
||||
and not self.server_args.enable_cfg_parallel
|
||||
)
|
||||
desired_key = (
|
||||
cache_dit_overrides_key(self._cache_dit_request_overrides)
|
||||
(
|
||||
cache_dit_overrides_key(self._cache_dit_request_overrides),
|
||||
has_separate_cfg,
|
||||
)
|
||||
if requested
|
||||
else None
|
||||
)
|
||||
@@ -1063,11 +1071,11 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
|
||||
steps_computation_policy=scm_policy,
|
||||
)
|
||||
else:
|
||||
scm_preset = None if scm_preset == "none" else scm_preset
|
||||
refresh_context_on_transformer(
|
||||
self.transformer,
|
||||
primary_num_steps,
|
||||
scm_preset=scm_preset,
|
||||
steps_computation_mask=steps_computation_mask,
|
||||
steps_computation_policy=scm_policy,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -1156,7 +1164,7 @@ class DenoisingStage(PipelineStage, RolloutDenoisingMixin):
|
||||
model_name="transformer",
|
||||
sp_group=sp_group,
|
||||
tp_group=tp_group,
|
||||
has_separate_cfg=batch.do_classifier_free_guidance,
|
||||
has_separate_cfg=has_separate_cfg,
|
||||
)
|
||||
logger.info(
|
||||
"cache-dit enabled on transformer (steps=%d, Fn=%d, Bn=%d, rdt=%.3f)",
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
"""Kandinsky6-specific pipeline stages."""
|
||||
|
||||
from .decoding import Kandinsky6AudioDecodingStage, Kandinsky6DecodingStage
|
||||
from .denoising import Kandinsky6DenoisingStage
|
||||
from .image_encoding import Kandinsky6ImageEncodingStage
|
||||
from .latent_preparation import Kandinsky6LatentPreparationStage
|
||||
|
||||
__all__ = [
|
||||
"Kandinsky6AudioDecodingStage",
|
||||
"Kandinsky6DecodingStage",
|
||||
"Kandinsky6DenoisingStage",
|
||||
"Kandinsky6ImageEncodingStage",
|
||||
"Kandinsky6LatentPreparationStage",
|
||||
]
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Decode audio before video, preserving the reference component order."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
StageValidators as V,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
VerificationResult,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
|
||||
class Kandinsky6DecodingStage(DecodingStage):
|
||||
"""Convert video to channel-first for shared VAE decode; attach the decoded audio."""
|
||||
|
||||
def __init__(self, vae, pipeline=None) -> None:
|
||||
super().__init__(vae=vae, pipeline=pipeline)
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
|
||||
if batch.latents is None:
|
||||
raise ValueError("latents must be available before Kandinsky6 decoding.")
|
||||
batch.latents = batch.latents.permute(0, 4, 1, 2, 3).contiguous()
|
||||
output_batch = super().forward(batch, server_args)
|
||||
output_batch.audio = batch.audio
|
||||
output_batch.audio_sample_rate = batch.audio_sample_rate
|
||||
return output_batch
|
||||
|
||||
|
||||
class Kandinsky6AudioDecodingStage(PipelineStage):
|
||||
"""Decode audio through the bundled mel VAE/vocoder into batch.audio."""
|
||||
|
||||
def __init__(self, audio_vae) -> None:
|
||||
super().__init__()
|
||||
self.audio_vae = audio_vae
|
||||
|
||||
@property
|
||||
def role_affinity(self) -> RoleType:
|
||||
return RoleType.DECODER
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
stage_name = self._component_stage_name(stage_name)
|
||||
return [ComponentUse(stage_name, "audio_vae")]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
if batch.audio_latents is None:
|
||||
return batch
|
||||
|
||||
device = get_local_torch_device()
|
||||
with self.use_declared_component(
|
||||
component_name="audio_vae", module=self.audio_vae
|
||||
) as audio_vae:
|
||||
assert audio_vae is not None
|
||||
self.audio_vae = audio_vae
|
||||
|
||||
# Cast to the module's own loaded weight dtype directly -- no
|
||||
# autocast / precision-config knob, matching the existing
|
||||
# MMAudio-lineage audio decoding stages in this codebase.
|
||||
decoder_dtype = next(audio_vae.parameters()).dtype
|
||||
|
||||
# apply checkpoint latent denormalization before the VAE's mel denormalization
|
||||
scaling_factor = audio_vae.scaling_factor
|
||||
mean_value = audio_vae.mean_value
|
||||
latents = (batch.audio_latents / scaling_factor) + mean_value
|
||||
|
||||
# [B, A, D] -> [B, D, A] to match the audio VAE's 1D-conv
|
||||
# (channel, length) convention.
|
||||
latents = latents.to(device=device, dtype=decoder_dtype).transpose(1, 2)
|
||||
waveform = audio_vae.wrapped_decode(latents) # [B, 1, samples]
|
||||
|
||||
# preserve [B, samples] so the save path selects the audio track for each output
|
||||
batch.audio = waveform[:, 0].clamp(-1.0, 1.0).float().cpu()
|
||||
batch.audio_sample_rate = int(server_args.pipeline_config.audio_sample_rate)
|
||||
|
||||
return batch
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check("audio_latents", batch.audio_latents, V.none_or_tensor)
|
||||
return result
|
||||
+442
@@ -0,0 +1,442 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Joint video/audio denoising using the shared stage lifecycle, BCG and CFG.
|
||||
|
||||
Flow-Euler advances the video scheduler once per step; audio uses the same
|
||||
sigma delta without advancing that scheduler again. pi-Flow uses independent
|
||||
video/audio schedulers and retains fp32 state between bf16 DiT calls.
|
||||
|
||||
Adapted from FastVideo's Kandinsky6DenoisingStage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import math
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_parallel_utils import (
|
||||
run_cfg_parallel,
|
||||
run_two_branch_cfg_parallel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGBranch, CFGPolicy
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_classifier_free_guidance_world_size,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import (
|
||||
TAIL_COND_ACTIVE_EXTRA_KEY,
|
||||
VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
StageValidators as V,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
VerificationResult,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYPE
|
||||
|
||||
|
||||
class Kandinsky6DenoisingStage(DenoisingStage):
|
||||
"""Run the Kandinsky6 joint video+audio denoising loop."""
|
||||
|
||||
def __init__(self, transformer, scheduler, pipeline=None) -> None:
|
||||
super().__init__(
|
||||
transformer=transformer, scheduler=scheduler, pipeline=pipeline
|
||||
)
|
||||
|
||||
def _owns_compile_warmup_lifecycle(self) -> bool:
|
||||
# forward() overrides DenoisingStage.forward (the joint video+audio
|
||||
# loop cannot reuse the parent's single-tensor _denoise()), but it
|
||||
# still wraps the loop in `_offload_for_torch_compile_warmup` itself
|
||||
# below -- same opt-in as MiniMaxH3DenoisingStage, the codebase's
|
||||
# other joint-modality denoising stage.
|
||||
return True
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
stage_name = self._component_stage_name(stage_name)
|
||||
return [
|
||||
ComponentUse(
|
||||
stage_name,
|
||||
"transformer",
|
||||
phase="transformer",
|
||||
preferred_ready_after_request=True,
|
||||
memory_intensive=True,
|
||||
)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _text_rope_pos(mask: torch.Tensor, device: torch.device) -> torch.Tensor:
|
||||
seq_len = int(mask.sum(1).max().item())
|
||||
return torch.arange(seq_len, device=device)
|
||||
|
||||
def _call_transformer(self, transformer, **call_kwargs) -> Any:
|
||||
"""Dispatch through BCG when enabled, otherwise call the transformer directly."""
|
||||
runner = self._maybe_get_bcg_runner(transformer)
|
||||
if runner is not None:
|
||||
return self._bcg_run(runner, call_kwargs, transformer)
|
||||
return transformer(**call_kwargs)
|
||||
|
||||
def _build_cfg_policy(
|
||||
self,
|
||||
batch: Req,
|
||||
server_args: ServerArgs,
|
||||
*,
|
||||
prompt_embeds: torch.Tensor,
|
||||
pooled: torch.Tensor,
|
||||
text_rope_pos: torch.Tensor,
|
||||
) -> CFGPolicy:
|
||||
"""Reuse the pipeline CFG policy with joint-modality conditioning kwargs."""
|
||||
branches = [
|
||||
CFGBranch(
|
||||
"conditional",
|
||||
True,
|
||||
{
|
||||
"encoder_hidden_states": prompt_embeds,
|
||||
"pooled_projections": pooled,
|
||||
"text_rope_pos": text_rope_pos,
|
||||
},
|
||||
)
|
||||
]
|
||||
if batch.do_classifier_free_guidance and batch.negative_prompt_embeds:
|
||||
neg_prompt_embeds = server_args.pipeline_config.get_neg_prompt_embeds(
|
||||
batch
|
||||
).to(device=prompt_embeds.device, dtype=prompt_embeds.dtype)
|
||||
if not batch.neg_pooled_embeds:
|
||||
raise ValueError(
|
||||
"Kandinsky6 requires CLIP negative pooled projections for CFG."
|
||||
)
|
||||
neg_pooled = batch.neg_pooled_embeds[0].to(
|
||||
device=prompt_embeds.device, dtype=prompt_embeds.dtype
|
||||
)
|
||||
if not batch.negative_attention_mask:
|
||||
raise ValueError(
|
||||
"Kandinsky6 requires Qwen (Reason1) negative attention masks for CFG."
|
||||
)
|
||||
negative_text_rope_pos = self._text_rope_pos(
|
||||
batch.negative_attention_mask[0].to(prompt_embeds.device),
|
||||
prompt_embeds.device,
|
||||
)
|
||||
branches.append(
|
||||
CFGBranch(
|
||||
"unconditional",
|
||||
False,
|
||||
{
|
||||
"encoder_hidden_states": neg_prompt_embeds,
|
||||
"pooled_projections": neg_pooled,
|
||||
"text_rope_pos": negative_text_rope_pos,
|
||||
},
|
||||
)
|
||||
)
|
||||
return dataclasses.replace(
|
||||
server_args.pipeline_config.cfg_policy, branches=branches
|
||||
)
|
||||
|
||||
def _predict_joint_velocity(
|
||||
self,
|
||||
*,
|
||||
transformer,
|
||||
cfg_policy: CFGPolicy,
|
||||
batch: Req,
|
||||
server_args: ServerArgs,
|
||||
step_index: int,
|
||||
video_input: torch.Tensor,
|
||||
audio_input: torch.Tensor,
|
||||
t_expand: torch.Tensor,
|
||||
visual_rope_pos: list[torch.Tensor],
|
||||
scale_factor: tuple[float, ...],
|
||||
sparse_params: dict[str, Any] | None,
|
||||
visual_token_type_ids: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run sequential or distributed CFG branches and combine video/audio velocity."""
|
||||
|
||||
def predict_fn(branch: CFGBranch) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
branch.configure_batch(batch)
|
||||
with set_forward_context(
|
||||
current_timestep=step_index, attn_metadata=None, forward_batch=batch
|
||||
):
|
||||
model_output = self._call_transformer(
|
||||
transformer,
|
||||
hidden_states=video_input,
|
||||
hidden_states_audio=audio_input,
|
||||
timestep=t_expand,
|
||||
visual_rope_pos=visual_rope_pos,
|
||||
scale_factor=scale_factor,
|
||||
sparse_params=sparse_params,
|
||||
visual_token_type_ids=visual_token_type_ids,
|
||||
# BCG clones tuple leaves so the next CFG replay cannot
|
||||
# overwrite the previous branch's predictions
|
||||
return_dict=False,
|
||||
**branch.kwargs,
|
||||
)
|
||||
return model_output
|
||||
|
||||
cfg_scale = server_args.pipeline_config.get_classifier_free_guidance_scale(
|
||||
batch, batch.guidance_scale
|
||||
)
|
||||
|
||||
if server_args.enable_cfg_parallel:
|
||||
if (
|
||||
len(cfg_policy.branches) == 2
|
||||
and get_classifier_free_guidance_world_size() == 2
|
||||
and not cfg_policy.parallel_uses_serial_arithmetic
|
||||
):
|
||||
video_vel, audio_vel = run_two_branch_cfg_parallel(
|
||||
cfg_policy,
|
||||
predict_fn,
|
||||
cfg_scale,
|
||||
batch,
|
||||
server_args.pipeline_config,
|
||||
)
|
||||
return video_vel, audio_vel
|
||||
predictions = run_cfg_parallel(cfg_policy, predict_fn)
|
||||
else:
|
||||
predictions = [predict_fn(branch) for branch in cfg_policy.branches]
|
||||
|
||||
video_vel, audio_vel = cfg_policy.combine(
|
||||
predictions,
|
||||
batch,
|
||||
cfg_scale,
|
||||
server_args.pipeline_config,
|
||||
cfg_parallel=server_args.enable_cfg_parallel,
|
||||
)
|
||||
return video_vel, audio_vel
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
with self._offload_for_torch_compile_warmup(batch):
|
||||
return self._run_joint_denoise_loop(batch, server_args)
|
||||
|
||||
def _run_joint_denoise_loop(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
if batch.timesteps is None:
|
||||
raise ValueError("timesteps must be prepared before Kandinsky6 denoising.")
|
||||
if batch.latents is None or batch.audio_latents is None:
|
||||
raise ValueError(
|
||||
"video and audio latents must be prepared before Kandinsky6 denoising."
|
||||
)
|
||||
scheduler = batch.scheduler
|
||||
if scheduler is None:
|
||||
raise ValueError("scheduler must be set for Kandinsky6 denoising.")
|
||||
use_piflow = bool(getattr(scheduler, "is_piflow", False))
|
||||
if use_piflow and (
|
||||
not math.isfinite(batch.guidance_scale)
|
||||
or abs(batch.guidance_scale - 1.0) > 1e-6
|
||||
):
|
||||
raise ValueError("Kandinsky6 PiFlow requires guidance_scale=1.0.")
|
||||
audio_scheduler = deepcopy(scheduler) if use_piflow else None
|
||||
|
||||
pipeline_config = server_args.pipeline_config
|
||||
arch = pipeline_config.dit_config.arch_config
|
||||
# The runtime transformer already rejects attention_engine="nabla"
|
||||
# at construction time (NABLA sparse attention is not yet ported),
|
||||
# so sparse_params stays None on every currently-reachable
|
||||
# configuration. A future NABLA-capable DiT would fill this in.
|
||||
if arch.attention_engine == "nabla":
|
||||
raise NotImplementedError(
|
||||
"Kandinsky6 NABLA sparse-attention metadata construction is not implemented; "
|
||||
"the runtime transformer also rejects attention_engine='nabla' at construction "
|
||||
"time. Use attention_engine='auto' or 'sdpa'."
|
||||
)
|
||||
sparse_params = None
|
||||
|
||||
device = get_local_torch_device()
|
||||
target_dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision]
|
||||
|
||||
# .clone(): batch.latents was created by Kandinsky6LatentPreparationStage
|
||||
# under an active torch.inference_mode() context further up the
|
||||
# pipeline, which marks it as an "inference tensor" -- such tensors
|
||||
# can only be mutated in place while still inside an inference_mode
|
||||
# context. This stage runs under plain @torch.no_grad() instead, and
|
||||
# mutates `video` in place every denoising step (video[..., :num_channels]
|
||||
# = ...), so clone once here to get an ordinary, freely-mutable tensor
|
||||
# before the loop starts. audio is never mutated in place (only
|
||||
# reassigned via out-of-place `+`), so it needs no clone.
|
||||
video = batch.latents.clone()
|
||||
audio = batch.audio_latents
|
||||
num_channels = int(arch.in_visual_dim)
|
||||
|
||||
tail_cond = bool(batch.extra.get(TAIL_COND_ACTIVE_EXTRA_KEY, False))
|
||||
visual_token_type_ids = batch.extra.get(VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY)
|
||||
|
||||
prompt_embeds = pipeline_config.get_pos_prompt_embeds(batch).to(
|
||||
device=device, dtype=target_dtype
|
||||
)
|
||||
if not batch.pooled_embeds:
|
||||
raise ValueError("Kandinsky6 requires CLIP pooled projections.")
|
||||
pooled = batch.pooled_embeds[0].to(device=device, dtype=target_dtype)
|
||||
if not batch.prompt_attention_mask:
|
||||
raise ValueError(
|
||||
"Kandinsky6 requires Qwen (Reason1) prompt attention masks."
|
||||
)
|
||||
text_rope_pos = self._text_rope_pos(
|
||||
batch.prompt_attention_mask[0].to(device), device
|
||||
)
|
||||
|
||||
cfg_policy = self._build_cfg_policy(
|
||||
batch,
|
||||
server_args,
|
||||
prompt_embeds=prompt_embeds,
|
||||
pooled=pooled,
|
||||
text_rope_pos=text_rope_pos,
|
||||
)
|
||||
|
||||
height = int(batch.height)
|
||||
width = int(batch.width)
|
||||
spatial_ratio = pipeline_config.vae_config.arch_config.spatial_compression_ratio
|
||||
patch_size = arch.patch_size
|
||||
|
||||
# Constant per-request geometry -- computed once outside the loop,
|
||||
# not per-step.
|
||||
num_video_frames = video.shape[1] - (1 if tail_cond else 0)
|
||||
t_positions = torch.arange(num_video_frames, device=device)
|
||||
if tail_cond:
|
||||
# The appended reference frame reuses T-position 0's rope row
|
||||
# (RoPE3D is a pure position -> table lookup, so duplicating the
|
||||
# first row is equivalent to the diffusers reference's
|
||||
# ``torch.cat([rope, rope[:1]])``).
|
||||
t_positions = torch.cat([t_positions, t_positions.new_zeros(1)])
|
||||
visual_rope_pos = [
|
||||
t_positions,
|
||||
torch.arange(height // spatial_ratio // patch_size[1], device=device),
|
||||
torch.arange(width // spatial_ratio // patch_size[2], device=device),
|
||||
]
|
||||
# Fixed per-checkpoint RoPE frequency scaling read from the DiT's
|
||||
# arch config (transformer/config.json's "scale_factor", both real
|
||||
# Pro checkpoints ship [1.0, 2.0, 2.0]) -- matches the diffusers
|
||||
# reference, which resolves this once in
|
||||
# ``Kandinsky6TI2VAPipeline.__init__`` and reuses it for every
|
||||
# request regardless of the request's own height/width. NOT a
|
||||
# function of the request's resolution.
|
||||
scale_factor = arch.scale_factor
|
||||
image_latent = batch.image_latent
|
||||
|
||||
total_steps = int(batch.timesteps.shape[0])
|
||||
self._maybe_enable_cache_dit_and_torch_compile(total_steps, batch)
|
||||
|
||||
# PiFlow's noisy video state is tracked in fp32 across steps here,
|
||||
# separately from the `video` buffer's bf16 storage dtype -- see the
|
||||
# module docstring. Always a Tensor (not `Tensor | None`) even on the
|
||||
# flow-Euler path, where it is simply unused: the initial cast is one
|
||||
# cheap op, and keeping it non-Optional avoids re-narrowing it after
|
||||
# every `if use_piflow:` branch re-entry in the loop below.
|
||||
video_state = video[..., :num_channels].to(torch.float32)
|
||||
|
||||
with self.use_declared_component(
|
||||
component_name="transformer", module=self.transformer
|
||||
) as transformer:
|
||||
assert transformer is not None
|
||||
self.transformer = transformer
|
||||
|
||||
with self.progress_bar(
|
||||
total=total_steps, batch=batch, desc="Kandinsky6 Denoising"
|
||||
) as progress_bar:
|
||||
for i, timestep in enumerate(batch.timesteps):
|
||||
t_expand = (
|
||||
timestep.unsqueeze(0)
|
||||
.repeat(video.shape[0])
|
||||
.to(device=device, dtype=target_dtype)
|
||||
)
|
||||
|
||||
if use_piflow:
|
||||
video_input = video_state.to(dtype=target_dtype)
|
||||
if video.shape[-1] > num_channels:
|
||||
video_input = torch.cat(
|
||||
[
|
||||
video_input,
|
||||
video[..., num_channels:].to(dtype=target_dtype),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
else:
|
||||
video_input = video.to(dtype=target_dtype)
|
||||
|
||||
video_vel, audio_vel = self._predict_joint_velocity(
|
||||
transformer=transformer,
|
||||
cfg_policy=cfg_policy,
|
||||
batch=batch,
|
||||
server_args=server_args,
|
||||
step_index=i,
|
||||
video_input=video_input,
|
||||
audio_input=audio.to(dtype=target_dtype),
|
||||
t_expand=t_expand,
|
||||
visual_rope_pos=visual_rope_pos,
|
||||
scale_factor=scale_factor,
|
||||
sparse_params=sparse_params,
|
||||
visual_token_type_ids=visual_token_type_ids,
|
||||
)
|
||||
|
||||
# scheduler.step() is called exactly once per iteration,
|
||||
# against video only -- it owns the internal step index.
|
||||
if use_piflow:
|
||||
video_state = scheduler.step(
|
||||
video_vel, timestep, video_state, return_dict=False
|
||||
)[0]
|
||||
video[..., :num_channels] = video_state.to(video.dtype)
|
||||
else:
|
||||
video[..., :num_channels] = scheduler.step(
|
||||
video_vel,
|
||||
timestep,
|
||||
video[..., :num_channels],
|
||||
return_dict=False,
|
||||
)[0]
|
||||
if tail_cond:
|
||||
ref_frame = image_latent.to(
|
||||
device=video.device, dtype=video.dtype
|
||||
)
|
||||
video[:, -1:, :, :, :num_channels] = ref_frame
|
||||
if use_piflow:
|
||||
video_state[:, -1:] = ref_frame.to(torch.float32)
|
||||
|
||||
# Manual Euler update for audio, using the SAME per-step
|
||||
# sigma delta the scheduler.step() call above just
|
||||
# consumed for video.
|
||||
if use_piflow:
|
||||
audio = audio_scheduler.step(
|
||||
audio_vel, timestep, audio, return_dict=False
|
||||
)[0]
|
||||
else:
|
||||
step_size = scheduler.sigmas[i + 1] - scheduler.sigmas[i]
|
||||
audio = (
|
||||
audio
|
||||
+ step_size.to(device=audio.device, dtype=audio.dtype)
|
||||
* audio_vel
|
||||
)
|
||||
|
||||
if progress_bar is not None:
|
||||
progress_bar.update()
|
||||
|
||||
video = video[..., :num_channels]
|
||||
if tail_cond:
|
||||
video = video[:, :-1]
|
||||
|
||||
batch.latents = video
|
||||
batch.audio_latents = audio
|
||||
return batch
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
|
||||
result.add_check(
|
||||
"audio_latents", batch.audio_latents, [V.is_tensor, V.with_dims(3)]
|
||||
)
|
||||
result.add_check("prompt_embeds", batch.prompt_embeds, V.list_not_empty)
|
||||
result.add_check("timesteps", batch.timesteps, [V.is_tensor, V.with_dims(1)])
|
||||
return result
|
||||
|
||||
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
|
||||
result.add_check(
|
||||
"audio_latents", batch.audio_latents, [V.is_tensor, V.with_dims(3)]
|
||||
)
|
||||
return result
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Optional image conditioning, adapted from FastVideo's Kandinsky6ImageEncodingStage.
|
||||
|
||||
Append a clean reference latent frame and mark its token type. Denoising re-pins
|
||||
that frame every step and removes it before decoding."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import PIL.Image
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
StageValidators as V,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
VerificationResult,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.condition_expansion import (
|
||||
PromptToSampleBatchExpander,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.runtime.utils.precision import (
|
||||
autocast_context,
|
||||
autocast_enabled,
|
||||
resolve_precision,
|
||||
temporary_module_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.vision import (
|
||||
normalize,
|
||||
numpy_to_pt,
|
||||
pil_to_numpy,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# model-specific reference-frame state shared through Req.extra
|
||||
TAIL_COND_ACTIVE_EXTRA_KEY = "kandinsky6_tail_cond_active"
|
||||
VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY = "kandinsky6_visual_token_type_ids"
|
||||
|
||||
|
||||
class Kandinsky6ImageEncodingStage(PipelineStage):
|
||||
"""Optional IT2VA conditioning: no-op for a pure T2VA call.
|
||||
|
||||
See module docstring for the tail-cond scheme.
|
||||
"""
|
||||
|
||||
def __init__(self, vae: ParallelTiledVAE) -> None:
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
stage_name = self._component_stage_name(stage_name)
|
||||
return [ComponentUse(stage_name, "vae", target_dtype=vae_dtype)]
|
||||
|
||||
@staticmethod
|
||||
def _cover_resize_dims(
|
||||
src_h: int, src_w: int, height: int, width: int
|
||||
) -> tuple[int, int]:
|
||||
"""Resize to cover (height, width), preserving aspect before the centre crop."""
|
||||
scale = min(src_h / height, src_w / width)
|
||||
# clamp float round-trip undershoot on the constraining axis before centre crop
|
||||
new_h = max(height, int(src_h / scale))
|
||||
new_w = max(width, int(src_w / scale))
|
||||
return new_h, new_w
|
||||
|
||||
@classmethod
|
||||
def _preprocess(cls, image, height: int, width: int) -> torch.Tensor:
|
||||
if isinstance(image, PIL.Image.Image):
|
||||
src_w, src_h = image.size
|
||||
new_h, new_w = cls._cover_resize_dims(src_h, src_w, height, width)
|
||||
image = image.resize((new_w, new_h), resample=PIL.Image.BILINEAR)
|
||||
top, left = (new_h - height) // 2, (new_w - width) // 2
|
||||
image = image.crop((left, top, left + width, top + height))
|
||||
image = numpy_to_pt(pil_to_numpy(image))
|
||||
return normalize(image)
|
||||
if image.min() >= 0:
|
||||
logger.warning(
|
||||
"Kandinsky6 conditioning image tensor has no negative values; "
|
||||
"assuming range [0, 1] and normalizing to [-1, 1]. Pass a "
|
||||
"[-1, 1] tensor with negative values to skip normalization."
|
||||
)
|
||||
image = normalize(image)
|
||||
if image.ndim == 3:
|
||||
image = image.unsqueeze(0)
|
||||
src_h, src_w = image.shape[-2], image.shape[-1]
|
||||
if (src_h, src_w) != (height, width):
|
||||
new_h, new_w = cls._cover_resize_dims(src_h, src_w, height, width)
|
||||
image = torch.nn.functional.interpolate(
|
||||
image.float(), size=(new_h, new_w), mode="bilinear", antialias=True
|
||||
)
|
||||
top, left = (new_h - height) // 2, (new_w - width) // 2
|
||||
image = image[..., top : top + height, left : left + width]
|
||||
return image
|
||||
|
||||
@staticmethod
|
||||
def _encode_scale_and_shift(
|
||||
latents: torch.Tensor, scaling_factor, shift_factor
|
||||
) -> torch.Tensor:
|
||||
if shift_factor is not None:
|
||||
if isinstance(shift_factor, torch.Tensor):
|
||||
shift_factor = shift_factor.to(latents.device, latents.dtype)
|
||||
latents = latents - shift_factor
|
||||
if isinstance(scaling_factor, torch.Tensor):
|
||||
scaling_factor = scaling_factor.to(latents.device, latents.dtype)
|
||||
return latents * scaling_factor
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
if batch.condition_image is None:
|
||||
return batch
|
||||
if batch.latents is None:
|
||||
raise ValueError(
|
||||
"Kandinsky6 video latents must be prepared before image conditioning."
|
||||
)
|
||||
|
||||
image_source = batch.condition_image
|
||||
if isinstance(image_source, list):
|
||||
image_source = image_source[0]
|
||||
|
||||
device = get_local_torch_device()
|
||||
height = int(batch.height)
|
||||
width = int(batch.width)
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
vae_autocast_enabled = autocast_enabled(vae_dtype, server_args.disable_autocast)
|
||||
|
||||
image = self._preprocess(image_source, height, width)
|
||||
image = image.to(device=device, dtype=torch.float32).unsqueeze(2) # [B,C,1,H,W]
|
||||
|
||||
generator = batch.generator
|
||||
if isinstance(generator, list) and len(generator) != image.shape[0]:
|
||||
generator = generator[0]
|
||||
|
||||
with self.use_declared_component(component_name="vae", module=self.vae) as vae:
|
||||
assert vae is not None
|
||||
self.vae = vae
|
||||
prev_use_tiling = vae.use_tiling
|
||||
vae.use_tiling = False
|
||||
try:
|
||||
with autocast_context(vae_dtype, server_args.disable_autocast):
|
||||
should_cast_vae = not vae_autocast_enabled
|
||||
if not vae_autocast_enabled:
|
||||
image = image.to(vae_dtype)
|
||||
with temporary_module_dtype(
|
||||
vae, vae_dtype, enabled=should_cast_vae
|
||||
) as vae_cast:
|
||||
image_latent = vae_cast.encode(image).sample(
|
||||
generator=generator
|
||||
)
|
||||
finally:
|
||||
vae.use_tiling = prev_use_tiling
|
||||
|
||||
scaling_factor, shift_factor = (
|
||||
server_args.pipeline_config.get_decode_scale_and_shift(
|
||||
image_latent.device, image_latent.dtype, vae
|
||||
)
|
||||
)
|
||||
|
||||
image_latent = self._encode_scale_and_shift(
|
||||
image_latent, scaling_factor, shift_factor
|
||||
)
|
||||
# [B,C,1,H,W] -> [B,1,H,W,C] channel-last, matching the video latent.
|
||||
image_latent = image_latent.permute(0, 2, 3, 4, 1).contiguous()
|
||||
batch.image_latent = image_latent
|
||||
|
||||
latents = batch.latents
|
||||
if batch.image_latent.shape[0] != latents.shape[0]:
|
||||
# encode one reference image, then expand it to the request's output batch
|
||||
expander = PromptToSampleBatchExpander(
|
||||
prompt_batch_size=batch.image_latent.shape[0],
|
||||
sample_batch_size=latents.shape[0],
|
||||
)
|
||||
expander.expand_field(batch, "image_latent")
|
||||
|
||||
image_latent = batch.image_latent.to(device=latents.device, dtype=latents.dtype)
|
||||
num_channels = image_latent.shape[-1]
|
||||
|
||||
ref_frame = image_latent
|
||||
if latents.shape[-1] > num_channels:
|
||||
# [real, cond, mask]: write the reference only into real; cond stays zero
|
||||
cond_block = torch.zeros_like(image_latent)
|
||||
mask_block = torch.ones_like(image_latent[..., :1])
|
||||
ref_frame = torch.cat([image_latent, cond_block, mask_block], dim=-1)
|
||||
|
||||
batch.latents = torch.cat([latents, ref_frame], dim=1)
|
||||
batch.extra[TAIL_COND_ACTIVE_EXTRA_KEY] = True
|
||||
|
||||
num_video_frames = latents.shape[1]
|
||||
token_type_ids = torch.zeros(
|
||||
(latents.shape[0], num_video_frames + 1), dtype=torch.long, device=device
|
||||
)
|
||||
token_type_ids[:, -1] = 1
|
||||
batch.extra[VISUAL_TOKEN_TYPE_IDS_EXTRA_KEY] = token_type_ids
|
||||
|
||||
return batch
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
|
||||
return result
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Joint noise preparation, adapted from FastVideo's Kandinsky6LatentPreparationStage.
|
||||
|
||||
Draw video then audio from the same generator. Image conditioning is appended later."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
StageValidators as V,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
VerificationResult,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision_types import PRECISION_TO_TYPE
|
||||
|
||||
|
||||
def audio_latent_duration(
|
||||
video_latent_frames: int,
|
||||
*,
|
||||
fps: float,
|
||||
audio_sample_rate: int,
|
||||
audio_downsample_factor: int,
|
||||
) -> int:
|
||||
"""Audio latent length from causal video duration at the request's fps."""
|
||||
pixel_frames = (video_latent_frames - 1) * 4 + 1
|
||||
return int(
|
||||
math.ceil(pixel_frames / fps * audio_sample_rate / audio_downsample_factor)
|
||||
)
|
||||
|
||||
|
||||
class Kandinsky6LatentPreparationStage(PipelineStage):
|
||||
"""Draw initial video *and* audio noise latents."""
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
if batch.height is None or batch.width is None:
|
||||
raise ValueError("height and width must be provided for Kandinsky6.")
|
||||
height = int(batch.height)
|
||||
width = int(batch.width)
|
||||
num_frames = int(batch.num_frames)
|
||||
|
||||
pipeline_config = server_args.pipeline_config
|
||||
arch = pipeline_config.dit_config.arch_config
|
||||
temporal_ratio = (
|
||||
pipeline_config.vae_config.arch_config.temporal_compression_ratio
|
||||
)
|
||||
spatial_ratio = pipeline_config.vae_config.arch_config.spatial_compression_ratio
|
||||
patch_size = arch.patch_size
|
||||
|
||||
if num_frames % temporal_ratio != 1:
|
||||
num_frames = num_frames // temporal_ratio * temporal_ratio + 1
|
||||
batch.num_frames = num_frames
|
||||
|
||||
required_divisor_h = spatial_ratio * patch_size[1]
|
||||
required_divisor_w = spatial_ratio * patch_size[2]
|
||||
if height % required_divisor_h != 0 or width % required_divisor_w != 0:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 height must be divisible by {required_divisor_h} and width by "
|
||||
f"{required_divisor_w}; got height={height}, width={width}."
|
||||
)
|
||||
|
||||
batch_size = batch.batch_size
|
||||
dtype = PRECISION_TO_TYPE[pipeline_config.dit_precision]
|
||||
device = get_local_torch_device()
|
||||
num_latent_frames = (num_frames - 1) // temporal_ratio + 1
|
||||
num_channels = arch.in_visual_dim
|
||||
video_shape = (
|
||||
batch_size,
|
||||
num_latent_frames,
|
||||
height // spatial_ratio,
|
||||
width // spatial_ratio,
|
||||
num_channels,
|
||||
)
|
||||
|
||||
generator = batch.generator
|
||||
if batch.latents is None:
|
||||
video = randn_tensor(
|
||||
video_shape, generator=generator, device=device, dtype=dtype
|
||||
)
|
||||
else:
|
||||
video = batch.latents.to(device=device, dtype=dtype)
|
||||
if tuple(video.shape) != video_shape:
|
||||
raise ValueError(
|
||||
f"Provided latents shape {list(video.shape)} does not match expected "
|
||||
f"Kandinsky6 video latent shape {list(video_shape)}."
|
||||
)
|
||||
|
||||
if bool(arch.visual_cond):
|
||||
cond = torch.zeros_like(video)
|
||||
mask = torch.zeros(
|
||||
(*video.shape[:-1], 1), device=video.device, dtype=video.dtype
|
||||
)
|
||||
video = torch.cat([video, cond, mask], dim=-1)
|
||||
|
||||
num_audio_channels = arch.in_audio_dim
|
||||
# align audio duration with request fps, not the pipeline default
|
||||
num_audio_frames = audio_latent_duration(
|
||||
num_latent_frames,
|
||||
fps=float(batch.fps),
|
||||
audio_sample_rate=pipeline_config.audio_sample_rate,
|
||||
audio_downsample_factor=pipeline_config.audio_downsample_factor,
|
||||
)
|
||||
audio_shape = (batch_size, num_audio_frames, num_audio_channels)
|
||||
if batch.audio_latents is None:
|
||||
# Drawn from the same generator right after the video noise, so
|
||||
# it is automatically an independent sample -- no manual seed
|
||||
# offset needed.
|
||||
audio = randn_tensor(
|
||||
audio_shape, generator=generator, device=device, dtype=dtype
|
||||
)
|
||||
else:
|
||||
audio = batch.audio_latents.to(device=device, dtype=dtype)
|
||||
if tuple(audio.shape) != audio_shape:
|
||||
raise ValueError(
|
||||
f"Provided audio_latents shape {list(audio.shape)} does not match expected "
|
||||
f"Kandinsky6 audio latent shape {list(audio_shape)}."
|
||||
)
|
||||
|
||||
batch.latents = video
|
||||
batch.audio_latents = audio
|
||||
batch.raw_latent_shape = (
|
||||
batch_size,
|
||||
num_channels,
|
||||
num_latent_frames,
|
||||
height // spatial_ratio,
|
||||
width // spatial_ratio,
|
||||
)
|
||||
return batch
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check("num_frames", batch.num_frames, V.positive_int)
|
||||
result.add_check("height", batch.height, V.positive_int)
|
||||
result.add_check("width", batch.width, V.positive_int)
|
||||
return result
|
||||
|
||||
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check("latents", batch.latents, [V.is_tensor, V.with_dims(5)])
|
||||
result.add_check(
|
||||
"audio_latents", batch.audio_latents, [V.is_tensor, V.with_dims(3)]
|
||||
)
|
||||
return result
|
||||
+8
@@ -0,0 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Kandinsky 6 video super-resolution (VSR) stages and framework-independent helpers.
|
||||
|
||||
Import the stage classes from their modules (``input_stage``, ``encode_stage``,
|
||||
``latent_prep_stage``, ``denoising_stage``, ``decode_stage``, ``output_stage``); this package
|
||||
``__init__`` stays empty so that configs can import the pure helpers (``video_utils``,
|
||||
``tiling``, ...) cheaply.
|
||||
"""
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Decode stage of Kandinsky 6 video SR: denoised latent chunks -> uint8 tiles."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_decode_parallel_group_coordinator,
|
||||
get_local_torch_device,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.common import (
|
||||
can_install_spatial_shard_parallel_decode,
|
||||
has_decode_parallel_world,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_DENOISED_KEY,
|
||||
SR_TILES_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
decode_chunks,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
|
||||
|
||||
|
||||
class Kandinsky6SRDecodeStage(PipelineStage):
|
||||
"""VAE-decodes every denoised chunk into row-major uint8 tiles (``[3, T, Hb, Wb]``)."""
|
||||
|
||||
def __init__(self, vae) -> None:
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
return [
|
||||
ComponentUse(
|
||||
self._component_stage_name(stage_name),
|
||||
"vae",
|
||||
phase="decode_tiles",
|
||||
target_dtype=vae_dtype,
|
||||
)
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
denoised = batch.extra.pop(SR_DENOISED_KEY)
|
||||
device = get_local_torch_device()
|
||||
group = None
|
||||
if (
|
||||
server_args.pipeline_config.vae_config.use_parallel_tiling
|
||||
and has_decode_parallel_world()
|
||||
and not can_install_spatial_shard_parallel_decode(
|
||||
server_args.pipeline_config.vae_config
|
||||
)
|
||||
):
|
||||
group = get_decode_parallel_group_coordinator()
|
||||
with self.use_declared_component(
|
||||
component_name="vae", module=self.vae, phase="decode_tiles"
|
||||
) as vae:
|
||||
assert vae is not None
|
||||
self.vae = vae
|
||||
tiles = decode_chunks(
|
||||
denoised,
|
||||
vae,
|
||||
scaling_factor=vae.scaling_factor,
|
||||
device=device,
|
||||
group=group,
|
||||
)
|
||||
batch.extra[SR_TILES_KEY] = tiles
|
||||
return batch
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Text-free SR tile denoising with shared residency, compile and cache-DiT hooks.
|
||||
|
||||
Each chunk resets its scheduler; the generic single-latent CFG loop and
|
||||
prompt-padding BCG path do not apply."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_CHUNKS_KEY,
|
||||
SR_DENOISED_KEY,
|
||||
SR_DIT_SPEC_KEY,
|
||||
SR_SAMPLING_SPEC_KEY,
|
||||
effective_scheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
denoise_chunks,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
StageValidators as V,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
VerificationResult,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision import precision_to_dtype
|
||||
|
||||
|
||||
class Kandinsky6SRDenoisingStage(DenoisingStage):
|
||||
"""Runs the bundle's scheduler over every tile chunk prepared by the latent-prep stage."""
|
||||
|
||||
def __init__(self, transformer, scheduler, pipeline=None) -> None:
|
||||
super().__init__(
|
||||
transformer=transformer,
|
||||
scheduler=scheduler,
|
||||
pipeline=pipeline,
|
||||
transformer_2=None,
|
||||
vae=None,
|
||||
)
|
||||
|
||||
def _owns_compile_warmup_lifecycle(self) -> bool:
|
||||
# ``forward`` below does not go through the shared ``_denoise`` loop that this guard
|
||||
# normally protects, so claim ownership explicitly (same as MiniMaxH3DenoisingStage):
|
||||
# the offload/restore wrapper is still entered through ``_offload_for_torch_compile_warmup``.
|
||||
return True
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
"""Validate tile-chunk state instead of the shared text/CFG loop inputs."""
|
||||
result = VerificationResult()
|
||||
result.add_check(
|
||||
"extra[SR_SAMPLING_SPEC_KEY]",
|
||||
batch.extra.get(SR_SAMPLING_SPEC_KEY),
|
||||
V.not_none,
|
||||
)
|
||||
result.add_check(
|
||||
"extra[SR_DIT_SPEC_KEY]", batch.extra.get(SR_DIT_SPEC_KEY), V.not_none
|
||||
)
|
||||
result.add_check(
|
||||
"extra[SR_CHUNKS_KEY]", batch.extra.get(SR_CHUNKS_KEY), V.list_not_empty
|
||||
)
|
||||
return result
|
||||
|
||||
def verify_output(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check(
|
||||
"extra[SR_DENOISED_KEY]", batch.extra.get(SR_DENOISED_KEY), V.list_not_empty
|
||||
)
|
||||
return result
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
dit_dtype = precision_to_dtype(
|
||||
server_args.pipeline_config.dit_precision, "dit_precision"
|
||||
)
|
||||
return [
|
||||
ComponentUse(
|
||||
self._component_stage_name(stage_name),
|
||||
"transformer",
|
||||
phase="denoise_tiles",
|
||||
target_dtype=dit_dtype,
|
||||
preferred_ready_after_request=True,
|
||||
memory_intensive=True,
|
||||
)
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
with self._offload_for_torch_compile_warmup(batch):
|
||||
return self._denoise_tiles(batch, server_args)
|
||||
|
||||
def _denoise_tiles(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
spec = batch.extra.pop(SR_SAMPLING_SPEC_KEY)
|
||||
dit_spec = batch.extra.pop(SR_DIT_SPEC_KEY)
|
||||
chunks = batch.extra.pop(SR_CHUNKS_KEY)
|
||||
device = get_local_torch_device()
|
||||
|
||||
# Reuse the shared cache-dit / torch.compile wiring (both are no-ops unless the
|
||||
# corresponding server arg actually requests them): ``spec.steps_per_chunk`` is this
|
||||
# request's DiT-call count, the closest analogue of the global ``num_inference_steps``
|
||||
# those hooks expect.
|
||||
self._maybe_enable_cache_dit_and_torch_compile(spec.steps_per_chunk, batch)
|
||||
|
||||
scheduler = effective_scheduler(spec, self.scheduler)
|
||||
with self.use_declared_component(
|
||||
component_name="transformer", module=self.transformer, phase="denoise_tiles"
|
||||
) as transformer:
|
||||
assert transformer is not None
|
||||
self.transformer = transformer
|
||||
total = spec.steps_per_chunk * len(chunks)
|
||||
with self.progress_bar(
|
||||
total=total, batch=batch, desc="Kandinsky6 SR denoising"
|
||||
) as progress:
|
||||
denoised = denoise_chunks(
|
||||
chunks,
|
||||
transformer,
|
||||
scheduler,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
step_context=lambda step: set_forward_context(
|
||||
current_timestep=step, attn_metadata=None, forward_batch=batch
|
||||
),
|
||||
on_step=progress.update,
|
||||
)
|
||||
self._finish_active_component_use()
|
||||
batch.extra[SR_DENOISED_KEY] = denoised
|
||||
return batch
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Whole-video KVAE encode of the latent-upscaler (LU) path of Kandinsky 6 video SR."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_LR_LATENT_KEY,
|
||||
SR_TILING_SCALE_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
uses_latent_path,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
encode_video_to_lr_latent,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision import resolve_precision
|
||||
|
||||
|
||||
class Kandinsky6SREncodeStage(PipelineStage):
|
||||
"""Encode the whole clip only for the LU path; pixel mode encodes tiles later."""
|
||||
|
||||
def __init__(self, vae, latent_upscaler=None) -> None:
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
# Only consulted to pick the path (does the bank have this scale?).
|
||||
self.latent_upscaler = latent_upscaler
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
return [
|
||||
ComponentUse(
|
||||
self._component_stage_name(stage_name),
|
||||
"vae",
|
||||
phase="encode_video",
|
||||
target_dtype=vae_dtype,
|
||||
start_at_stage_entry=False,
|
||||
)
|
||||
]
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
|
||||
if not uses_latent_path(self.latent_upscaler, tiling_scale):
|
||||
return batch
|
||||
video = batch.extra.pop(SR_VIDEO_KEY)
|
||||
with self.use_declared_component(
|
||||
component_name="vae", module=self.vae, phase="encode_video"
|
||||
) as vae:
|
||||
assert vae is not None
|
||||
self.vae = vae
|
||||
lr_latent = encode_video_to_lr_latent(
|
||||
video, vae, device=get_local_torch_device()
|
||||
)
|
||||
batch.extra[SR_LR_LATENT_KEY] = lr_latent.cpu()
|
||||
return batch
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Input stage of Kandinsky 6 video SR: source video -> aligned clip (+ source audio)."""
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_REQUESTED_HW_KEY,
|
||||
SR_TILING_SCALE_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
VAE_SPATIAL_FACTOR,
|
||||
pad_to_spatial_factor,
|
||||
pre_upscale_video,
|
||||
resolve_scale_request,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
|
||||
SOURCE_AUDIO_SAMPLE_RATE,
|
||||
DecodedClip,
|
||||
decode_clip,
|
||||
extract_source_audio,
|
||||
synthetic_clip,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
# Warmup requests carry no (usable) video: run the real path on a small synthetic clip.
|
||||
WARMUP_CLIP_FRAMES = 9
|
||||
WARMUP_CLIP_HW = (256, 384)
|
||||
|
||||
|
||||
class Kandinsky6SRInputStage(PipelineStage):
|
||||
"""Decode [T, 3, H, W] uint8 video, align fps/frames/size and retain source audio.
|
||||
|
||||
Record requested dimensions before VAE padding so output can remove it."""
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
tiling_scale, pre_upscale = resolve_scale_request(batch.sr_resolution_scale)
|
||||
clip = self._load_clip(batch)
|
||||
video = clip.frames
|
||||
if pre_upscale != 1.0:
|
||||
video = pre_upscale_video(video, pre_upscale, VAE_SPATIAL_FACTOR)
|
||||
frames = video.shape[0]
|
||||
# Align to the VAE spatial factor for every scale, not only 2.25 (whose pixel
|
||||
# pre-upscale happens to already land on an aligned size): without this, a source
|
||||
# whose dims are not already a multiple of 16 (e.g. a plain 1080p source at x2/x4,
|
||||
# which runs no pixel resize at all) fails partway through the whole-video KVAE
|
||||
# encode's four halvings instead of at a clear boundary. Padding (not resizing) keeps
|
||||
# every source pixel; batch.height / batch.width below, and the output stage's final
|
||||
# crop, restore the originally requested dimensions.
|
||||
video, (requested_h, requested_w) = pad_to_spatial_factor(
|
||||
video, VAE_SPATIAL_FACTOR
|
||||
)
|
||||
|
||||
batch.extra[SR_VIDEO_KEY] = video
|
||||
batch.extra[SR_TILING_SCALE_KEY] = tiling_scale
|
||||
batch.extra[SR_REQUESTED_HW_KEY] = (requested_h, requested_w)
|
||||
batch.fps = clip.fps
|
||||
batch.num_frames = frames
|
||||
batch.height = requested_h * tiling_scale
|
||||
batch.width = requested_w * tiling_scale
|
||||
self._attach_source_audio(batch, frames=frames, fps=clip.fps)
|
||||
return batch
|
||||
|
||||
def _load_clip(self, batch: Req) -> DecodedClip:
|
||||
if batch.is_warmup:
|
||||
height, width = WARMUP_CLIP_HW
|
||||
frames = synthetic_clip(
|
||||
height=height, width=width, frames=WARMUP_CLIP_FRAMES
|
||||
)
|
||||
return DecodedClip(frames=frames, fps=24, source_fps=24.0)
|
||||
path = batch.sampling_params.source_video_path()
|
||||
if path is None or not os.path.isfile(path):
|
||||
raise ValueError(f"Kandinsky6 SR input video not found: {path!r}")
|
||||
self.log_info("Decoding %s", path)
|
||||
return decode_clip(path)
|
||||
|
||||
@staticmethod
|
||||
def _attach_source_audio(batch: Req, *, frames: int, fps: int) -> None:
|
||||
if batch.is_warmup:
|
||||
return
|
||||
path = batch.sampling_params.source_video_path()
|
||||
audio = extract_source_audio(
|
||||
path, sample_rate=SOURCE_AUDIO_SAMPLE_RATE, max_seconds=frames / fps
|
||||
)
|
||||
if audio is None:
|
||||
batch.audio = None
|
||||
batch.audio_sample_rate = None
|
||||
return
|
||||
batch.audio = torch.from_numpy(audio)
|
||||
batch.audio_sample_rate = SOURCE_AUDIO_SAMPLE_RATE
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Prepare noisy tile chunks through the latent upscaler or pixel/VAE fallback.
|
||||
|
||||
Validate sampler/head compatibility and step count before processing tiles."""
|
||||
|
||||
from functools import partial
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_CHUNKS_KEY,
|
||||
SR_DIT_SPEC_KEY,
|
||||
SR_LR_LATENT_KEY,
|
||||
SR_PLAN_KEY,
|
||||
SR_SAMPLING_SPEC_KEY,
|
||||
SR_TILING_SCALE_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
build_dit_spec,
|
||||
build_sampling_spec,
|
||||
check_denoising_request,
|
||||
uses_latent_path,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
DitSpec,
|
||||
SamplingSpec,
|
||||
module_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
TilePlan,
|
||||
plan_tiles,
|
||||
prepare_lu_tile_latents,
|
||||
prepare_pixel_tile_latents,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
VAE_SPATIAL_FACTOR,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.precision import (
|
||||
resolve_precision,
|
||||
)
|
||||
|
||||
|
||||
class Kandinsky6SRLatentPrepStage(PipelineStage):
|
||||
"""LU-or-pixel tile encode + upscale -> initial noisy latent, one chunk at a time."""
|
||||
|
||||
def __init__(self, vae, transformer, latent_upscaler=None, scheduler=None) -> None:
|
||||
super().__init__()
|
||||
self.vae = vae
|
||||
self.transformer = transformer # arch-config / dtype only: never called here
|
||||
self.latent_upscaler = latent_upscaler
|
||||
self.scheduler = (
|
||||
scheduler # resolved sampler validation only: never stepped here
|
||||
)
|
||||
|
||||
def component_uses(
|
||||
self, server_args: ServerArgs, stage_name: str | None = None
|
||||
) -> list[ComponentUse]:
|
||||
stage_name = self._component_stage_name(stage_name)
|
||||
vae_dtype = resolve_precision(
|
||||
server_args, "vae", precision_attr="vae_precision"
|
||||
)
|
||||
uses: list[ComponentUse] = []
|
||||
if self.latent_upscaler is not None:
|
||||
uses.append(
|
||||
ComponentUse(
|
||||
stage_name,
|
||||
"latent_upscaler",
|
||||
phase="upscale_tiles",
|
||||
target_dtype=torch.bfloat16,
|
||||
)
|
||||
)
|
||||
uses.append(
|
||||
ComponentUse(
|
||||
stage_name, "vae", phase="encode_tiles", target_dtype=vae_dtype
|
||||
)
|
||||
)
|
||||
return uses
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
arch = server_args.pipeline_config.dit_config.arch_config
|
||||
tiling_scale = batch.extra[SR_TILING_SCALE_KEY]
|
||||
seed = batch.seed if isinstance(batch.seed, int) else batch.seed[0]
|
||||
spec = build_sampling_spec(
|
||||
arch=arch,
|
||||
tiling_scale=tiling_scale,
|
||||
seed=seed,
|
||||
num_steps=batch.num_inference_steps,
|
||||
tiles_batch_size=batch.sr_tiles_batch_size,
|
||||
tile_min_overlap=batch.sr_tile_min_overlap,
|
||||
scheduler=self.scheduler,
|
||||
)
|
||||
check_denoising_request(spec, batch.num_inference_steps)
|
||||
dit_spec = build_dit_spec(self.transformer)
|
||||
device = get_local_torch_device()
|
||||
|
||||
plan, chunks = self._prepare_chunks(batch, spec, dit_spec, device)
|
||||
batch.extra[SR_PLAN_KEY] = plan
|
||||
batch.extra[SR_CHUNKS_KEY] = chunks
|
||||
batch.extra[SR_SAMPLING_SPEC_KEY] = spec
|
||||
batch.extra[SR_DIT_SPEC_KEY] = dit_spec
|
||||
return batch
|
||||
|
||||
def _prepare_chunks(
|
||||
self, batch: Req, spec: SamplingSpec, dit_spec: DitSpec, device: torch.device
|
||||
) -> tuple[TilePlan, list[torch.Tensor]]:
|
||||
common = dict(
|
||||
visual_size=spec.visual_size,
|
||||
tiling_scale=spec.tiling_scale,
|
||||
tile_min_overlap=spec.tile_min_overlap,
|
||||
)
|
||||
if uses_latent_path(self.latent_upscaler, spec.tiling_scale):
|
||||
lr_latent = batch.extra.pop(SR_LR_LATENT_KEY)
|
||||
plan = plan_tiles(
|
||||
frame_hw=(
|
||||
lr_latent.shape[-2] * VAE_SPATIAL_FACTOR,
|
||||
lr_latent.shape[-1] * VAE_SPATIAL_FACTOR,
|
||||
),
|
||||
**common,
|
||||
)
|
||||
with self.use_declared_component(
|
||||
component_name="latent_upscaler",
|
||||
module=self.latent_upscaler,
|
||||
phase="upscale_tiles",
|
||||
) as bank:
|
||||
assert bank is not None
|
||||
self.latent_upscaler = bank
|
||||
chunks = prepare_lu_tile_latents(
|
||||
lr_latent,
|
||||
plan,
|
||||
upscale_fn=partial(bank.upscale, scale=spec.tiling_scale),
|
||||
lu_dtype=module_dtype(bank),
|
||||
scaling_factor=self.vae.scaling_factor,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
return plan, chunks
|
||||
|
||||
video = batch.extra.pop(SR_VIDEO_KEY)
|
||||
plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common)
|
||||
with self.use_declared_component(
|
||||
component_name="vae", module=self.vae, phase="encode_tiles"
|
||||
) as vae:
|
||||
assert vae is not None
|
||||
self.vae = vae
|
||||
chunks = prepare_pixel_tile_latents(
|
||||
video,
|
||||
plan,
|
||||
vae=vae,
|
||||
scaling_factor=vae.scaling_factor,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
return plan, chunks
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Vendored from sr_core (parity-tested against the k6_video reference); adapted from
|
||||
# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0).
|
||||
"""SR latent construction with reference RNG ordering.
|
||||
|
||||
Initial noise uses a per-request generator; conditioning noise uses the global RNG."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
import torch
|
||||
|
||||
InstructType = Literal["noise", "channel", "hybrid", "hybrid_anchor"]
|
||||
LqNoiseType = Literal["linear", "ddpm"]
|
||||
|
||||
|
||||
def degrade_lq_latent(
|
||||
lq_latent: torch.Tensor,
|
||||
noise_scale: float = 0.7,
|
||||
noise_type: LqNoiseType = "linear",
|
||||
generator: torch.Generator | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Mix linear or variance-preserving (ddpm) noise; nonpositive scales are a no-op."""
|
||||
if noise_scale <= 0:
|
||||
return lq_latent
|
||||
eps = torch.randn(
|
||||
lq_latent.shape,
|
||||
device=lq_latent.device,
|
||||
dtype=lq_latent.dtype,
|
||||
generator=generator,
|
||||
)
|
||||
if noise_type == "ddpm":
|
||||
return (1 - noise_scale**2) ** 0.5 * lq_latent + noise_scale * eps
|
||||
return (1 - noise_scale) * lq_latent + noise_scale * eps
|
||||
|
||||
|
||||
def _cast_floating(value: torch.Tensor, dtype: torch.dtype | None) -> torch.Tensor:
|
||||
"""Cast floating-point ``value`` to ``dtype`` (no-op for ``None`` / non-float / same dtype)."""
|
||||
if dtype is not None and value.is_floating_point() and value.dtype != dtype:
|
||||
return value.to(dtype=dtype)
|
||||
return value
|
||||
|
||||
|
||||
def build_initial_latent( # noqa: PLR0913
|
||||
*,
|
||||
instruct_type: InstructType,
|
||||
visual_cond: bool,
|
||||
in_visual_dim: int,
|
||||
lq_latent: torch.Tensor,
|
||||
batch_size: int,
|
||||
duration: int,
|
||||
height: int,
|
||||
width: int,
|
||||
device: str | int | torch.device,
|
||||
seed: int,
|
||||
lq_noise_scale: float,
|
||||
lq_noise_type: LqNoiseType,
|
||||
lq_channel_noise_scale: float,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Build [starting | conditioning | mask] for the selected instruction mode.
|
||||
|
||||
Inputs and output use [B*T, H, W, C]; the output has 2C+1 channels except
|
||||
unconditioned noise mode. Cast LQ to the DiT dtype before drawing noise."""
|
||||
lq_latent = lq_latent.to(device)
|
||||
lq_latent = _cast_floating(lq_latent, dtype)
|
||||
|
||||
if instruct_type == "noise":
|
||||
g_noise = torch.Generator(device=device)
|
||||
g_noise.manual_seed(seed)
|
||||
degraded_lq = degrade_lq_latent(
|
||||
lq_latent, lq_noise_scale, lq_noise_type, generator=g_noise
|
||||
)
|
||||
if not visual_cond:
|
||||
# No conditioning channels in the model input layer (e.g. the
|
||||
# t2v-initialized KVAE SR DiT) — feed the bare C-channel latent.
|
||||
return degraded_lq
|
||||
zero_cond = torch.zeros_like(degraded_lq)
|
||||
zero_mask = torch.zeros(
|
||||
[*degraded_lq.shape[:-1], 1], dtype=degraded_lq.dtype, device=device
|
||||
)
|
||||
return torch.cat([degraded_lq, zero_cond, zero_mask], dim=-1)
|
||||
|
||||
if instruct_type in ("channel", "hybrid"):
|
||||
if instruct_type == "hybrid":
|
||||
g_noise = torch.Generator(device=device)
|
||||
g_noise.manual_seed(seed)
|
||||
starting = degrade_lq_latent(
|
||||
lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise
|
||||
)
|
||||
else:
|
||||
g = torch.Generator(device=device)
|
||||
g.manual_seed(seed)
|
||||
starting = torch.randn(
|
||||
batch_size * duration,
|
||||
height,
|
||||
width,
|
||||
in_visual_dim,
|
||||
device=device,
|
||||
generator=g,
|
||||
)
|
||||
channel_lq = degrade_lq_latent(
|
||||
lq_latent, lq_channel_noise_scale, noise_type="linear"
|
||||
)
|
||||
mask = torch.ones_like(lq_latent[..., :1])
|
||||
return torch.cat([starting, channel_lq, mask], dim=-1)
|
||||
|
||||
if instruct_type == "hybrid_anchor":
|
||||
# Anchor-free inference (tiled real-world LQ has no HR ground truth):
|
||||
# a zeroed anchor + zeroed mask runs the model with no anchor signal,
|
||||
# equivalent to anchor_indices=[] for the packed-latent datasets.
|
||||
anchor_latent = torch.zeros_like(lq_latent)
|
||||
anchor_mask = torch.zeros_like(lq_latent[..., :1])
|
||||
g_noise = torch.Generator(device=device)
|
||||
g_noise.manual_seed(seed)
|
||||
starting = degrade_lq_latent(
|
||||
lq_latent.clone(), lq_noise_scale, lq_noise_type, generator=g_noise
|
||||
)
|
||||
anchor = degrade_lq_latent(
|
||||
anchor_latent, lq_channel_noise_scale, noise_type="linear"
|
||||
)
|
||||
return torch.cat([starting, anchor, anchor_mask], dim=-1)
|
||||
|
||||
msg = f"generate_sample_sr does not support instruct_type={instruct_type!r}"
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"InstructType",
|
||||
"LqNoiseType",
|
||||
"build_initial_latent",
|
||||
"degrade_lq_latent",
|
||||
]
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Stitch / crop / resize / output stage of Kandinsky 6 video SR."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
|
||||
resolve_target_hw,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import PipelineStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_PLAN_KEY,
|
||||
SR_REQUESTED_HW_KEY,
|
||||
SR_TILES_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
stitch_tiles,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
crop_to_hw,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
|
||||
to_output_video,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
|
||||
resize_to_target,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
|
||||
class Kandinsky6SROutputStage(PipelineStage):
|
||||
"""Hann-blends the decoded tiles, restores the requested output size, applies the
|
||||
optional delivery resize and returns ``output = [1, 3, T, H, W]`` (fp16 in [0, 1])
|
||||
plus the mono source audio.
|
||||
|
||||
CPU only: every model phase already ran. ``batch.height`` / ``batch.width`` are set
|
||||
to the final size, ``batch.fps`` was set by the input stage (and now also carried on
|
||||
the returned ``OutputBatch``, so a worker-resampled fps reaches the client-side save
|
||||
path unchanged).
|
||||
"""
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
|
||||
tiles = batch.extra.pop(SR_TILES_KEY)
|
||||
plan = batch.extra.pop(SR_PLAN_KEY)
|
||||
video = stitch_tiles(tiles, plan)
|
||||
del tiles
|
||||
|
||||
requested_hw = batch.extra.pop(SR_REQUESTED_HW_KEY, None)
|
||||
if requested_hw is not None:
|
||||
requested_h, requested_w = requested_hw
|
||||
video = crop_to_hw(
|
||||
video,
|
||||
(requested_h * plan.tiling_scale, requested_w * plan.tiling_scale),
|
||||
)
|
||||
|
||||
target_hw = resolve_target_hw(
|
||||
batch.sr_target_resolution,
|
||||
tuple(video.shape[-2:]),
|
||||
batch.sr_target_resize_mode,
|
||||
)
|
||||
if target_hw is not None:
|
||||
video = resize_to_target(video, target_hw)
|
||||
batch.height, batch.width = video.shape[-2:]
|
||||
return OutputBatch(
|
||||
output=to_output_video(video),
|
||||
audio=batch.audio,
|
||||
audio_sample_rate=batch.audio_sample_rate,
|
||||
fps=batch.fps,
|
||||
metrics=batch.metrics,
|
||||
usage=batch.usage,
|
||||
)
|
||||
+221
@@ -0,0 +1,221 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""SR request state and sampler resolution.
|
||||
|
||||
batch.extra contains only CPU tensors and serializable data, never model objects."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRArchConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
|
||||
PiflowScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
DitSpec,
|
||||
PiflowParams,
|
||||
SamplingSpec,
|
||||
module_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
SR_VIDEO_KEY = (
|
||||
"kandinsky6_sr_video" # [T, 3, H, W] uint8 clip (after pre-upscale + pad)
|
||||
)
|
||||
SR_TILING_SCALE_KEY = "kandinsky6_sr_tiling_scale" # int tile scale (2 or 4)
|
||||
SR_LR_LATENT_KEY = "kandinsky6_sr_lr_latent" # [T', C, h, w] fp32 (LU path)
|
||||
SR_PLAN_KEY = "kandinsky6_sr_tile_plan" # tiled.TilePlan
|
||||
SR_TILES_KEY = "kandinsky6_sr_tiles" # list of uint8 [3, T, Hb, Wb]
|
||||
# Handoff between the three model-phase stages (latent prep -> denoise -> decode), split out
|
||||
# of the single combined stage so the denoising stage can subclass the shared DenoisingStage
|
||||
# (see denoising_stage.py): SamplingSpec / DitSpec are immutable msgspec.Struct values (plain
|
||||
# data, like TilePlan above), and the chunk lists are CPU tensors throughout.
|
||||
SR_SAMPLING_SPEC_KEY = "kandinsky6_sr_sampling_spec" # sampling.SamplingSpec
|
||||
SR_DIT_SPEC_KEY = "kandinsky6_sr_dit_spec" # sampling.DitSpec
|
||||
SR_CHUNKS_KEY = (
|
||||
"kandinsky6_sr_chunks" # list of fp32 [B, T', H', W', 2C+1] (pre-denoise)
|
||||
)
|
||||
SR_DENOISED_KEY = (
|
||||
"kandinsky6_sr_denoised" # list of fp32 [B, T', H', W', C] (post-denoise)
|
||||
)
|
||||
# (H, W) of the clip actually requested, before any VAE-spatial-factor alignment padding the
|
||||
# input stage added so whole-video KVAE encoding survives every supported sr_resolution_scale
|
||||
# (not only 2.25, whose pixel pre-upscale happens to already land on an aligned size). The
|
||||
# output stage crops the stitched result from (padded_H * scale, padded_W * scale) down to
|
||||
# (H * scale, W * scale) so the delivered video matches what was actually asked for.
|
||||
SR_REQUESTED_HW_KEY = "kandinsky6_sr_requested_hw"
|
||||
|
||||
|
||||
def uses_latent_path(latent_upscaler: Any, tiling_scale: int) -> bool:
|
||||
"""LU path iff the bank has an entry for the tiling scale (else the pixel path)."""
|
||||
return (
|
||||
latent_upscaler is not None
|
||||
and latent_upscaler.for_scale(tiling_scale) is not None
|
||||
)
|
||||
|
||||
|
||||
def build_dit_spec(dit: Any) -> DitSpec:
|
||||
"""Resolved DiT input and conditioning layout."""
|
||||
return DitSpec(
|
||||
instruct_type=dit.instruct_type,
|
||||
visual_cond=bool(dit.visual_cond),
|
||||
in_visual_dim=dit.in_visual_dim,
|
||||
use_motion_score=bool(dit.use_motion_score),
|
||||
patch_size=tuple(dit.patch_size),
|
||||
dtype=module_dtype(dit),
|
||||
)
|
||||
|
||||
|
||||
def _rope_scale_factor(arch: Kandinsky6SRArchConfig) -> tuple[float, float, float]:
|
||||
table = arch.sr_scale_factor
|
||||
visual_size = arch.sr_visual_size[0]
|
||||
if table is None:
|
||||
raise ValueError(
|
||||
"transformer/config.json carries no sr_params (visual_size / scale_factor); "
|
||||
"an official Diffusers Kandinsky6SRPipeline repo is required"
|
||||
)
|
||||
values = table.get(str(visual_size), table.get(visual_size))
|
||||
if values is None:
|
||||
raise ValueError(
|
||||
f"sr_scale_factor has no entry for visual size {visual_size}: {table}"
|
||||
)
|
||||
return tuple(float(value) for value in values)
|
||||
|
||||
|
||||
def piflow_params_from_scheduler(scheduler: Any) -> PiflowParams | None:
|
||||
"""pi-Flow sampler values of a ``PiflowScheduler`` component (``None``: flow-Euler)."""
|
||||
if not isinstance(scheduler, PiflowScheduler):
|
||||
return None
|
||||
config = scheduler.config
|
||||
if config["nfe"] is None:
|
||||
return None
|
||||
return PiflowParams(
|
||||
nfe=int(config["nfe"]),
|
||||
num_policy_substeps=int(config["num_policy_substeps"]),
|
||||
final_step_size_scale=float(config["final_step_size_scale"]),
|
||||
shift=float(config["shift"]),
|
||||
n_grid=int(config["n_grid"]),
|
||||
eps=float(config["eps"]),
|
||||
)
|
||||
|
||||
|
||||
def _describe_scheduler(scheduler: Any) -> str:
|
||||
if scheduler is None:
|
||||
return "missing"
|
||||
if isinstance(scheduler, PiflowScheduler):
|
||||
return "a PiflowScheduler without nfe"
|
||||
return f"a {type(scheduler).__name__}"
|
||||
|
||||
|
||||
def _check_sampler_fits_head(
|
||||
*, arch: Kandinsky6SRArchConfig, piflow: PiflowParams | None, scheduler: Any
|
||||
) -> None:
|
||||
"""Reject incompatible scheduler and DiT head widths before encoding tiles."""
|
||||
head, grid = arch.head_width, arch.base_out_visual_dim
|
||||
if piflow is None:
|
||||
if head != grid:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR: the transformer head is {head} channels wide (a DX / "
|
||||
f"pi-Flow head: {grid} channels per latent), but the sampler resolved "
|
||||
"to flow-Euler: the scheduler component is "
|
||||
f"{_describe_scheduler(scheduler)}, and only a PiflowScheduler with "
|
||||
"nfe selects pi-Flow. Use the scheduler of the same repo as the "
|
||||
"transformer."
|
||||
)
|
||||
return
|
||||
if piflow.nfe < 1:
|
||||
raise ValueError(
|
||||
"Kandinsky6SR: the pi-Flow sampler needs nfe >= 1 model calls per tile, "
|
||||
f"got nfe={piflow.nfe}."
|
||||
)
|
||||
if head != piflow.n_grid * grid:
|
||||
raise ValueError(
|
||||
f"Kandinsky6SR: the PiflowScheduler (nfe {piflow.nfe}, n_grid "
|
||||
f"{piflow.n_grid}) needs a {piflow.n_grid} x {grid} = "
|
||||
f"{piflow.n_grid * grid} channel DX head, but the transformer head is "
|
||||
f"{head} channels wide (out_visual_dim {arch.out_visual_dim}). The "
|
||||
"scheduler and the transformer come from different repos."
|
||||
)
|
||||
|
||||
|
||||
def build_sampling_spec(
|
||||
*,
|
||||
arch: Kandinsky6SRArchConfig,
|
||||
tiling_scale: int,
|
||||
seed: int,
|
||||
num_steps: int,
|
||||
tiles_batch_size: int,
|
||||
tile_min_overlap: float,
|
||||
scheduler: Any = None,
|
||||
) -> SamplingSpec:
|
||||
"""Resolve sampling from the bundle scheduler, falling back to legacy DiT fields.
|
||||
|
||||
num_steps counts DiT calls, not Diffusers timestep grid points."""
|
||||
piflow = piflow_params_from_scheduler(scheduler)
|
||||
if piflow is None and arch.is_piflow:
|
||||
piflow = PiflowParams(
|
||||
nfe=arch.piflow_nfe,
|
||||
num_policy_substeps=arch.piflow_num_policy_substeps,
|
||||
final_step_size_scale=arch.piflow_final_step_size_scale,
|
||||
shift=arch.piflow_shift,
|
||||
n_grid=arch.n_grid,
|
||||
eps=arch.piflow_eps,
|
||||
)
|
||||
_check_sampler_fits_head(arch=arch, piflow=piflow, scheduler=scheduler)
|
||||
return SamplingSpec(
|
||||
tiling_scale=tiling_scale,
|
||||
tiles_batch_size=tiles_batch_size,
|
||||
seed=seed,
|
||||
num_steps=num_steps,
|
||||
tile_min_overlap=tile_min_overlap,
|
||||
visual_size=arch.sr_visual_size[0],
|
||||
scale_factor=_rope_scale_factor(arch),
|
||||
scheduler_scale=arch.sr_scheduler_scale,
|
||||
lq_noise_scale=arch.sr_lq_noise_scale,
|
||||
lq_noise_type=arch.sr_lq_noise_type,
|
||||
lq_channel_noise_scale=arch.sr_lq_channel_noise_scale,
|
||||
cap_noise_timestep=arch.sr_cap_noise_timestep,
|
||||
piflow=piflow,
|
||||
)
|
||||
|
||||
|
||||
def effective_scheduler(spec: SamplingSpec, scheduler: Any) -> Any:
|
||||
"""Use the loaded scheduler when compatible; synthesize one for legacy configs."""
|
||||
if spec.piflow is not None:
|
||||
if isinstance(scheduler, PiflowScheduler):
|
||||
return scheduler
|
||||
return PiflowScheduler(
|
||||
nfe=spec.piflow.nfe,
|
||||
n_grid=spec.piflow.n_grid,
|
||||
shift=spec.piflow.shift,
|
||||
eps=spec.piflow.eps,
|
||||
final_step_size_scale=spec.piflow.final_step_size_scale,
|
||||
num_policy_substeps=spec.piflow.num_policy_substeps,
|
||||
)
|
||||
if isinstance(scheduler, FlowMatchEulerDiscreteScheduler):
|
||||
return scheduler
|
||||
return FlowMatchEulerDiscreteScheduler(shift=spec.scheduler_scale)
|
||||
|
||||
|
||||
def check_denoising_request(spec: SamplingSpec, num_inference_steps: int) -> None:
|
||||
"""Validate the requested DiT-call count before denoising.
|
||||
|
||||
Diffusers counts timestep grid points instead: its default 5 is 4 calls here."""
|
||||
if num_inference_steps < 1:
|
||||
raise ValueError(
|
||||
f"Kandinsky6 SR: num_inference_steps must be >= 1, got {num_inference_steps}"
|
||||
)
|
||||
if spec.piflow is not None and num_inference_steps != spec.piflow.nfe:
|
||||
logger.warning(
|
||||
"Kandinsky6 SR: num_inference_steps=%d was requested, but this is a pi-Flow "
|
||||
"bundle distilled for nfe=%d; num_inference_steps is ignored for pi-Flow "
|
||||
"bundles and every tile will run exactly nfe=%d DiT calls instead.",
|
||||
num_inference_steps,
|
||||
spec.piflow.nfe,
|
||||
spec.piflow.nfe,
|
||||
)
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Scheduler-driven denoising for Kandinsky 6 SR tile chunks."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
# (latent [B, T, H, W, C], model time [B] (already * 1000), step index) -> DiT output
|
||||
DitFn = Callable[[torch.Tensor, torch.Tensor, int], torch.Tensor]
|
||||
StepCallback = Callable[[], None]
|
||||
StepContext = Callable[[int], AbstractContextManager]
|
||||
|
||||
|
||||
class PiflowParams(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""pi-Flow parameters from the scheduler or legacy flat DiT config."""
|
||||
|
||||
nfe: int
|
||||
num_policy_substeps: int
|
||||
final_step_size_scale: float
|
||||
shift: float
|
||||
n_grid: int
|
||||
eps: float
|
||||
|
||||
|
||||
class DitSpec(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Post-override behaviour of the DiT that the sampling code needs."""
|
||||
|
||||
instruct_type: str | None
|
||||
visual_cond: bool
|
||||
in_visual_dim: int
|
||||
use_motion_score: bool
|
||||
patch_size: tuple[int, int, int]
|
||||
# The DiT's loaded parameter dtype (``module_dtype(dit)``); the reference casts the
|
||||
# LQ latent to this dtype before building the initial latent, so a bf16-loaded
|
||||
# checkpoint's LQ-noise mix rounds in bf16, not fp32. ``None`` (parameter-less DiT,
|
||||
# e.g. a mocked one in a test) keeps the LQ latent's own dtype.
|
||||
dtype: torch.dtype | None = None
|
||||
|
||||
|
||||
class SamplingSpec(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Everything besides the components that defines one tiled SR run."""
|
||||
|
||||
tiling_scale: int
|
||||
tiles_batch_size: int
|
||||
seed: int
|
||||
num_steps: int
|
||||
tile_min_overlap: float
|
||||
visual_size: int
|
||||
scale_factor: tuple[float, float, float]
|
||||
scheduler_scale: float
|
||||
lq_noise_scale: float
|
||||
lq_noise_type: str
|
||||
lq_channel_noise_scale: float
|
||||
cap_noise_timestep: bool
|
||||
piflow: PiflowParams | None
|
||||
|
||||
@property
|
||||
def steps_per_chunk(self) -> int:
|
||||
"""DiT calls per chunk of tiles: ``nfe`` (pi-Flow), else ``num_steps``."""
|
||||
return self.piflow.nfe if self.piflow is not None else self.num_steps
|
||||
|
||||
|
||||
def bf16_autocast(device: torch.device | str) -> AbstractContextManager:
|
||||
"""CUDA bf16 autocast (a no-op on other devices), as the reference wraps its models."""
|
||||
if torch.device(device).type != "cuda":
|
||||
return nullcontext()
|
||||
return torch.autocast(device_type="cuda", dtype=torch.bfloat16)
|
||||
|
||||
|
||||
def module_dtype(module: torch.nn.Module) -> torch.dtype | None:
|
||||
"""Dtype of the first parameter (``None`` for a parameter-less module)."""
|
||||
return next((param.dtype for param in module.parameters()), None)
|
||||
|
||||
|
||||
def cast_to_module_dtype(module: torch.nn.Module, value: torch.Tensor) -> torch.Tensor:
|
||||
"""Cast a floating tensor to the module's parameter dtype (reference helper)."""
|
||||
dtype = module_dtype(module)
|
||||
if dtype is not None and value.is_floating_point() and value.dtype != dtype:
|
||||
return value.to(dtype=dtype)
|
||||
return value
|
||||
|
||||
|
||||
def euler_start_timestep(
|
||||
*, cap_noise_timestep: bool, lq_noise_scale: float, instruct_type: str | None
|
||||
) -> float:
|
||||
"""Euler start time: the LQ noise level when capped for noise-type conditioning."""
|
||||
capped = cap_noise_timestep and instruct_type in (
|
||||
"noise",
|
||||
"hybrid",
|
||||
"hybrid_anchor",
|
||||
)
|
||||
return lq_noise_scale if capped else 1.0
|
||||
|
||||
|
||||
def check_sampler_options(
|
||||
*, piflow: PiflowParams | None, cap_noise_timestep: bool, instruct_type: str | None
|
||||
) -> None:
|
||||
"""Reject option combinations the reference sampler rejects."""
|
||||
if (
|
||||
piflow is not None
|
||||
and cap_noise_timestep
|
||||
and instruct_type in ("noise", "hybrid")
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"piflow sampler does not support cap_noise_timestep for noise/hybrid "
|
||||
"instruct"
|
||||
)
|
||||
|
||||
|
||||
def make_dit_fn(
|
||||
dit: Callable[..., torch.Tensor],
|
||||
*,
|
||||
latent_frames_hw: tuple[int, int, int],
|
||||
patch_size: tuple[int, int, int],
|
||||
scale_factor: tuple[float, float, float],
|
||||
use_motion_score: bool,
|
||||
step_context: StepContext | None = None,
|
||||
) -> DitFn:
|
||||
"""Bind per-chunk RoPE/motion inputs and an optional per-step context."""
|
||||
frames, height, width = latent_frames_hw
|
||||
context = step_context or (lambda step: nullcontext())
|
||||
|
||||
def call(x: torch.Tensor, model_time: torch.Tensor, step: int) -> torch.Tensor:
|
||||
rope_pos = [
|
||||
torch.arange(frames, device=x.device),
|
||||
torch.arange(height // patch_size[1], device=x.device),
|
||||
torch.arange(width // patch_size[2], device=x.device),
|
||||
]
|
||||
motion = (
|
||||
torch.full((1,), 900.0, device=x.device, dtype=x.dtype)
|
||||
if use_motion_score
|
||||
else None
|
||||
)
|
||||
with context(step):
|
||||
return dit(
|
||||
x,
|
||||
model_time,
|
||||
rope_pos,
|
||||
scale_factor=scale_factor,
|
||||
motion_score=motion,
|
||||
)
|
||||
|
||||
return call
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def denoise_with_scheduler(
|
||||
x: torch.Tensor,
|
||||
dit_fn: DitFn,
|
||||
scheduler: Any,
|
||||
*,
|
||||
channels: int,
|
||||
is_piflow: bool,
|
||||
on_step: StepCallback | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Denoise one chunk after set_timesteps; keep conditioning channels fixed.
|
||||
|
||||
Promote predictions to fp32 so flow-Euler does not downcast its running
|
||||
state to the bf16 prediction dtype. pi-Flow already preserves the state dtype."""
|
||||
state = x[..., :channels].float()
|
||||
cond = x[..., channels:]
|
||||
for step, t in enumerate(scheduler.timesteps):
|
||||
model_time = t.to(device=x.device, dtype=torch.float32).expand(state.shape[0])
|
||||
hidden_states = torch.cat([state, cond], dim=-1) if cond.shape[-1] else state
|
||||
prediction = dit_fn(hidden_states, model_time, step)
|
||||
if not is_piflow:
|
||||
prediction = prediction.float()
|
||||
state = scheduler.step(prediction, t, state, return_dict=False)[0]
|
||||
if on_step is not None:
|
||||
on_step()
|
||||
return state.float()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DitFn",
|
||||
"DitSpec",
|
||||
"PiflowParams",
|
||||
"SamplingSpec",
|
||||
"StepCallback",
|
||||
"StepContext",
|
||||
"bf16_autocast",
|
||||
"cast_to_module_dtype",
|
||||
"check_sampler_options",
|
||||
"denoise_with_scheduler",
|
||||
"euler_start_timestep",
|
||||
"make_dit_fn",
|
||||
"module_dtype",
|
||||
]
|
||||
+413
@@ -0,0 +1,413 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Tile preparation, denoising and blending for Kandinsky 6 SR.
|
||||
|
||||
Stages group work by component to avoid swapping weights per tile.
|
||||
Each chunk uses seed + its first tile index, matching the reference.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed.group_coordinator import GroupCoordinator
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latents import (
|
||||
build_initial_latent,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
DitSpec,
|
||||
SamplingSpec,
|
||||
StepCallback,
|
||||
StepContext,
|
||||
bf16_autocast,
|
||||
cast_to_module_dtype,
|
||||
check_sampler_options,
|
||||
denoise_with_scheduler,
|
||||
euler_start_timestep,
|
||||
make_dit_fn,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
RESOLUTIONS,
|
||||
VAE_SPATIAL_FACTOR,
|
||||
TileGrid,
|
||||
closest_base_resolution,
|
||||
compute_tile_grid_even,
|
||||
extract_all_tiles,
|
||||
latent_tile_grid_from_pixel_grid,
|
||||
stitch_tiles_hanning,
|
||||
upsample_tiles_to_base,
|
||||
)
|
||||
|
||||
LatentUpscaleFn = Callable[[torch.Tensor], torch.Tensor]
|
||||
STITCH_FRAME_CHUNK = 8
|
||||
|
||||
|
||||
class TilePlan(msgspec.Struct, frozen=True):
|
||||
"""Tile geometry of one run (all sizes in pixels of the frame that is tiled)."""
|
||||
|
||||
tiling_scale: int
|
||||
frame_hw: tuple[int, int]
|
||||
base_hw: tuple[int, int]
|
||||
tile_hw: tuple[int, int]
|
||||
pixel_grid: TileGrid
|
||||
|
||||
|
||||
def plan_tiles(
|
||||
*,
|
||||
frame_hw: tuple[int, int],
|
||||
visual_size: int,
|
||||
tiling_scale: int,
|
||||
tile_min_overlap: float,
|
||||
spatial_factor: int = VAE_SPATIAL_FACTOR,
|
||||
resolutions: Mapping[int, Sequence[tuple[int, int]]] = RESOLUTIONS,
|
||||
) -> TilePlan:
|
||||
"""Even tile grid for a frame; the frame must hold at least one full tile."""
|
||||
height, width = frame_hw
|
||||
base_hw = closest_base_resolution(height, width, visual_size, resolutions)
|
||||
if any(size % tiling_scale for size in base_hw):
|
||||
raise ValueError(
|
||||
f"resolution_scale={tiling_scale} does not divide the base resolution "
|
||||
f"{base_hw[0]}x{base_hw[1]} exactly"
|
||||
)
|
||||
tile_hw = (base_hw[0] // tiling_scale, base_hw[1] // tiling_scale)
|
||||
grid = compute_tile_grid_even(
|
||||
height, width, tile_hw, tile_min_overlap, spatial_factor
|
||||
)
|
||||
if height < tile_hw[0] or width < tile_hw[1]:
|
||||
raise ValueError(
|
||||
f"The {width}x{height} frame is smaller than one SR tile "
|
||||
f"({tile_hw[1]}x{tile_hw[0]}) at x{tiling_scale}; use a larger input or a "
|
||||
"higher scale."
|
||||
)
|
||||
return TilePlan(
|
||||
tiling_scale=tiling_scale,
|
||||
frame_hw=(height, width),
|
||||
base_hw=base_hw,
|
||||
tile_hw=tile_hw,
|
||||
pixel_grid=grid,
|
||||
)
|
||||
|
||||
|
||||
def chunk_ranges(num_tiles: int, tiles_batch_size: int) -> Iterator[tuple[int, int]]:
|
||||
"""``(start, stop)`` tile ranges; ``start`` also offsets the chunk's seed."""
|
||||
for start in range(0, num_tiles, tiles_batch_size):
|
||||
yield start, min(start + tiles_batch_size, num_tiles)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def encode_video_to_lr_latent(
|
||||
video: torch.Tensor, vae: torch.nn.Module, *, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
"""Encode a whole ``[T, C, H, W]`` uint8 video to the raw latent ``[T', C', h, w]``.
|
||||
|
||||
The latent is *not* multiplied by the VAE scaling factor (the LU does that itself).
|
||||
"""
|
||||
if video.ndim != 4:
|
||||
raise ValueError(f"video must be [T, C, H, W], got {tuple(video.shape)}")
|
||||
pixel = video.permute(1, 0, 2, 3).unsqueeze(0).to(device=device)
|
||||
pixel = cast_to_module_dtype(vae, vae.normalize_data(pixel.float()))
|
||||
latent = vae.encode(pixel)[0]
|
||||
return latent.squeeze(0).permute(1, 0, 2, 3).float()
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def upscale_lr_latent_tile(
|
||||
tile: torch.Tensor,
|
||||
upscale_fn: LatentUpscaleFn,
|
||||
*,
|
||||
lu_dtype: torch.dtype | None,
|
||||
scaling_factor: float,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
"""LU on one raw tile ``[T', C', h, w]`` -> scaled latent ``[T', Hb, Wb, C']``."""
|
||||
z = tile.permute(1, 0, 2, 3).unsqueeze(0).to(device=device, dtype=torch.float32)
|
||||
if lu_dtype is not None and z.dtype != lu_dtype:
|
||||
z = z.to(dtype=lu_dtype)
|
||||
with bf16_autocast(device):
|
||||
upscaled = upscale_fn(z * scaling_factor)
|
||||
return upscaled.squeeze(0).permute(1, 2, 3, 0).float()
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def encode_pixel_tile(
|
||||
tile: torch.Tensor,
|
||||
vae: torch.nn.Module,
|
||||
*,
|
||||
scaling_factor: float,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
"""Pixel path: ``[T, Hb, Wb, 3]`` floats in [0, 255] -> scaled ``[T', Hb', Wb', C']``.
|
||||
|
||||
The tile goes through bf16 first (also on CPU), like the reference.
|
||||
"""
|
||||
lq = tile.permute(3, 0, 1, 2).unsqueeze(0).to(device=device, dtype=torch.bfloat16)
|
||||
lq = cast_to_module_dtype(vae, vae.normalize_data(lq))
|
||||
latent = vae.encode(lq)[0]
|
||||
return latent.squeeze(0).permute(1, 2, 3, 0).float() * scaling_factor
|
||||
|
||||
|
||||
def build_chunk_latent(
|
||||
lq_tiles: Sequence[torch.Tensor],
|
||||
*,
|
||||
seed: int,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
"""Initial noisy latent ``[B, T', H', W', C]`` of one chunk of scaled LQ tiles."""
|
||||
lq = torch.cat(list(lq_tiles), dim=0)
|
||||
batch = len(lq_tiles)
|
||||
x = build_initial_latent(
|
||||
instruct_type=dit_spec.instruct_type,
|
||||
visual_cond=dit_spec.visual_cond,
|
||||
in_visual_dim=dit_spec.in_visual_dim,
|
||||
lq_latent=lq,
|
||||
batch_size=batch,
|
||||
duration=lq.shape[0] // batch,
|
||||
height=lq.shape[1],
|
||||
width=lq.shape[2],
|
||||
device=device,
|
||||
seed=seed,
|
||||
lq_noise_scale=spec.lq_noise_scale,
|
||||
lq_noise_type=spec.lq_noise_type,
|
||||
lq_channel_noise_scale=spec.lq_channel_noise_scale,
|
||||
# The DiT's own loaded dtype (reference: cast_to_module_dtype(dit, ...) before
|
||||
# building the initial latent), not a hard-coded fp32 regardless of precision.
|
||||
dtype=dit_spec.dtype,
|
||||
)
|
||||
return x.reshape(batch, -1, *x.shape[1:])
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def prepare_lu_tile_latents(
|
||||
lr_latent: torch.Tensor,
|
||||
plan: TilePlan,
|
||||
*,
|
||||
upscale_fn: LatentUpscaleFn,
|
||||
lu_dtype: torch.dtype | None,
|
||||
scaling_factor: float,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
device: torch.device,
|
||||
spatial_factor: int = VAE_SPATIAL_FACTOR,
|
||||
) -> list[torch.Tensor]:
|
||||
"""LU path: initial latents (CPU) of every chunk from the whole-video latent."""
|
||||
latent_grid = latent_tile_grid_from_pixel_grid(plan.pixel_grid, spatial_factor)
|
||||
tiles = extract_all_tiles(lr_latent, latent_grid)
|
||||
chunks: list[torch.Tensor] = []
|
||||
for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size):
|
||||
lq_tiles = [
|
||||
upscale_lr_latent_tile(
|
||||
tile,
|
||||
upscale_fn,
|
||||
lu_dtype=lu_dtype,
|
||||
scaling_factor=scaling_factor,
|
||||
device=device,
|
||||
)
|
||||
for tile in tiles[start:stop]
|
||||
]
|
||||
chunk = build_chunk_latent(
|
||||
lq_tiles,
|
||||
seed=spec.seed + start,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
chunks.append(chunk.cpu())
|
||||
return chunks
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def prepare_pixel_tile_latents(
|
||||
video: torch.Tensor,
|
||||
plan: TilePlan,
|
||||
*,
|
||||
vae: torch.nn.Module,
|
||||
scaling_factor: float,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
device: torch.device,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Pixel path: bilinear tile -> KVAE encode -> initial latents (CPU) per chunk."""
|
||||
tiles = extract_all_tiles(video, plan.pixel_grid)
|
||||
chunks: list[torch.Tensor] = []
|
||||
for start, stop in chunk_ranges(len(tiles), spec.tiles_batch_size):
|
||||
enlarged = upsample_tiles_to_base(tiles[start:stop], *plan.base_hw)
|
||||
lq_tiles = [
|
||||
encode_pixel_tile(tile, vae, scaling_factor=scaling_factor, device=device)
|
||||
for tile in enlarged
|
||||
]
|
||||
chunk = build_chunk_latent(
|
||||
lq_tiles,
|
||||
seed=spec.seed + start,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
chunks.append(chunk.cpu())
|
||||
return chunks
|
||||
|
||||
|
||||
def reset_scheduler_for_chunk(scheduler: Any) -> None:
|
||||
"""Reset step and begin indices so each tile chunk starts at the same timestep."""
|
||||
scheduler._step_index = None
|
||||
scheduler.set_begin_index(0)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def denoise_chunk(
|
||||
x: torch.Tensor,
|
||||
dit: Callable[..., torch.Tensor],
|
||||
scheduler: Any,
|
||||
*,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
step_context: StepContext | None = None,
|
||||
on_step: StepCallback | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Denoise one [B, T, H, W, C] chunk, resetting the configured scheduler."""
|
||||
is_piflow = spec.piflow is not None
|
||||
check_sampler_options(
|
||||
piflow=spec.piflow,
|
||||
cap_noise_timestep=spec.cap_noise_timestep,
|
||||
instruct_type=dit_spec.instruct_type,
|
||||
)
|
||||
if not is_piflow:
|
||||
# The Euler start timestep is only meaningful for the scheduler this chunk is about to
|
||||
# run: re-derive it instead of baking a stale one into ``set_timesteps`` up front.
|
||||
start = euler_start_timestep(
|
||||
cap_noise_timestep=spec.cap_noise_timestep,
|
||||
lq_noise_scale=spec.lq_noise_scale,
|
||||
instruct_type=dit_spec.instruct_type,
|
||||
)
|
||||
sigmas = torch.linspace(start, 0.0, spec.num_steps + 1)[:-1].tolist()
|
||||
scheduler.set_timesteps(sigmas=sigmas, device=x.device)
|
||||
reset_scheduler_for_chunk(scheduler)
|
||||
dit_fn = make_dit_fn(
|
||||
dit,
|
||||
latent_frames_hw=tuple(x.shape[1:4]),
|
||||
patch_size=dit_spec.patch_size,
|
||||
scale_factor=spec.scale_factor,
|
||||
use_motion_score=dit_spec.use_motion_score and not is_piflow,
|
||||
step_context=step_context,
|
||||
)
|
||||
with bf16_autocast(x.device):
|
||||
return denoise_with_scheduler(
|
||||
x,
|
||||
dit_fn,
|
||||
scheduler,
|
||||
channels=dit_spec.in_visual_dim,
|
||||
is_piflow=is_piflow,
|
||||
on_step=on_step,
|
||||
)
|
||||
|
||||
|
||||
def denoise_chunks(
|
||||
chunks: Sequence[torch.Tensor],
|
||||
dit: Callable[..., torch.Tensor],
|
||||
scheduler: Any,
|
||||
*,
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
device: torch.device,
|
||||
step_context: StepContext | None = None,
|
||||
on_step: StepCallback | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Denoise chunks sequentially on device and return CPU latents."""
|
||||
if spec.piflow is not None:
|
||||
scheduler.set_timesteps(spec.piflow.nfe, device=device)
|
||||
return [
|
||||
denoise_chunk(
|
||||
chunk.to(device),
|
||||
dit,
|
||||
scheduler,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
step_context=step_context,
|
||||
on_step=on_step,
|
||||
).cpu()
|
||||
for chunk in chunks
|
||||
]
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def _decode_tile(
|
||||
latents: torch.Tensor,
|
||||
vae: torch.nn.Module,
|
||||
*,
|
||||
scaling_factor: float,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
"""Decode one ``[1, T', H', W', C]`` tile to uint8 ``[3, T, H, W]`` on device."""
|
||||
with bf16_autocast(device):
|
||||
z = (latents.to(device) / scaling_factor).permute(0, 4, 1, 2, 3)
|
||||
latent = cast_to_module_dtype(vae, z)
|
||||
# preserve the reference's FP32 rescale before uint8 quantization
|
||||
decoded = vae.decode(latent).sample.float()
|
||||
pixels = (vae.denormalize_data(decoded) / 255.0).clamp(0.0, 1.0)
|
||||
return (pixels * 255.0).to(torch.uint8).squeeze(0)
|
||||
|
||||
|
||||
def decode_chunks(
|
||||
chunks: Sequence[torch.Tensor],
|
||||
vae: torch.nn.Module,
|
||||
*,
|
||||
scaling_factor: float,
|
||||
device: torch.device,
|
||||
group: GroupCoordinator | None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Decode in tile order, optionally sharing independent tiles within one DP replica."""
|
||||
latents = [tile for chunk in chunks for tile in chunk.split(1)]
|
||||
if group is None or group.world_size == 1 or len(latents) == 1:
|
||||
return [
|
||||
_decode_tile(tile, vae, scaling_factor=scaling_factor, device=device).cpu()
|
||||
for tile in latents
|
||||
]
|
||||
|
||||
tiles = []
|
||||
for start in range(0, len(latents), group.world_size):
|
||||
index = start + group.rank_in_group
|
||||
if index < len(latents):
|
||||
decoded = _decode_tile(
|
||||
latents[index], vae, scaling_factor=scaling_factor, device=device
|
||||
)
|
||||
else:
|
||||
_, frames, height, width, _ = latents[start].shape
|
||||
decoded = torch.zeros(
|
||||
(
|
||||
3,
|
||||
1 + (frames - 1) * vae.temporal_factor,
|
||||
height * vae.spatial_factor,
|
||||
width * vae.spatial_factor,
|
||||
),
|
||||
device=device,
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
# bound device staging to one tile per rank, not the whole decoded video
|
||||
gathered = group.all_gather(decoded.unsqueeze(0))[: len(latents) - start].cpu()
|
||||
del decoded
|
||||
tiles.extend(gathered.unbind(0))
|
||||
return tiles
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def stitch_tiles(tiles: Sequence[torch.Tensor], plan: TilePlan) -> torch.Tensor:
|
||||
"""Hann-blend [C, T, H, W] tiles in frame chunks to bound memory."""
|
||||
height, width = plan.frame_hw
|
||||
scale = plan.tiling_scale
|
||||
channels, frames = tiles[0].shape[:2]
|
||||
out = torch.empty(
|
||||
(channels, frames, height * scale, width * scale), dtype=torch.uint8
|
||||
)
|
||||
for start in range(0, frames, STITCH_FRAME_CHUNK):
|
||||
stop = min(start + STITCH_FRAME_CHUNK, frames)
|
||||
blended = stitch_tiles_hanning(
|
||||
[tile[:, start:stop].float() for tile in tiles],
|
||||
plan.pixel_grid,
|
||||
height,
|
||||
width,
|
||||
scale=scale,
|
||||
)
|
||||
out[:, start:stop] = blended.clamp(0, 255).to(torch.uint8)
|
||||
return out
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Vendored from sr_core (parity-tested against the k6_video reference); adapted from
|
||||
# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0).
|
||||
"""VAE-aligned tile grids and Hann blending for Kandinsky 6 video SR."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
from torch.nn import functional
|
||||
|
||||
VAE_SPATIAL_FACTOR: int = 16
|
||||
VAE_TEMPORAL_FACTOR: int = 4
|
||||
|
||||
# trained base resolutions: visual_size -> [(H, W), ...]
|
||||
RESOLUTIONS: dict[int, list[tuple[int, int]]] = {
|
||||
512: [(512, 512), (512, 768), (768, 512)],
|
||||
}
|
||||
|
||||
|
||||
class TileGrid(NamedTuple):
|
||||
"""Tile starts are authoritative; gaps can differ near the frame edges."""
|
||||
|
||||
tile_h: int
|
||||
tile_w: int
|
||||
tops: tuple[int, ...]
|
||||
lefts: tuple[int, ...]
|
||||
|
||||
@property
|
||||
def n_h(self) -> int:
|
||||
"""Number of tile rows."""
|
||||
return len(self.tops)
|
||||
|
||||
@property
|
||||
def n_w(self) -> int:
|
||||
"""Number of tile columns."""
|
||||
return len(self.lefts)
|
||||
|
||||
@property
|
||||
def total_tiles(self) -> int:
|
||||
"""Total number of tiles (``n_h * n_w``)."""
|
||||
return self.n_h * self.n_w
|
||||
|
||||
|
||||
def extract_all_tiles(
|
||||
video: torch.Tensor,
|
||||
grid: TileGrid,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Extract [T, C, tile_h, tile_w] views in row-major order."""
|
||||
tiles: list[torch.Tensor] = []
|
||||
for top in grid.tops:
|
||||
for left in grid.lefts:
|
||||
tile = video[:, :, top : top + grid.tile_h, left : left + grid.tile_w]
|
||||
tiles.append(tile)
|
||||
return tiles
|
||||
|
||||
|
||||
def _normalize_blend(acc: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
|
||||
"""Normalize Hann-weighted tiles without flooring positive weights.
|
||||
|
||||
Corner weights are tiny but valid; clamping them darkens tile boundaries.
|
||||
Pixels outside all tiles have zero weight and remain zero."""
|
||||
covered = weight > 0
|
||||
safe_weight = torch.where(covered, weight, torch.ones_like(weight))
|
||||
return torch.where(covered, acc / safe_weight, torch.zeros_like(acc))
|
||||
|
||||
|
||||
def hanning_window_2d(h: int, w: int, device: torch.device) -> torch.Tensor:
|
||||
"""Create a [h, w] Hann window with nonzero endpoints."""
|
||||
wy = torch.hann_window(h + 2, device=device)[1:-1]
|
||||
wx = torch.hann_window(w + 2, device=device)[1:-1]
|
||||
return wy[:, None] * wx[None, :]
|
||||
|
||||
|
||||
def stitch_tiles_hanning(
|
||||
tiles: list[torch.Tensor],
|
||||
grid: TileGrid,
|
||||
original_h: int,
|
||||
original_w: int,
|
||||
scale: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Blend row-major [C, T, H, W] tiles into a scale-enlarged frame."""
|
||||
first = tiles[0]
|
||||
c, t = first.shape[0], first.shape[1]
|
||||
hr_tile_h, hr_tile_w = first.shape[2], first.shape[3]
|
||||
device = first.device
|
||||
|
||||
window = hanning_window_2d(hr_tile_h, hr_tile_w, device)
|
||||
window = window.unsqueeze(0).unsqueeze(0) # [1, 1, hr_tile_h, hr_tile_w]
|
||||
|
||||
out_h = original_h * scale
|
||||
out_w = original_w * scale
|
||||
pred_acc = torch.zeros(c, t, out_h, out_w, device=device)
|
||||
weight_acc = torch.zeros(1, 1, out_h, out_w, device=device)
|
||||
|
||||
tile_idx = 0
|
||||
for top in grid.tops:
|
||||
for left in grid.lefts:
|
||||
y = top * scale
|
||||
x = left * scale
|
||||
tile = tiles[tile_idx]
|
||||
pred_acc[:, :, y : y + hr_tile_h, x : x + hr_tile_w] += tile * window
|
||||
weight_acc[:, :, y : y + hr_tile_h, x : x + hr_tile_w] += window
|
||||
tile_idx += 1
|
||||
|
||||
return _normalize_blend(pred_acc, weight_acc)
|
||||
|
||||
|
||||
def axis_positions_even(
|
||||
length: int, tile: int, min_overlap: float, snap: int
|
||||
) -> tuple[int, ...]:
|
||||
"""Cover the axis with the fewest evenly spaced, snap-aligned tiles.
|
||||
|
||||
Fall back to pixel alignment for unaligned sizes; latent-grid conversion
|
||||
validates VAE alignment separately."""
|
||||
if tile <= 0 or snap <= 0 or not 0 <= min_overlap < 1:
|
||||
msg = f"invalid axis spec: tile={tile}, snap={snap}, min_overlap={min_overlap}"
|
||||
raise ValueError(msg)
|
||||
if tile >= length:
|
||||
return (0,)
|
||||
unit = snap if length % snap == 0 and tile % snap == 0 else 1
|
||||
span_units = (length - tile) // unit
|
||||
max_stride_units = max(1, math.floor(tile * (1.0 - min_overlap) / unit))
|
||||
count = math.ceil(span_units / max_stride_units) + 1
|
||||
return tuple(round(i * span_units / (count - 1)) * unit for i in range(count))
|
||||
|
||||
|
||||
def compute_tile_grid_even(
|
||||
h: int,
|
||||
w: int,
|
||||
tile_hw: tuple[int, int],
|
||||
min_overlap: float,
|
||||
snap: int,
|
||||
) -> TileGrid:
|
||||
"""Build a grid with evenly distributed, aligned tile starts."""
|
||||
tile_h, tile_w = tile_hw
|
||||
return TileGrid(
|
||||
tile_h=tile_h,
|
||||
tile_w=tile_w,
|
||||
tops=axis_positions_even(h, tile_h, min_overlap, snap),
|
||||
lefts=axis_positions_even(w, tile_w, min_overlap, snap),
|
||||
)
|
||||
|
||||
|
||||
def latent_tile_grid_from_pixel_grid(
|
||||
pixel_grid: TileGrid, spatial_factor: int
|
||||
) -> TileGrid:
|
||||
"""Convert an aligned pixel grid to the corresponding latent grid."""
|
||||
values = (pixel_grid.tile_h, pixel_grid.tile_w, *pixel_grid.tops, *pixel_grid.lefts)
|
||||
if any(value % spatial_factor for value in values):
|
||||
raise ValueError(
|
||||
f"Pixel tile grid (tile {pixel_grid.tile_h}x{pixel_grid.tile_w}, "
|
||||
f"tops={pixel_grid.tops}, lefts={pixel_grid.lefts}) not aligned to VAE spatial factor "
|
||||
f"{spatial_factor}."
|
||||
)
|
||||
return TileGrid(
|
||||
pixel_grid.tile_h // spatial_factor,
|
||||
pixel_grid.tile_w // spatial_factor,
|
||||
tuple(top // spatial_factor for top in pixel_grid.tops),
|
||||
tuple(left // spatial_factor for left in pixel_grid.lefts),
|
||||
)
|
||||
|
||||
|
||||
def closest_base_resolution(
|
||||
h: int,
|
||||
w: int,
|
||||
visual_size: int,
|
||||
resolutions: Mapping[int, Sequence[tuple[int, int]]] = RESOLUTIONS,
|
||||
) -> tuple[int, int]:
|
||||
"""Choose the trained base resolution with the closest aspect ratio."""
|
||||
if visual_size not in resolutions:
|
||||
raise ValueError(
|
||||
f"Unsupported SR visual_size={visual_size}; known sizes: {sorted(resolutions)}"
|
||||
)
|
||||
ratio = w / h if h else 1.0
|
||||
return min(resolutions[visual_size], key=lambda hw: abs(hw[1] / hw[0] - ratio))
|
||||
|
||||
|
||||
def upsample_tiles_to_base(
|
||||
raw_tiles: list[torch.Tensor], base_h: int, base_w: int
|
||||
) -> list[torch.Tensor]:
|
||||
"""Bilinearly resize ``[T, C, h, w]`` tiles to ``base_h x base_w`` and return ``[T, H, W, C]``."""
|
||||
return [
|
||||
functional.interpolate(
|
||||
tile.float(), size=(base_h, base_w), mode="bilinear", align_corners=False
|
||||
).permute(0, 2, 3, 1)
|
||||
for tile in raw_tiles
|
||||
]
|
||||
|
||||
|
||||
def resolve_scale_request(scale: float) -> tuple[int, float]:
|
||||
"""Decompose total scale into (integer tiling scale, pixel pre-upscale)."""
|
||||
requested = float(scale)
|
||||
if requested == 2.25: # the one supported fractional total
|
||||
return 2, 1.125
|
||||
if requested in (2.0, 4.0):
|
||||
return int(requested), 1.0
|
||||
raise ValueError("SR supports total scales 2, 4, and 2.25")
|
||||
|
||||
|
||||
def pre_upscale_video(
|
||||
video: torch.Tensor, factor: float, spatial_multiple: int
|
||||
) -> torch.Tensor:
|
||||
"""Bilinearly enlarge [T, C, H, W] uint8 video, aligning target dimensions."""
|
||||
if video.ndim != 4:
|
||||
raise ValueError(f"video must have rank 4 [T,C,H,W], got {tuple(video.shape)}")
|
||||
if factor <= 0 or spatial_multiple <= 0:
|
||||
raise ValueError("factor and spatial_multiple must be positive")
|
||||
height, width = video.shape[-2:]
|
||||
target_h = max(
|
||||
spatial_multiple, round(height * factor / spatial_multiple) * spatial_multiple
|
||||
)
|
||||
target_w = max(
|
||||
spatial_multiple, round(width * factor / spatial_multiple) * spatial_multiple
|
||||
)
|
||||
resized = functional.interpolate(
|
||||
video.float(),
|
||||
size=(target_h, target_w),
|
||||
mode="bilinear",
|
||||
align_corners=False,
|
||||
)
|
||||
return resized.round_().clamp_(0, 255).to(torch.uint8)
|
||||
|
||||
|
||||
def pad_to_spatial_factor(
|
||||
video: torch.Tensor, spatial_factor: int
|
||||
) -> tuple[torch.Tensor, tuple[int, int]]:
|
||||
"""Pad bottom/right to the VAE spatial factor; return the original size.
|
||||
|
||||
Crop the scaled padding after SR to preserve the source framing."""
|
||||
if video.ndim != 4:
|
||||
raise ValueError(f"video must have rank 4 [T,C,H,W], got {tuple(video.shape)}")
|
||||
height, width = video.shape[-2:]
|
||||
pad_h = (-height) % spatial_factor
|
||||
pad_w = (-width) % spatial_factor
|
||||
if pad_h == 0 and pad_w == 0:
|
||||
return video, (height, width)
|
||||
padded = functional.pad(video, (0, pad_w, 0, pad_h), mode="replicate")
|
||||
return padded, (height, width)
|
||||
|
||||
|
||||
def crop_to_hw(video: torch.Tensor, hw: tuple[int, int]) -> torch.Tensor:
|
||||
"""Crop [..., H, W] to hw from the top-left, undoing bottom/right padding."""
|
||||
height, width = hw
|
||||
return video[..., :height, :width]
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Streaming video / audio input and output helpers of Kandinsky 6 video SR.
|
||||
|
||||
Decoding follows the reference CLI (``read_video_tchw_uint8`` ->
|
||||
``resample_to_target_fps`` -> ``clip_to_aligned_frames``) frame for frame, but streams:
|
||||
only the frames that survive fps resampling and the 121-frame cap are converted to RGB
|
||||
and kept, and decoding stops as soon as the outcome is known. PyAV is imported lazily
|
||||
(it is part of the ``diffusion`` extra).
|
||||
"""
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
|
||||
MAX_NUM_FRAMES,
|
||||
RESAMPLE_FPS_TOLERANCE,
|
||||
TARGET_FPS,
|
||||
clip_to_aligned_frames,
|
||||
)
|
||||
|
||||
SOURCE_AUDIO_SAMPLE_RATE = 44100
|
||||
OUTPUT_FRAME_CHUNK = 8
|
||||
|
||||
|
||||
class DecodedClip(msgspec.Struct, frozen=True):
|
||||
"""Frames ready for SR plus the rate they play at."""
|
||||
|
||||
frames: torch.Tensor # [T, 3, H, W] uint8, T = 1 + 8k <= 121
|
||||
fps: int # effective fps (24 when downsampled, else round(source fps))
|
||||
source_fps: float
|
||||
|
||||
|
||||
class FrameSelector:
|
||||
"""Streaming form of ``resample_to_target_fps`` + the first-121-frames cap.
|
||||
|
||||
Feed source frames in order: :meth:`keep` says whether frame ``index`` is one of the
|
||||
selected ones (call :meth:`mark_kept` for each), :meth:`enough` says when no further
|
||||
frame can change the result, and :meth:`limit` trims the selection once the source
|
||||
length is known. For a source faster than the target the selected indices are
|
||||
``round(i * step)``, ``step = source_fps / target_fps``, of which there are
|
||||
``int(total / step)`` (those below ``total``), exactly as in the reference;
|
||||
otherwise every frame is selected.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
source_fps: float,
|
||||
*,
|
||||
target_fps: int = TARGET_FPS,
|
||||
max_frames: int = MAX_NUM_FRAMES,
|
||||
) -> None:
|
||||
self.max_frames = max_frames
|
||||
self.downsample = (
|
||||
abs(source_fps - target_fps) >= RESAMPLE_FPS_TOLERANCE
|
||||
and source_fps > target_fps
|
||||
)
|
||||
self.step = source_fps / target_fps
|
||||
self.effective_fps = target_fps if self.downsample else round(source_fps)
|
||||
self._kept = 0
|
||||
self._next_index = 0
|
||||
|
||||
def keep(self, index: int) -> bool:
|
||||
if self._kept >= self.max_frames:
|
||||
return False
|
||||
return not self.downsample or index == self._next_index
|
||||
|
||||
def mark_kept(self) -> None:
|
||||
self._kept += 1
|
||||
if self.downsample:
|
||||
self._next_index = round(self._kept * self.step)
|
||||
|
||||
def enough(self, frames_seen: int) -> bool:
|
||||
"""True once ``max_frames`` are kept and the source is known to be long enough."""
|
||||
if self._kept < self.max_frames:
|
||||
return False
|
||||
return not self.downsample or int(frames_seen / self.step) >= self.max_frames
|
||||
|
||||
def limit(self, total_frames: int, kept: int) -> int:
|
||||
"""Number of kept frames the reference would return for a source of this length."""
|
||||
if self.downsample:
|
||||
kept = min(kept, int(total_frames / self.step))
|
||||
return min(kept, self.max_frames)
|
||||
|
||||
|
||||
def decode_clip(
|
||||
path: str,
|
||||
*,
|
||||
target_fps: int = TARGET_FPS,
|
||||
max_frames: int = MAX_NUM_FRAMES,
|
||||
) -> DecodedClip:
|
||||
"""Decode the first video stream into the aligned uint8 clip the SR model expects."""
|
||||
import av
|
||||
|
||||
frames: list[torch.Tensor] = []
|
||||
seen = 0
|
||||
with av.open(str(path), mode="r") as container:
|
||||
stream = container.streams.video[0]
|
||||
source_fps = float(stream.average_rate or stream.base_rate or 0.0)
|
||||
if source_fps <= 0:
|
||||
raise ValueError(f"Video {path} reports no usable fps (got {source_fps}).")
|
||||
selector = FrameSelector(
|
||||
source_fps, target_fps=target_fps, max_frames=max_frames
|
||||
)
|
||||
if selector.effective_fps < 1:
|
||||
raise ValueError(f"Video {path} has an unusable fps of {source_fps}.")
|
||||
stream.thread_type = "AUTO"
|
||||
for frame in container.decode(stream):
|
||||
if selector.keep(seen):
|
||||
array = frame.to_ndarray(format="rgb24")
|
||||
frames.append(torch.from_numpy(array).permute(2, 0, 1))
|
||||
selector.mark_kept()
|
||||
seen += 1
|
||||
if selector.enough(seen):
|
||||
break
|
||||
kept = selector.limit(seen, len(frames))
|
||||
if kept == 0:
|
||||
raise ValueError(f"Video {path} has no readable frames.")
|
||||
video = torch.stack(frames[:kept]).contiguous()
|
||||
return DecodedClip(
|
||||
frames=clip_to_aligned_frames(video, max_frames),
|
||||
fps=selector.effective_fps,
|
||||
source_fps=source_fps,
|
||||
)
|
||||
|
||||
|
||||
def extract_source_audio(
|
||||
path: str,
|
||||
*,
|
||||
sample_rate: int = SOURCE_AUDIO_SAMPLE_RATE,
|
||||
max_seconds: float | None = None,
|
||||
) -> np.ndarray | None:
|
||||
"""Decode the first audio stream as mono float32 at ``sample_rate`` (None if absent).
|
||||
|
||||
``max_seconds`` stops decoding once that much audio has been produced; the result is
|
||||
trimmed to exactly that length.
|
||||
"""
|
||||
import av
|
||||
|
||||
if sample_rate <= 0:
|
||||
raise ValueError("sample_rate must be positive")
|
||||
limit = None if max_seconds is None else round(max_seconds * sample_rate)
|
||||
chunks: list[np.ndarray] = []
|
||||
produced = 0
|
||||
with av.open(str(path), mode="r") as container:
|
||||
if not container.streams.audio:
|
||||
return None
|
||||
resampler = av.AudioResampler(format="fltp", layout="mono", rate=sample_rate)
|
||||
for frame in container.decode(audio=0):
|
||||
for resampled in resampler.resample(frame):
|
||||
chunk = resampled.to_ndarray().reshape(-1)
|
||||
chunks.append(chunk)
|
||||
produced += chunk.shape[0]
|
||||
if limit is not None and produced >= limit:
|
||||
break
|
||||
else:
|
||||
for resampled in resampler.resample(None):
|
||||
chunks.append(resampled.to_ndarray().reshape(-1))
|
||||
if not chunks:
|
||||
return None
|
||||
audio = np.concatenate(chunks).astype(np.float32, copy=False)
|
||||
return audio if limit is None else audio[:limit]
|
||||
|
||||
|
||||
def synthetic_clip(*, height: int, width: int, frames: int) -> torch.Tensor:
|
||||
"""Deterministic ``[T, 3, H, W]`` uint8 clip (warmup input; no file needed)."""
|
||||
shape = (frames, 1, height, width)
|
||||
ys = torch.linspace(0, 255, height).view(1, 1, height, 1).expand(shape)
|
||||
xs = torch.linspace(0, 255, width).view(1, 1, 1, width).expand(shape)
|
||||
ts = torch.linspace(0, 255, frames).view(frames, 1, 1, 1).expand(shape)
|
||||
return (torch.cat([ys, xs, ts], dim=1) / 2 + 64).clamp(0, 255).to(torch.uint8)
|
||||
|
||||
|
||||
def to_output_video(video: torch.Tensor) -> torch.Tensor:
|
||||
"""uint8 ``[3, T, H, W]`` -> fp16 ``[1, 3, T, H, W]`` in [0, 1] that saves losslessly.
|
||||
|
||||
The framework writers truncate ``(x * 255)`` to uint8. fp16 halves the memory of a
|
||||
large clip, but a plain ``k / 255`` stored in fp16 truncates to ``k - 1`` for over half
|
||||
of the values when a consumer upcasts before multiplying. ``(k + 0.5) / 255`` stays
|
||||
inside the bucket of ``k`` in every evaluation order. Converted in frame chunks to
|
||||
bound the float32 temporaries.
|
||||
"""
|
||||
channels, frames, height, width = video.shape
|
||||
out = torch.empty((1, channels, frames, height, width), dtype=torch.float16)
|
||||
for start in range(0, frames, OUTPUT_FRAME_CHUNK):
|
||||
chunk = video[:, start : start + OUTPUT_FRAME_CHUNK]
|
||||
out[0, :, start : start + OUTPUT_FRAME_CHUNK] = (
|
||||
(chunk.float() + 0.5) / 255.0
|
||||
).to(torch.float16)
|
||||
return out
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Vendored from sr_core (parity-tested against the k6_video reference); adapted from
|
||||
# the Kandinsky 6 SR inference reference (k6_video, Apache-2.0).
|
||||
"""Frame selection and bounded-memory output resize for Kandinsky 6 SR."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
import torch
|
||||
from torch.nn import functional as f # noqa: N812
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr_resolution import (
|
||||
ASPECT_MISMATCH_TOLERANCE,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --- Model / pipeline contract (from the reference ``constants.py``; NOT configurable) ---
|
||||
# The SR model was trained on 5 s clips at 24 fps -> 121 pixel frames
|
||||
# (= 1 + 8*15); feeding more frames or another rate is out of contract.
|
||||
TARGET_FPS: int = 24
|
||||
MAX_NUM_FRAMES: int = 121
|
||||
# Treat ``|src_fps - TARGET_FPS| < tolerance`` as already-at-target (no
|
||||
# resample): covers 23.98 (NTSC) and 25.00 sources.
|
||||
RESAMPLE_FPS_TOLERANCE: float = 1.5
|
||||
# Frames per interpolate call when downscaling the output (a 2K clip is
|
||||
# ~2.4 GB as uint8, so the resize walks the clip in chunks).
|
||||
RESIZE_FRAME_CHUNK: int = 8
|
||||
|
||||
|
||||
def select_frame_indices(
|
||||
total_frames: int, src_fps: float, target_fps: float
|
||||
) -> list[int]:
|
||||
"""Select rounded frame indices to downsample src_fps to target_fps."""
|
||||
step = src_fps / target_fps
|
||||
indices = [round(i * step) for i in range(int(total_frames / step))]
|
||||
return [i for i in indices if i < total_frames]
|
||||
|
||||
|
||||
def resample_to_target_fps(
|
||||
video: torch.Tensor,
|
||||
src_fps: float,
|
||||
target_fps: int = TARGET_FPS,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""Downsample high-fps inputs; retain near-target and lower-fps inputs.
|
||||
|
||||
Return [T, C, H, W] video and its effective playback rate."""
|
||||
if abs(src_fps - target_fps) < RESAMPLE_FPS_TOLERANCE:
|
||||
return video, round(src_fps)
|
||||
if src_fps > target_fps:
|
||||
indices = select_frame_indices(video.shape[0], src_fps, target_fps)
|
||||
logger.warning(
|
||||
"Source fps %.2f > target %sfps: downsampling %s frames -> %s (fixed-stride); "
|
||||
"source temporal detail beyond %sfps is discarded.",
|
||||
src_fps,
|
||||
target_fps,
|
||||
video.shape[0],
|
||||
len(indices),
|
||||
target_fps,
|
||||
)
|
||||
return video[indices], target_fps
|
||||
logger.warning(
|
||||
"Source fps %.2f < target %sfps: keeping native frames (no minterpolate upsample); "
|
||||
"output is mildly out of distribution.",
|
||||
src_fps,
|
||||
target_fps,
|
||||
)
|
||||
return video, round(src_fps)
|
||||
|
||||
|
||||
def clip_to_aligned_frames(
|
||||
video: torch.Tensor, max_num_frames: int = MAX_NUM_FRAMES
|
||||
) -> torch.Tensor:
|
||||
"""Cap the frame count, then floor-align to 1+8k; reject empty video."""
|
||||
if max_num_frames > 0:
|
||||
video = video[:max_num_frames]
|
||||
aligned = 1 + 8 * ((video.shape[0] - 1) // 8) if video.shape[0] > 0 else 0
|
||||
if aligned == 0:
|
||||
msg = "Video has no readable frames."
|
||||
raise ValueError(msg)
|
||||
return video[:aligned]
|
||||
|
||||
|
||||
def resize_to_target(video: torch.Tensor, target_hw: tuple[int, int]) -> torch.Tensor:
|
||||
"""Downscale [C, T, H, W] uint8 video in bounded-memory chunks, without cropping."""
|
||||
_c, _t, height, width = video.shape
|
||||
target_h, target_w = target_hw
|
||||
if (height, width) == (target_h, target_w):
|
||||
return video
|
||||
if target_h > height or target_w > width:
|
||||
msg = (
|
||||
f"target {target_w}x{target_h} exceeds the SR result {width}x{height} — that would "
|
||||
f"upscale. Pick a lower tier or a higher --resolution-scale for this source."
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
source_ratio, target_ratio = width / height, target_w / target_h
|
||||
if abs(source_ratio - target_ratio) / target_ratio > ASPECT_MISMATCH_TOLERANCE:
|
||||
logger.warning(
|
||||
"Aspect mismatch: SR %sx%s is %.3f, target %sx%s is %.3f — resizing anisotropically "
|
||||
"(no crop), the picture will be squeezed by %.1f%%.",
|
||||
width,
|
||||
height,
|
||||
source_ratio,
|
||||
target_w,
|
||||
target_h,
|
||||
target_ratio,
|
||||
abs(source_ratio / target_ratio - 1) * 100,
|
||||
)
|
||||
|
||||
channels, frames = video.shape[0], video.shape[1]
|
||||
out = torch.empty((channels, frames, target_h, target_w), dtype=torch.uint8)
|
||||
for start in range(0, frames, RESIZE_FRAME_CHUNK):
|
||||
chunk = video[:, start : start + RESIZE_FRAME_CHUNK].permute(1, 0, 2, 3).float()
|
||||
resized = f.interpolate(
|
||||
chunk,
|
||||
size=(target_h, target_w),
|
||||
mode="bilinear",
|
||||
antialias=True,
|
||||
align_corners=False,
|
||||
)
|
||||
out[:, start : start + RESIZE_FRAME_CHUNK] = (
|
||||
resized.clamp(0, 255).round().to(torch.uint8).permute(1, 0, 2, 3)
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Final output: %sx%s -> %sx%s (downscale x%.2f / x%.2f)",
|
||||
width,
|
||||
height,
|
||||
target_w,
|
||||
target_h,
|
||||
width / target_w,
|
||||
height / target_h,
|
||||
)
|
||||
return out
|
||||
@@ -12,7 +12,13 @@ import inspect
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.encoders import BaseEncoderOutput
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import TextConditioningOutput
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import (
|
||||
PipelineConfig,
|
||||
TextConditioningOutput,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.cache.conditioning import (
|
||||
cached_encoder_call,
|
||||
prefer_conditioning_cache,
|
||||
@@ -106,6 +112,26 @@ def _data_parallel_text_encode(forward_fn, forward_kwargs: dict, group):
|
||||
)
|
||||
|
||||
|
||||
def _text_encoder_max_length_is_fixed(
|
||||
pipeline_config: PipelineConfig, encoder_index: int
|
||||
) -> bool:
|
||||
"""True when ``encoder_index``'s tokenizer ``max_length`` is
|
||||
architecturally fixed and must never be overridden by a request's
|
||||
``max_sequence_length``.
|
||||
|
||||
Flux v1's encoder 0 and Kandinsky6 TI2VA's encoder 1 are both CLIP with
|
||||
a fixed 77-token pooled-embedding context; overriding it corrupts CLIP
|
||||
tokenization. Every other encoder (Flux v1's encoder 1 T5, Kandinsky6's
|
||||
encoder 0 Reason1/Qwen, and every other pipeline's encoders) is meant to
|
||||
take the request's override, so this returns False for them.
|
||||
"""
|
||||
if pipeline_config.is_flux_v1():
|
||||
return encoder_index == 0
|
||||
if isinstance(pipeline_config, Kandinsky6TI2VAPipelineConfig):
|
||||
return encoder_index == 1
|
||||
return False
|
||||
|
||||
|
||||
def stack_tensors(name: str, tensors: list[torch.Tensor]) -> torch.Tensor:
|
||||
base_shape = list(tensors[0].shape)
|
||||
for tensor in tensors[1:]:
|
||||
@@ -593,10 +619,12 @@ class TextEncodingStage(ConditionEncodingStage):
|
||||
encoder_config.tokenizer_kwargs,
|
||||
**text_encoder_extra_arg,
|
||||
)
|
||||
# Pass max_length to tokenizer if specified in the request. Flux v1 encoder 0
|
||||
# is CLIP with a fixed 77-token context; overriding breaks tokenization.
|
||||
is_flux_v1 = server_args.pipeline_config.is_flux_v1()
|
||||
if max_length is not None and not (is_flux_v1 and i == 0):
|
||||
# Pass max_length to tokenizer if specified in the request, except
|
||||
# for a text encoder whose context is architecturally fixed (see
|
||||
# _text_encoder_max_length_is_fixed).
|
||||
if max_length is not None and not _text_encoder_max_length_is_fixed(
|
||||
server_args.pipeline_config, i
|
||||
):
|
||||
tok_kwargs["max_length"] = max_length
|
||||
|
||||
text_inputs: dict = server_args.pipeline_config.tokenize_prompt(
|
||||
|
||||
@@ -386,7 +386,8 @@ def _verify_diffusers_model_complete(path: str) -> bool:
|
||||
component_keys = [
|
||||
key
|
||||
for key, value in model_index.items()
|
||||
if _is_diffusers_component_entry(value)
|
||||
if not key.startswith("_")
|
||||
and _is_diffusers_component_entry(value)
|
||||
and any(item is not None for item in value)
|
||||
]
|
||||
if component_keys:
|
||||
@@ -561,6 +562,35 @@ def get_diffusers_component_config(
|
||||
lambda acc, path: acc | load_dict(path), config_file_paths, {}
|
||||
)
|
||||
|
||||
if "_class_name" not in combined_config:
|
||||
# Some checkpoints (e.g. real Kandinsky6 exports, which declare
|
||||
# "uses_patched_diffusers": true) omit "_class_name" from a
|
||||
# component's own config.json, relying on the parent
|
||||
# model_index.json's [library, class_name] entry for that component
|
||||
# subfolder instead. Fall back to that so component loaders (which
|
||||
# unconditionally pop "_class_name") still resolve the right class.
|
||||
model_index_path = os.path.join(
|
||||
os.path.dirname(os.path.normpath(component_path)), "model_index.json"
|
||||
)
|
||||
component_key = os.path.basename(os.path.normpath(component_path))
|
||||
if os.path.exists(model_index_path):
|
||||
try:
|
||||
with open(model_index_path) as f:
|
||||
model_index = json.load(f)
|
||||
entry = model_index.get(component_key)
|
||||
class_name = (
|
||||
entry[-1] if isinstance(entry, (list, tuple)) and entry else entry
|
||||
)
|
||||
if class_name:
|
||||
combined_config["_class_name"] = str(class_name)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"Failed to backfill _class_name for %s from %s: %s",
|
||||
component_path,
|
||||
model_index_path,
|
||||
exc,
|
||||
)
|
||||
|
||||
quant_config = combined_config.get("quantization_config")
|
||||
if quant_config is not None:
|
||||
combined_config["quantization_config"] = normalize_flat_modelopt_quant_config(
|
||||
@@ -736,7 +766,8 @@ def verify_model_config_and_directory(model_path: str) -> dict[str, Any]:
|
||||
component_keys = [
|
||||
key
|
||||
for key, value in config.items()
|
||||
if isinstance(value, (list, tuple))
|
||||
if not key.startswith("_")
|
||||
and isinstance(value, (list, tuple))
|
||||
and len(value) == 2
|
||||
and all(isinstance(item, str) for item in value)
|
||||
]
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Opt-in checkpoint test; SGLANG_KANDINSKY6_TEST_SERVER_ARGS selects topology."""
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import av
|
||||
import numpy as np
|
||||
import pytest
|
||||
import requests
|
||||
import torch
|
||||
from huggingface_hub import hf_hub_download
|
||||
from PIL import Image, ImageDraw
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from sglang.multimodal_gen.test.server.test_server_utils import ServerManager
|
||||
from sglang.multimodal_gen.test.test_utils import (
|
||||
extract_audio_pcm_from_video_bytes,
|
||||
get_dynamic_server_port,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not os.environ.get("SGLANG_KANDINSKY6_TEST_MODEL"),
|
||||
reason="set SGLANG_KANDINSKY6_TEST_MODEL for full-checkpoint HTTP tests",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
manager = ServerManager(
|
||||
model=os.environ["SGLANG_KANDINSKY6_TEST_MODEL"],
|
||||
port=get_dynamic_server_port(),
|
||||
extra_args=(
|
||||
"--performance-mode speed --attention-backend fa --warmup-mode off "
|
||||
+ os.environ.get("SGLANG_KANDINSKY6_TEST_SERVER_ARGS", "--num-gpus 1")
|
||||
),
|
||||
)
|
||||
context = manager.start()
|
||||
try:
|
||||
yield context
|
||||
finally:
|
||||
context.cleanup()
|
||||
|
||||
|
||||
def _generate_video(server, *, image=None, size="384x256", prompt=None, **overrides):
|
||||
url = f"http://127.0.0.1:{server.port}/v1/videos"
|
||||
payload = {
|
||||
"prompt": prompt or "A red ceramic teapot on a wooden table. Quiet jazz music.",
|
||||
"size": size,
|
||||
"num_frames": 17,
|
||||
"fps": 24,
|
||||
"num_inference_steps": 2,
|
||||
"seed": 42,
|
||||
} | overrides
|
||||
with requests.Session() as client:
|
||||
if image is None:
|
||||
response = client.post(url, json=payload, timeout=120)
|
||||
else:
|
||||
response = client.post(
|
||||
url,
|
||||
data=payload,
|
||||
files={"input_reference": ("reference.png", image, "image/png")},
|
||||
timeout=120,
|
||||
)
|
||||
assert response.ok, response.text
|
||||
job_url = f"{url}/{response.json()['id']}"
|
||||
try:
|
||||
deadline = time.monotonic() + 300
|
||||
while True:
|
||||
response = client.get(job_url, timeout=30)
|
||||
assert response.ok, response.text
|
||||
job = response.json()
|
||||
assert job["status"] not in {"failed", "cancelled", "deleted"}, job
|
||||
if job["status"] == "completed":
|
||||
break
|
||||
assert time.monotonic() < deadline, (job, server.log_tail())
|
||||
time.sleep(0.5)
|
||||
response = client.get(f"{job_url}/content", timeout=60)
|
||||
assert response.ok, response.text
|
||||
content = response.content
|
||||
finally:
|
||||
client.delete(job_url, timeout=30)
|
||||
with av.open(io.BytesIO(content)) as video:
|
||||
frames = np.stack(
|
||||
[frame.to_ndarray(format="rgb24") for frame in video.decode(video=0)]
|
||||
)
|
||||
audio = extract_audio_pcm_from_video_bytes(content)
|
||||
width, height = map(int, size.split("x"))
|
||||
assert frames.shape == (17, height, width, 3)
|
||||
assert frames.std() > 1, "degenerate video"
|
||||
assert audio.size > 0 and np.isfinite(audio).all()
|
||||
assert np.sqrt(np.mean(np.square(audio, dtype=np.float64))) > 1e-5, "silent audio"
|
||||
return frames, audio
|
||||
|
||||
|
||||
def test_repeated_requests_and_conditioning_isolation(server):
|
||||
reference = Image.new("RGB", (384, 256), "white")
|
||||
ImageDraw.Draw(reference).rectangle((96, 64, 288, 192), fill="red")
|
||||
buffer = io.BytesIO()
|
||||
reference.save(buffer, format="PNG")
|
||||
|
||||
first = _generate_video(server)
|
||||
repeated = _generate_video(server)
|
||||
for actual, expected in zip(repeated, first, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
|
||||
conditioned = _generate_video(server, image=buffer.getvalue())
|
||||
conditioned_repeat = _generate_video(server, image=buffer.getvalue())
|
||||
for actual, expected in zip(conditioned_repeat, conditioned, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
assert not np.array_equal(conditioned[0], first[0]), (
|
||||
"image conditioning was ignored"
|
||||
)
|
||||
|
||||
_generate_video(server, size="256x384", prompt="A blue glass vase. Rain sounds.")
|
||||
restored = _generate_video(server)
|
||||
for actual, expected in zip(restored, first, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
|
||||
|
||||
def test_request_cache_dit_refresh_and_opt_out(server):
|
||||
baseline = _generate_video(server, num_inference_steps=6, enable_cache_dit=False)
|
||||
no_skip = _generate_video(
|
||||
server,
|
||||
num_inference_steps=6,
|
||||
enable_cache_dit=True,
|
||||
cache_dit_params={"max_warmup_steps": 6},
|
||||
)
|
||||
for actual, expected in zip(no_skip, baseline, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
|
||||
cached_params = dict(
|
||||
num_inference_steps=6,
|
||||
enable_cache_dit=True,
|
||||
cache_dit_params={
|
||||
"max_warmup_steps": 1,
|
||||
"scm_compute_bins": [1, 1, 1],
|
||||
"scm_cache_bins": [1, 1, 1],
|
||||
"scm_policy": "static",
|
||||
},
|
||||
)
|
||||
first = _generate_video(server, **cached_params)
|
||||
assert any(not np.array_equal(a, b) for a, b in zip(first, baseline, strict=True))
|
||||
_generate_video(
|
||||
server,
|
||||
size="256x384",
|
||||
prompt="A blue glass vase. Rain sounds.",
|
||||
**cached_params,
|
||||
)
|
||||
repeated = _generate_video(server, **cached_params)
|
||||
for actual, expected in zip(repeated, first, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
restored = _generate_video(server, num_inference_steps=6, enable_cache_dit=False)
|
||||
for actual, expected in zip(restored, baseline, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def lora_adapter(tmp_path_factory):
|
||||
model = os.environ["SGLANG_KANDINSKY6_TEST_MODEL"]
|
||||
config_path = Path(model) / "transformer" / "config.json"
|
||||
if not config_path.is_file():
|
||||
config_path = Path(hf_hub_download(model, "transformer/config.json"))
|
||||
config = json.loads(config_path.read_text())
|
||||
layers = {
|
||||
"visual_transformer_blocks.0.video_dec_block.feed_forward.net.0.proj": (
|
||||
config["ff_dim"],
|
||||
config["model_dim"],
|
||||
),
|
||||
"visual_transformer_blocks.0.audio_dec_block.feed_forward.net.2": (
|
||||
config["model_dim_a"],
|
||||
config["ff_dim_a"],
|
||||
),
|
||||
"video_time_embeddings.timestep_embedder.linear_1": (
|
||||
config["time_dim"],
|
||||
config["model_dim"],
|
||||
),
|
||||
}
|
||||
generator = torch.Generator().manual_seed(827)
|
||||
weights = {}
|
||||
for name, (out_dim, in_dim) in layers.items():
|
||||
for suffix, shape in (("A", (2, in_dim)), ("B", (out_dim, 2))):
|
||||
weights[f"transformer.{name}.lora_{suffix}.weight"] = (
|
||||
torch.randn(shape, generator=generator) * 0.05
|
||||
)
|
||||
path = tmp_path_factory.mktemp("kandinsky6_lora") / "adapter.safetensors"
|
||||
save_file(weights, path)
|
||||
return str(path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("merge_mode", ["dynamic", "merge"])
|
||||
def test_lora_scaling_repeatability_and_restore(server, lora_adapter, merge_mode):
|
||||
base_url = f"http://127.0.0.1:{server.port}/v1"
|
||||
baseline = _generate_video(server, enable_cache_dit=False)
|
||||
|
||||
def set_strength(strength):
|
||||
response = requests.post(
|
||||
f"{base_url}/set_lora",
|
||||
json={
|
||||
"lora_nickname": "kandinsky6_synthetic",
|
||||
"lora_path": lora_adapter,
|
||||
"target": "transformer",
|
||||
"strength": strength,
|
||||
"merge_mode": merge_mode,
|
||||
},
|
||||
timeout=180,
|
||||
)
|
||||
assert response.ok, response.text
|
||||
|
||||
try:
|
||||
set_strength(1.0)
|
||||
adapted = _generate_video(server, enable_cache_dit=False)
|
||||
for actual, expected in zip(adapted, baseline, strict=True):
|
||||
assert not np.array_equal(actual, expected), (
|
||||
"LoRA did not change the output"
|
||||
)
|
||||
repeated = _generate_video(server, enable_cache_dit=False)
|
||||
for actual, expected in zip(repeated, adapted, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
set_strength(0.0)
|
||||
zero = _generate_video(server, enable_cache_dit=False)
|
||||
for actual, expected in zip(zero, baseline, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
set_strength(0.5)
|
||||
scaled = _generate_video(server, enable_cache_dit=False)
|
||||
assert any(
|
||||
not np.array_equal(a, b) for a, b in zip(scaled, adapted, strict=True)
|
||||
)
|
||||
set_strength(1.0)
|
||||
restored_adapter = _generate_video(server, enable_cache_dit=False)
|
||||
for actual, expected in zip(restored_adapter, adapted, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
finally:
|
||||
response = requests.post(
|
||||
f"{base_url}/unmerge_lora_weights",
|
||||
json={"target": "transformer"},
|
||||
timeout=180,
|
||||
)
|
||||
assert response.ok, response.text
|
||||
restored_base = _generate_video(server, enable_cache_dit=False)
|
||||
for actual, expected in zip(restored_base, baseline, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__]))
|
||||
@@ -0,0 +1,172 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Opt-in SR checkpoint test covering real JSON and multipart video requests."""
|
||||
|
||||
import io
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import av
|
||||
import numpy as np
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from sglang.multimodal_gen.test.server.test_server_utils import ServerManager
|
||||
from sglang.multimodal_gen.test.test_utils import (
|
||||
extract_audio_pcm_from_video_bytes,
|
||||
get_dynamic_server_port,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not os.environ.get("SGLANG_KANDINSKY6_SR_TEST_MODEL"),
|
||||
reason="set SGLANG_KANDINSKY6_SR_TEST_MODEL for full-checkpoint HTTP tests",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def source_videos(tmp_path_factory):
|
||||
directory = tmp_path_factory.mktemp("kandinsky6_sr_source")
|
||||
source = directory / "source.mp4"
|
||||
silent = directory / "silent.mp4"
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"testsrc2=size=384x256:rate=24",
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
"sine=frequency=440:sample_rate=44100",
|
||||
"-frames:v",
|
||||
"33",
|
||||
"-t",
|
||||
str(33 / 24),
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-crf",
|
||||
"18",
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-c:a",
|
||||
"aac",
|
||||
str(source),
|
||||
],
|
||||
check=True,
|
||||
timeout=60,
|
||||
)
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-i",
|
||||
str(source),
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-an",
|
||||
str(silent),
|
||||
],
|
||||
check=True,
|
||||
timeout=60,
|
||||
)
|
||||
return source, silent
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server(tmp_path_factory):
|
||||
directory = tmp_path_factory.mktemp("kandinsky6_sr_server")
|
||||
manager = ServerManager(
|
||||
model=os.environ["SGLANG_KANDINSKY6_SR_TEST_MODEL"],
|
||||
port=get_dynamic_server_port(),
|
||||
extra_args=(
|
||||
"--performance-mode speed --attention-backend fa --warmup-mode off "
|
||||
f"--input-save-path {shlex.quote(str(directory / 'uploads'))} "
|
||||
f"--output-path {shlex.quote(str(directory / 'outputs'))} "
|
||||
+ os.environ.get("SGLANG_KANDINSKY6_SR_TEST_SERVER_ARGS", "--num-gpus 1")
|
||||
),
|
||||
)
|
||||
context = manager.start()
|
||||
try:
|
||||
yield context
|
||||
finally:
|
||||
context.cleanup()
|
||||
|
||||
|
||||
def _generate(
|
||||
server, source, *, scale=2, tiles_batch_size=1, upload=False, has_audio=True
|
||||
):
|
||||
url = f"http://127.0.0.1:{server.port}/v1/videos"
|
||||
payload = {
|
||||
"sr_resolution_scale": scale,
|
||||
"sr_tiles_batch_size": tiles_batch_size,
|
||||
"num_inference_steps": 2,
|
||||
"seed": 42,
|
||||
}
|
||||
with requests.Session() as client:
|
||||
if upload:
|
||||
with source.open("rb") as stream:
|
||||
response = client.post(
|
||||
url,
|
||||
data=payload,
|
||||
files={"video_reference": (source.name, stream, "video/mp4")},
|
||||
timeout=120,
|
||||
)
|
||||
else:
|
||||
response = client.post(
|
||||
url, json=payload | {"video_path": str(source)}, timeout=120
|
||||
)
|
||||
assert response.ok, response.text
|
||||
job_url = f"{url}/{response.json()['id']}"
|
||||
try:
|
||||
deadline = time.monotonic() + 300
|
||||
while True:
|
||||
response = client.get(job_url, timeout=30)
|
||||
assert response.ok, response.text
|
||||
job = response.json()
|
||||
assert job["status"] not in {"failed", "cancelled", "deleted"}, job
|
||||
if job["status"] == "completed":
|
||||
break
|
||||
assert time.monotonic() < deadline, (job, server.log_tail())
|
||||
time.sleep(0.5)
|
||||
response = client.get(f"{job_url}/content", timeout=60)
|
||||
assert response.ok, response.text
|
||||
content = response.content
|
||||
finally:
|
||||
deleted = client.delete(job_url, timeout=30)
|
||||
assert deleted.ok, deleted.text
|
||||
with av.open(io.BytesIO(content)) as video:
|
||||
assert bool(video.streams.audio) == has_audio
|
||||
assert float(video.streams.video[0].average_rate) == 24
|
||||
frames = np.stack(
|
||||
[frame.to_ndarray(format="rgb24") for frame in video.decode(video=0)]
|
||||
)
|
||||
assert frames.shape == (33, int(256 * scale), int(384 * scale), 3)
|
||||
assert frames.std() > 1, "degenerate video"
|
||||
audio = extract_audio_pcm_from_video_bytes(content) if has_audio else np.empty(0)
|
||||
assert np.isfinite(audio).all()
|
||||
if has_audio:
|
||||
assert audio.size > 0
|
||||
assert np.sqrt(np.mean(np.square(audio, dtype=np.float64))) > 1e-5
|
||||
return frames, audio
|
||||
|
||||
|
||||
def test_sr_requests_preserve_video_and_audio(server, source_videos):
|
||||
source, silent = source_videos
|
||||
first = _generate(server, source)
|
||||
uploaded = _generate(server, source, upload=True)
|
||||
for actual, expected in zip(uploaded, first, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
_generate(server, source, scale=4, upload=True)
|
||||
_generate(server, source, scale=2.25, tiles_batch_size=2)
|
||||
silent_output = _generate(server, silent, upload=True, has_audio=False)
|
||||
np.testing.assert_array_equal(silent_output[0], first[0])
|
||||
restored = _generate(server, source)
|
||||
for actual, expected in zip(restored, first, strict=True):
|
||||
np.testing.assert_array_equal(actual, expected)
|
||||
@@ -0,0 +1,46 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The real component configs (no weights) of the two official Kandinsky 6 VSR Diffusers repos, verbatim from the Hub.
|
||||
|
||||
* flow-matching ``kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers``: ``FlowMatchEulerDiscreteScheduler``, a 64-wide DiT head;
|
||||
* 2-step pi-Flow ``kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers``: ``PiflowScheduler`` (``nfe`` 2,
|
||||
``n_grid`` 10), a 640-wide DiT head.
|
||||
|
||||
Both repos have the same DiT tensor names (only ``out_layer`` differs in width) and byte-identical ``vae/`` and
|
||||
``latent_upscaler/`` files.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
# --- kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers (flow-matching) ---
|
||||
# The dict-valued ``_kandinsky6_sr`` entry of its model_index.json and the root ``sr_config.json`` exist only in this repo.
|
||||
FLOW_MODEL_INDEX = json.loads(
|
||||
r"""{"_class_name":"Kandinsky6SRPipeline","_diffusers_version":"0.39.0","_kandinsky6_sr":{"format_version":1,"latent_upscaler":"latent_upscaler","portable_components":true,"source_checkpoint":"9454578e88f86dd9c40df6a8a79529a7e5a5ecc9","uses_patched_diffusers":true,"vae_name":"video-kvae"},"latent_upscaler":["diffusers","Kandinsky6SRLatentUpscalerBank"],"scheduler":["diffusers","FlowMatchEulerDiscreteScheduler"],"transformer":["diffusers","Kandinsky6SRTransformer3DModel"],"vae":["diffusers","Kandinsky6SRVAE"]}"""
|
||||
)
|
||||
FLOW_SR_CONFIG = json.loads(
|
||||
r"""{"component_loading":"Diffusers ModelMixin wrappers","default_resolution_scale":2.25,"description":"Kandinsky 6 SR Diffusers bundle"}"""
|
||||
)
|
||||
FLOW_TRANSFORMER_CONFIG = json.loads(
|
||||
r"""{"attention_params":{"1024":{"causal":false,"chunk":false,"glob":false,"local":false,"type":"flash","window":3},"512":{"P":0.8,"P_warmup_start":0.95,"P_warmup_steps":200,"add_sta":true,"causal":false,"chunk":false,"glob":false,"local":false,"method":"topcdf","type":"nabla","wH":7,"wT":11,"wW":7,"window":3}},"attribute_overrides":{},"axes_dims":[16,24,24],"ff_dim":7168,"in_text_dim":3584,"in_text_dim2":768,"in_visual_dim":64,"instruct_type":"hybrid_anchor","model_dim":1792,"num_text_blocks":2,"num_visual_blocks":32,"out_visual_dim":64,"patch_size":[1,1,1],"sr_params":{"cap_noise_timestep":false,"fps":24,"lq_channel_noise_scale":0.0,"lq_noise_scale":0.7,"lq_noise_type":"ddpm","scale_factor":{"512":[1.0,2.0,2.0]},"scheduler_scale":5.0,"visual_size":[512]},"time_dim":512,"use_text":false,"visual_cond":true}"""
|
||||
)
|
||||
FLOW_SCHEDULER_CONFIG = json.loads(
|
||||
r"""{"_class_name":"FlowMatchEulerDiscreteScheduler","_diffusers_version":"0.39.0","base_image_seq_len":256,"base_shift":0.5,"invert_sigmas":false,"max_image_seq_len":4096,"max_shift":1.15,"num_train_timesteps":1000,"shift":5.0,"shift_terminal":null,"stochastic_sampling":false,"time_shift_type":"exponential","use_beta_sigmas":false,"use_dynamic_shifting":false,"use_exponential_sigmas":false,"use_karras_sigmas":false}"""
|
||||
)
|
||||
|
||||
# --- kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers (2-step pi-Flow) ---
|
||||
DISTILLED_MODEL_INDEX = json.loads(
|
||||
r"""{"_class_name":"Kandinsky6SRPipeline","_diffusers_version":"0.41.0.dev0","latent_upscaler":["diffusers","Kandinsky6SRLatentUpscalerBank"],"scheduler":["diffusers","PiflowScheduler"],"transformer":["diffusers","Kandinsky6SRTransformer3DModel"],"vae":["diffusers","Kandinsky6SRVAE"]}"""
|
||||
)
|
||||
DISTILLED_TRANSFORMER_CONFIG = json.loads(
|
||||
r"""{"attention_params":{"1024":{"causal":false,"chunk":false,"glob":false,"local":false,"type":"flash","window":3},"512":{"P":0.8,"P_warmup_start":0.95,"P_warmup_steps":0,"add_sta":true,"causal":false,"chunk":false,"glob":false,"local":false,"method":"topcdf","type":"nabla","wH":7,"wT":11,"wW":7,"window":3}},"attribute_overrides":{},"axes_dims":[16,24,24],"ff_dim":7168,"in_text_dim":3584,"in_text_dim2":768,"in_visual_dim":64,"instruct_type":"hybrid_anchor","model_dim":1792,"num_text_blocks":2,"num_visual_blocks":32,"out_visual_dim":640,"patch_size":[1,1,1],"sr_params":{"cap_noise_timestep":false,"fps":24,"lq_channel_noise_scale":0.0,"lq_noise_scale":0.7,"lq_noise_type":"ddpm","scale_factor":{"512":[1.0,2.0,2.0]},"scheduler_scale":3.5,"visual_size":[512]},"time_dim":512,"use_text":false,"visual_cond":true}"""
|
||||
)
|
||||
DISTILLED_SCHEDULER_CONFIG = json.loads(
|
||||
r"""{"_class_name":"PiflowScheduler","_diffusers_version":"0.41.0.dev0","eps":1e-06,"final_step_size_scale":0.5,"n_grid":10,"nfe":2,"num_policy_substeps":128,"num_train_timesteps":1000,"shift":3.5}"""
|
||||
)
|
||||
|
||||
# --- identical in both repos (the VAE config stores the *string* "None" for unused channel fields) ---
|
||||
VAE_CONFIG = json.loads(
|
||||
r"""{"decoder_config":{"ch":256,"ch_mult":[0.0625,1,2,4,8],"checkpoint_list":[true,true,true,true,true,true],"in_channels":"None","norm_type":"rms_norm","num_res_blocks":2,"out_ch":3,"padding_mode":"zeros","resolution":0,"temporal_compress_start_level":1,"temporal_compress_times":4,"z_channels":64},"encoder_config":{"ch":128,"ch_mult":[0.125,1,2,4,8],"checkpoint_list":[true,true,true,true,true,true],"double_z":true,"downsample_version":2,"fix_pxs":true,"in_channels":3,"norm_type":"rms_norm","num_res_blocks":2,"out_ch":"None","padding_mode":"zeros","resolution":0,"temporal_compress_start_level":1,"temporal_compress_times":4,"z_channels":64},"scaling_factor":0.910344004631042,"spatial_factor":16,"temporal_factor":4,"vae_type":"video-kvae"}"""
|
||||
)
|
||||
LATENT_UPSCALER_CONFIG = json.loads(
|
||||
r"""{"models":[{"model":{"architecture":"multi_scale","bare_stem":true,"bottleneck_channels":null,"depthwise":false,"dims":3,"enable_x2_entry":false,"expand_ratio":1,"gradient_checkpointing":false,"grn":false,"hidden_channels":2048,"in_channels":64,"input_skip":false,"kernel_size":3,"layer_scale_init":null,"loss_weight_2x":0.0,"loss_weight_4x":1.0,"modulated_norm":true,"modulated_output_proj":true,"num_mid_blocks":3,"num_post_blocks":3,"num_pre_blocks":5,"stage_channels":[2048,1024,512],"stochastic_depth_rate":0.0,"temporal_mix":false,"temporal_padding":"replicate","upsample_mode":"pxs_v2","upsample_padding_mode":"zeros","upscale_factor":4},"target_scale":"4x"},{"model":{"architecture":"multi_scale","bare_stem":true,"bottleneck_channels":null,"depthwise":false,"dims":3,"enable_x2_entry":true,"expand_ratio":1,"gradient_checkpointing":false,"grn":false,"hidden_channels":2048,"in_channels":64,"input_skip":false,"kernel_size":3,"layer_scale_init":null,"loss_weight_2x":0.0,"loss_weight_4x":1.0,"modulated_norm":true,"modulated_output_proj":true,"num_mid_blocks":3,"num_post_blocks":3,"num_pre_blocks":5,"stage_channels":[2048,1024,512],"stochastic_depth_rate":0.0,"temporal_mix":false,"temporal_padding":"replicate","upsample_mode":"pxs_v2","upsample_padding_mode":"zeros","upscale_factor":4,"x2_adapter_blocks":2,"x2_adapter_sources":[0,4],"x2_finisher":"pxs_residual","x2_tail_mode":"private_full"},"target_scale":"2x"}],"scaling_factor":0.910344004631042}"""
|
||||
)
|
||||
@@ -0,0 +1,259 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Tiny random Kandinsky 6 SR components and requests for the stage-level tests."""
|
||||
|
||||
import copy
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
Kandinsky6SRVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRTransformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
|
||||
Kandinsky6SRLatentUpscalerBank,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_REQUESTED_HW_KEY,
|
||||
SR_TILING_SCALE_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
DitSpec,
|
||||
SamplingSpec,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
LatentUpscaleFn,
|
||||
decode_chunks,
|
||||
denoise_chunks,
|
||||
encode_video_to_lr_latent,
|
||||
plan_tiles,
|
||||
prepare_lu_tile_latents,
|
||||
prepare_pixel_tile_latents,
|
||||
stitch_tiles,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
RESOLUTIONS,
|
||||
VAE_SPATIAL_FACTOR,
|
||||
)
|
||||
|
||||
# Flat (legacy) transformer config with a pi-Flow DX head: 3 grids of 4 channels.
|
||||
TINY_DIT = dict(
|
||||
in_visual_dim=4,
|
||||
in_text_dim=8,
|
||||
in_text_dim2=8,
|
||||
time_dim=16,
|
||||
out_visual_dim=4,
|
||||
patch_size=[1, 2, 2],
|
||||
model_dim=32,
|
||||
ff_dim=64,
|
||||
num_text_blocks=0,
|
||||
num_visual_blocks=2,
|
||||
axes_dims=[8, 4, 4],
|
||||
visual_cond=False,
|
||||
instruct_type="noise",
|
||||
use_text=False,
|
||||
n_grid=3,
|
||||
piflow_nfe=2,
|
||||
piflow_num_policy_substeps=8,
|
||||
piflow_final_step_size_scale=0.5,
|
||||
piflow_shift=5.0,
|
||||
piflow_eps=1e-6,
|
||||
attribute_overrides={"instruct_type": "noise"},
|
||||
sr_visual_size=[512],
|
||||
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
|
||||
)
|
||||
TINY_KVAE = dict(
|
||||
ch=8,
|
||||
ch_mult=(1, 1, 2, 2, 2),
|
||||
num_res_blocks=2,
|
||||
z_channels=4,
|
||||
temporal_compress_times=4,
|
||||
norm_type="rms_norm",
|
||||
)
|
||||
TINY_LU_MODEL = {
|
||||
"architecture": "multi_scale",
|
||||
"in_channels": 4,
|
||||
"hidden_channels": 8,
|
||||
"num_pre_blocks": 1,
|
||||
"num_mid_blocks": 2,
|
||||
"num_post_blocks": 2,
|
||||
"expand_ratio": 2,
|
||||
"dims": 3,
|
||||
"bare_stem": True,
|
||||
"input_skip": False,
|
||||
"modulated_norm": True,
|
||||
"modulated_output_proj": True,
|
||||
"temporal_padding": "replicate",
|
||||
"upsample_mode": "pxs_v2",
|
||||
"upsample_padding_mode": "zeros",
|
||||
"enable_x2_entry": True,
|
||||
"x2_adapter_blocks": 1,
|
||||
"x2_tail_mode": "private_full",
|
||||
"x2_finisher": "pxs_residual",
|
||||
}
|
||||
|
||||
|
||||
def _randomize(module, seed):
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
with torch.no_grad():
|
||||
for param in module.parameters():
|
||||
param.copy_(torch.randn(param.shape, generator=generator) * 0.05)
|
||||
|
||||
|
||||
def build_components(bank_scales, dit_arch=TINY_DIT):
|
||||
"""``(vae, dit, latent-upscaler bank or None, server args)`` with random weights."""
|
||||
vae_config = Kandinsky6SRVAEConfig()
|
||||
vae_config.update_model_arch(
|
||||
dict(
|
||||
vae_type="video-kvae",
|
||||
encoder_config=dict(TINY_KVAE, in_channels=3),
|
||||
decoder_config=dict(TINY_KVAE, out_ch=3),
|
||||
scaling_factor=0.5,
|
||||
)
|
||||
)
|
||||
vae = Kandinsky6SRVAE(vae_config).eval()
|
||||
_randomize(vae, 1)
|
||||
dit_config = Kandinsky6SRDitConfig()
|
||||
dit_config.update_model_arch(dict(dit_arch))
|
||||
dit = Kandinsky6SRTransformer3DModel(dit_config, dict(dit_arch)).eval()
|
||||
_randomize(dit, 2)
|
||||
bank = None
|
||||
if bank_scales:
|
||||
models = [
|
||||
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
|
||||
for scale in bank_scales
|
||||
]
|
||||
bank = Kandinsky6SRLatentUpscalerBank(
|
||||
models,
|
||||
0.5,
|
||||
scales=tuple(int(scale.removesuffix("x")) for scale in bank_scales),
|
||||
).eval()
|
||||
_randomize(bank, 3)
|
||||
# keep initialization reproducible on CPU, then exercise the native device kernels
|
||||
device = get_local_torch_device()
|
||||
vae.to(device)
|
||||
dit.to(device)
|
||||
if bank is not None:
|
||||
bank.to(device)
|
||||
pipeline_config = Kandinsky6SRPipelineConfig()
|
||||
pipeline_config.dit_config = dit_config
|
||||
server_args = SimpleNamespace(
|
||||
pipeline_config=pipeline_config,
|
||||
component_precisions={},
|
||||
# Kandinsky6SRDenoisingStage reuses the shared DenoisingStage's cache-dit /
|
||||
# torch.compile wiring (_maybe_enable_cache_dit_and_torch_compile), which reads
|
||||
# these; both stay off for this fixture.
|
||||
enable_breakable_cuda_graph=False,
|
||||
enable_torch_compile=False,
|
||||
)
|
||||
return vae, dit, bank, server_args
|
||||
|
||||
|
||||
def make_stage(stage_cls, server_args, *args, **kwargs):
|
||||
stage = stage_cls(*args, **kwargs)
|
||||
stage.server_args = server_args # component_uses() reads the bf16 precisions
|
||||
return stage
|
||||
|
||||
|
||||
def make_request(video, *, tiles_batch_size=1, num_steps=5):
|
||||
params = Kandinsky6SRSamplingParams(
|
||||
video_path="unused.mp4",
|
||||
sr_resolution_scale=2,
|
||||
sr_tiles_batch_size=tiles_batch_size,
|
||||
num_inference_steps=num_steps,
|
||||
seed=42,
|
||||
)
|
||||
batch = Req(sampling_params=params)
|
||||
batch.extra[SR_VIDEO_KEY] = video.clone()
|
||||
batch.extra[SR_TILING_SCALE_KEY] = 2
|
||||
batch.extra[SR_REQUESTED_HW_KEY] = (video.shape[-2], video.shape[-1])
|
||||
return batch
|
||||
|
||||
|
||||
def random_video(frames, height, width, seed=0):
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
shape = (frames, 3, height, width)
|
||||
return torch.randint(0, 256, shape, dtype=torch.uint8, generator=generator)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def super_resolve(
|
||||
video: torch.Tensor,
|
||||
*,
|
||||
vae: torch.nn.Module,
|
||||
scaling_factor: float,
|
||||
dit: Callable[..., torch.Tensor],
|
||||
dit_spec: DitSpec,
|
||||
spec: SamplingSpec,
|
||||
scheduler: Any,
|
||||
device: torch.device | str,
|
||||
upscale_fn: LatentUpscaleFn | None = None,
|
||||
lu_dtype: torch.dtype | None = None,
|
||||
spatial_factor: int = VAE_SPATIAL_FACTOR,
|
||||
resolutions: Mapping[int, Sequence[tuple[int, int]]] = RESOLUTIONS,
|
||||
) -> torch.Tensor:
|
||||
"""Tiled SR of a ``[T, C, H, W]`` uint8 video -> uint8 ``[3, T, H*s, W*s]``.
|
||||
|
||||
``upscale_fn`` selects the LU path; without it the pixel path runs. ``scheduler`` is the
|
||||
already-resolved effective scheduler (``run_spec.effective_scheduler``) -- a real loaded
|
||||
component, or one synthesized from ``spec.piflow`` / ``spec.scheduler_scale`` when none fits.
|
||||
"""
|
||||
device = torch.device(device)
|
||||
common = dict(
|
||||
visual_size=spec.visual_size,
|
||||
tiling_scale=spec.tiling_scale,
|
||||
tile_min_overlap=spec.tile_min_overlap,
|
||||
spatial_factor=spatial_factor,
|
||||
resolutions=resolutions,
|
||||
)
|
||||
if upscale_fn is not None:
|
||||
lr_latent = encode_video_to_lr_latent(video, vae, device=device).cpu()
|
||||
frame_hw = (
|
||||
lr_latent.shape[-2] * spatial_factor,
|
||||
lr_latent.shape[-1] * spatial_factor,
|
||||
)
|
||||
plan = plan_tiles(frame_hw=frame_hw, **common)
|
||||
chunks = prepare_lu_tile_latents(
|
||||
lr_latent,
|
||||
plan,
|
||||
upscale_fn=upscale_fn,
|
||||
lu_dtype=lu_dtype,
|
||||
scaling_factor=scaling_factor,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
spatial_factor=spatial_factor,
|
||||
)
|
||||
else:
|
||||
plan = plan_tiles(frame_hw=tuple(video.shape[-2:]), **common)
|
||||
chunks = prepare_pixel_tile_latents(
|
||||
video,
|
||||
plan,
|
||||
vae=vae,
|
||||
scaling_factor=scaling_factor,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=device,
|
||||
)
|
||||
denoised = denoise_chunks(
|
||||
chunks, dit, scheduler, dit_spec=dit_spec, spec=spec, device=device
|
||||
)
|
||||
tiles = decode_chunks(denoised, vae, scaling_factor=scaling_factor, device=device)
|
||||
return stitch_tiles(tiles, plan)
|
||||
@@ -5,7 +5,7 @@ import sys
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
|
||||
class _FakeDBCacheConfig:
|
||||
@@ -19,6 +19,7 @@ class _FakeDBCacheConfig:
|
||||
class _FakeForwardPattern:
|
||||
# A class (not a SimpleNamespace instance) so it is a valid type in
|
||||
# annotations like List[ForwardPattern], matching the real Enum.
|
||||
Pattern_0 = "Pattern_0"
|
||||
Pattern_2 = "Pattern_2"
|
||||
Pattern_3 = "Pattern_3"
|
||||
|
||||
@@ -155,8 +156,10 @@ def _install_torch_stub():
|
||||
torch_nn.Module = _FakeModule
|
||||
torch_dist.ProcessGroup = _FakeProcessGroup
|
||||
torch_dist.ReduceOp = _FakeReduceOp
|
||||
torch_dist.is_initialized = lambda: False
|
||||
torch.distributed = torch_dist
|
||||
torch.nn = torch_nn
|
||||
torch.Tensor = object
|
||||
|
||||
return {
|
||||
"torch": torch,
|
||||
@@ -218,7 +221,50 @@ def _import_module_with_legacy_stub():
|
||||
return module, stub_modules
|
||||
|
||||
|
||||
class TestCacheDitSimilarityGroups(unittest.TestCase):
|
||||
def test_independent_cfg_or_dp_peers_do_not_reduce_over_world(self):
|
||||
for group in (None, "singleton-sp"):
|
||||
with self.subTest(group=group):
|
||||
module = _import_module_with_stub()
|
||||
original = Mock(return_value=True)
|
||||
manager_type = type(
|
||||
"CachedContextManager", (), {"similarity": original}
|
||||
)
|
||||
contexts = types.ModuleType("cache_dit.caching.cache_contexts")
|
||||
contexts.cache_manager = types.SimpleNamespace(
|
||||
CachedContextManager=manager_type
|
||||
)
|
||||
module.dist.get_world_size = Mock(return_value=1)
|
||||
with patch.dict(sys.modules, {contexts.__name__: contexts}):
|
||||
module._patch_cache_dit_similarity()
|
||||
manager = manager_type()
|
||||
manager._sglang_sp_group = group
|
||||
manager.similarity(
|
||||
"previous", "current", threshold=0.1, parallelized=True
|
||||
)
|
||||
self.assertFalse(original.call_args.kwargs["parallelized"])
|
||||
|
||||
|
||||
class TestCacheDitRefreshContext(unittest.TestCase):
|
||||
def test_refresh_preserves_explicit_scm_mask_and_policy(self):
|
||||
module = _import_module_with_stub()
|
||||
module.refresh_context_on_transformer(
|
||||
"transformer",
|
||||
4,
|
||||
scm_preset="fast",
|
||||
steps_computation_mask=[1, 0, 1, 0],
|
||||
steps_computation_policy="static",
|
||||
)
|
||||
self.assertEqual(module.cache_dit.steps_mask_calls, [])
|
||||
self.assertEqual(
|
||||
module.cache_dit.refresh_calls[0]["cache_config"],
|
||||
{
|
||||
"num_inference_steps": 4,
|
||||
"steps_computation_mask": [1, 0, 1, 0],
|
||||
"steps_computation_policy": "static",
|
||||
},
|
||||
)
|
||||
|
||||
def test_refresh_context_without_scm_preset_skips_steps_mask(self):
|
||||
module = _import_module_with_stub()
|
||||
module.refresh_context_on_transformer(
|
||||
@@ -298,6 +344,51 @@ def _make_transformer(class_name, layers=None):
|
||||
|
||||
|
||||
class TestBuildCustomBlockAdapter(unittest.TestCase):
|
||||
def test_fsdp_subclass_preserves_native_adapter(self):
|
||||
module = _import_module_with_stub()
|
||||
for name, attr, multimodal, pattern in (
|
||||
(
|
||||
"Kandinsky6Transformer3DModel",
|
||||
"visual_transformer_blocks",
|
||||
True,
|
||||
"Pattern_0",
|
||||
),
|
||||
(
|
||||
"Kandinsky6Transformer3DModel",
|
||||
"visual_transformer_blocks",
|
||||
False,
|
||||
"Pattern_2",
|
||||
),
|
||||
("AnimaTransformer3DModel", "transformer_blocks", False, "Pattern_3"),
|
||||
):
|
||||
with self.subTest(name=name, multimodal=multimodal):
|
||||
blocks = ["block_0", "block_1"]
|
||||
native = type(name, (), {attr: blocks})
|
||||
fsdp_mixin = type("FSDPModule", (), {})
|
||||
transformer = type(f"FSDP{name}", (fsdp_mixin, native), {})()
|
||||
transformer.config = types.SimpleNamespace(is_multimodal=multimodal)
|
||||
adapter = module._build_custom_block_adapter(
|
||||
transformer, has_separate_cfg=True
|
||||
)
|
||||
self.assertIs(adapter.transformer, transformer)
|
||||
self.assertIs(adapter.blocks, blocks)
|
||||
self.assertEqual(adapter.forward_pattern, pattern)
|
||||
self.assertTrue(adapter.has_separate_cfg)
|
||||
|
||||
def test_kandinsky6_caches_both_joint_streams(self):
|
||||
module = _import_module_with_stub()
|
||||
transformer = _make_transformer("Kandinsky6Transformer3DModel")
|
||||
transformer.visual_transformer_blocks = ["joint_block"]
|
||||
for multimodal, pattern in ((True, "Pattern_0"), (False, "Pattern_2")):
|
||||
with self.subTest(multimodal=multimodal):
|
||||
transformer.config = types.SimpleNamespace(is_multimodal=multimodal)
|
||||
adapter = module._build_custom_block_adapter(
|
||||
transformer, has_separate_cfg=True
|
||||
)
|
||||
self.assertIs(adapter.blocks, transformer.visual_transformer_blocks)
|
||||
self.assertEqual(adapter.forward_pattern, pattern)
|
||||
self.assertTrue(adapter.has_separate_cfg)
|
||||
|
||||
def test_builds_adapter_for_registered_class(self):
|
||||
module = _import_module_with_stub()
|
||||
blocks = ["block_0", "block_1"]
|
||||
|
||||
@@ -99,10 +99,11 @@ def _batch(
|
||||
enable_cache_dit=None,
|
||||
cache_dit_params=None,
|
||||
is_warmup=False,
|
||||
do_classifier_free_guidance=False,
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
is_warmup=is_warmup,
|
||||
do_classifier_free_guidance=False,
|
||||
do_classifier_free_guidance=do_classifier_free_guidance,
|
||||
sampling_params=SimpleNamespace(
|
||||
enable_cache_dit=enable_cache_dit,
|
||||
cache_dit_params=cache_dit_params,
|
||||
@@ -116,6 +117,7 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase):
|
||||
self.stage.server_args = SimpleNamespace(
|
||||
enable_breakable_cuda_graph=False,
|
||||
enable_torch_compile=False,
|
||||
enable_cfg_parallel=False,
|
||||
)
|
||||
self.stage.transformer = object()
|
||||
self.stage.transformer_2 = None
|
||||
@@ -127,9 +129,12 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase):
|
||||
self.enable_calls = []
|
||||
self.disable_calls = []
|
||||
self.refresh_calls = []
|
||||
self.cfg_modes = []
|
||||
self.refresh_options = []
|
||||
|
||||
def fake_enable(transformer, config, **kwargs):
|
||||
self.enable_calls.append(config)
|
||||
self.cfg_modes.append(kwargs["has_separate_cfg"])
|
||||
return transformer
|
||||
|
||||
def fake_disable(transformer):
|
||||
@@ -138,6 +143,7 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase):
|
||||
|
||||
def fake_refresh(transformer, num_inference_steps, scm_preset=None, **kwargs):
|
||||
self.refresh_calls.append(num_inference_steps)
|
||||
self.refresh_options.append(kwargs)
|
||||
|
||||
patchers = [
|
||||
patch.object(denoising_module, "enable_cache_on_transformer", fake_enable),
|
||||
@@ -157,7 +163,42 @@ class TestPerRequestCacheDitTransitions(unittest.TestCase):
|
||||
self.stage._maybe_enable_cache_dit(8, _batch(enable_cache_dit=True))
|
||||
self.assertEqual(len(self.enable_calls), 1)
|
||||
self.assertTrue(self.stage._cache_dit_enabled)
|
||||
self.assertEqual(self.stage._cache_dit_active_key, cache_dit_overrides_key({}))
|
||||
self.assertEqual(
|
||||
self.stage._cache_dit_active_key, (cache_dit_overrides_key({}), False)
|
||||
)
|
||||
|
||||
def test_cfg_change_remounts_and_parallel_cfg_has_one_branch_per_rank(self):
|
||||
self.stage._maybe_enable_cache_dit(8, _batch(enable_cache_dit=True))
|
||||
cfg_batch = _batch(enable_cache_dit=True, do_classifier_free_guidance=True)
|
||||
self.stage._maybe_enable_cache_dit(8, cfg_batch)
|
||||
self.stage.server_args.enable_cfg_parallel = True
|
||||
self.stage._maybe_enable_cache_dit(8, cfg_batch)
|
||||
self.assertEqual(self.cfg_modes, [False, True, False])
|
||||
self.assertEqual(len(self.disable_calls), 2)
|
||||
|
||||
def test_repeated_custom_scm_request_keeps_mask_and_policy(self):
|
||||
batch = _batch(
|
||||
enable_cache_dit=True,
|
||||
cache_dit_params={
|
||||
"scm_compute_bins": [2, 2],
|
||||
"scm_cache_bins": [2, 2],
|
||||
"scm_policy": "static",
|
||||
},
|
||||
)
|
||||
self.stage._maybe_enable_cache_dit(8, batch)
|
||||
self.stage._maybe_enable_cache_dit(8, batch)
|
||||
self.assertEqual(len(self.enable_calls), 1)
|
||||
self.assertEqual(
|
||||
self.refresh_options,
|
||||
[
|
||||
{
|
||||
"steps_computation_mask": self.enable_calls[
|
||||
0
|
||||
].steps_computation_mask,
|
||||
"steps_computation_policy": "static",
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
def test_server_default_off_unmounts_after_request_enable(self):
|
||||
self.stage._maybe_enable_cache_dit(8, _batch(enable_cache_dit=True))
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import weakref
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch.nn.attention import SDPBackend, sdpa_kernel
|
||||
|
||||
from sglang.multimodal_gen.runtime.layers.parallel_conv import (
|
||||
disable_spatial_parallel_decode,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.hunyuanvae import (
|
||||
HunyuanVAEAttention,
|
||||
HunyuanVideoMidBlock3D,
|
||||
HunyuanVideoUpsampleCausal3D,
|
||||
prepare_causal_attention_mask,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("frames", [1, 3, 7])
|
||||
@pytest.mark.parametrize("batch_size,heads", [(1, 1), (2, 2)])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16, torch.float16])
|
||||
def test_frame_attention_matches_dense_causal_mask(frames, batch_size, heads, dtype):
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA is required for the production SDPA path")
|
||||
torch.manual_seed(0)
|
||||
attention = HunyuanVAEAttention(32, heads, 16, 1e-6, 4, True).cuda().to(dtype)
|
||||
hidden = torch.randn(batch_size, frames * 15, 32, device="cuda", dtype=dtype)
|
||||
mask = prepare_causal_attention_mask(frames, 15, dtype, hidden.device)
|
||||
with torch.inference_mode():
|
||||
expected = attention(hidden, attention_mask=mask)
|
||||
actual = attention(hidden, num_frames=frames)
|
||||
# unmasked prefixes may select a different fused SDPA reduction kernel
|
||||
tolerance = max(1e-6, torch.finfo(dtype).eps)
|
||||
torch.testing.assert_close(actual, expected, rtol=tolerance, atol=tolerance)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size,heads", [(1, 1), (2, 2)])
|
||||
def test_bf16_frame_attention_matches_dense_with_same_math_backend(batch_size, heads):
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA is required for the production SDPA path")
|
||||
torch.manual_seed(0)
|
||||
attention = HunyuanVAEAttention(32, heads, 16, 1e-6, 4, True).cuda().bfloat16()
|
||||
hidden = torch.randn(batch_size, 105, 32, device="cuda", dtype=torch.bfloat16)
|
||||
mask = prepare_causal_attention_mask(7, 15, hidden.dtype, hidden.device)
|
||||
with torch.inference_mode(), sdpa_kernel(SDPBackend.MATH):
|
||||
expected = attention(hidden, attention_mask=mask)
|
||||
actual = attention(hidden, num_frames=7)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_spatial_attention_avoids_dense_mask_and_preserves_height_shards():
|
||||
torch.manual_seed(0)
|
||||
attention = HunyuanVAEAttention(8, 2, 4, 1e-6, 2, True)
|
||||
stage = HunyuanVideoMidBlock3D.__new__(HunyuanVideoMidBlock3D)
|
||||
torch.nn.Module.__init__(stage)
|
||||
stage.spatial_parallel = True
|
||||
hidden = torch.randn(1, 8, 3, 5, 3)
|
||||
local = hidden[:, :, :, :2]
|
||||
mask = prepare_causal_attention_mask(3, 15, hidden.dtype, hidden.device)
|
||||
with torch.inference_mode():
|
||||
expected = attention(hidden.permute(0, 2, 3, 4, 1).flatten(1, 3), mask)
|
||||
expected = expected.unflatten(1, (3, 5, 3)).permute(0, 4, 1, 2, 3)
|
||||
with (
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.gather_variable_height",
|
||||
return_value=(hidden, [2, 3]),
|
||||
) as gather,
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.chunk_height_by_sizes",
|
||||
side_effect=lambda value, sizes: value[:, :, :, : sizes[0]],
|
||||
) as chunk,
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.prepare_causal_attention_mask",
|
||||
side_effect=AssertionError(
|
||||
"spatial decode must not build a dense mask"
|
||||
),
|
||||
),
|
||||
):
|
||||
actual = stage._run_attention(attention, local)
|
||||
gather.assert_called_once_with(local)
|
||||
assert chunk.call_args.args[1] == [2, 3]
|
||||
torch.testing.assert_close(actual, expected[:, :, :, :2])
|
||||
|
||||
|
||||
def test_frame_attention_preserves_gradient_flow():
|
||||
torch.manual_seed(0)
|
||||
attention = HunyuanVAEAttention(8, 2, 4, 1e-6, 2, True).double()
|
||||
hidden = torch.randn(1, 12, 8, dtype=torch.float64, requires_grad=True)
|
||||
mask = prepare_causal_attention_mask(3, 4, hidden.dtype, hidden.device)
|
||||
expected = attention(hidden, attention_mask=mask)
|
||||
expected_grad = torch.autograd.grad(expected.square().sum(), hidden)[0]
|
||||
actual = attention(hidden, num_frames=3)
|
||||
actual_grad = torch.autograd.grad(actual.square().sum(), hidden)[0]
|
||||
torch.testing.assert_close(actual, expected)
|
||||
torch.testing.assert_close(actual_grad, expected_grad)
|
||||
|
||||
|
||||
def test_disabled_spatial_decode_preserves_dense_attention():
|
||||
torch.manual_seed(0)
|
||||
attention = HunyuanVAEAttention(8, 2, 4, 1e-6, 2, True)
|
||||
stage = HunyuanVideoMidBlock3D.__new__(HunyuanVideoMidBlock3D)
|
||||
torch.nn.Module.__init__(stage)
|
||||
hidden = torch.randn(1, 8, 3, 5, 3)
|
||||
with torch.inference_mode():
|
||||
stage.spatial_parallel = False
|
||||
expected = stage._run_attention(attention, hidden)
|
||||
stage.spatial_parallel = True
|
||||
with (
|
||||
disable_spatial_parallel_decode(),
|
||||
patch.object(attention, "forward", wraps=attention.forward) as forward,
|
||||
):
|
||||
actual = stage._run_attention(attention, hidden)
|
||||
assert "num_frames" not in forward.call_args.kwargs
|
||||
assert forward.call_args.kwargs["attention_mask"].shape == (1, 45, 45)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_upsample_releases_intermediates_before_convolution():
|
||||
upsample = HunyuanVideoUpsampleCausal3D(4)
|
||||
hidden = torch.randn(1, 4, 3, 5, 7)
|
||||
interpolate = torch.nn.functional.interpolate
|
||||
buffers = []
|
||||
|
||||
def track_interpolate(*args, **kwargs):
|
||||
result = interpolate(*args, **kwargs)
|
||||
buffers.append(weakref.ref(result))
|
||||
return result
|
||||
|
||||
def check_input(value):
|
||||
assert all(buffer() is None for buffer in buffers)
|
||||
return value
|
||||
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.models.vaes.hunyuanvae.F.interpolate",
|
||||
side_effect=track_interpolate,
|
||||
),
|
||||
patch.object(upsample.conv, "forward", side_effect=check_input),
|
||||
):
|
||||
actual = upsample(hidden)
|
||||
expected = torch.cat(
|
||||
(
|
||||
interpolate(hidden[:, :, 0], scale_factor=2, mode="nearest").unsqueeze(2),
|
||||
interpolate(hidden[:, :, 1:].contiguous(), scale_factor=2, mode="nearest"),
|
||||
),
|
||||
dim=2,
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
@@ -1,14 +1,51 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a import (
|
||||
IpcA2AState,
|
||||
_peer_cuda_device,
|
||||
_Unsupported,
|
||||
ipc_a2a_ready,
|
||||
)
|
||||
|
||||
_IPC = "sglang.multimodal_gen.runtime.distributed.device_communicators.ipc_a2a"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rank", [0, 1])
|
||||
@pytest.mark.parametrize("nvlink", [False, True])
|
||||
def test_ipc_requires_nvlink_before_mapping_peer_memory(monkeypatch, rank, nvlink):
|
||||
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "3,1")
|
||||
state = IpcA2AState()
|
||||
with (
|
||||
patch(f"{_IPC}.dist.get_rank", return_value=rank),
|
||||
patch(f"{_IPC}.torch.cuda.current_device", return_value=rank),
|
||||
patch(f"{_IPC}._peer_cuda_device", return_value=1 - rank),
|
||||
patch(
|
||||
"sglang.multimodal_gen.runtime.platforms.current_platform.is_full_nvlink",
|
||||
return_value=nvlink,
|
||||
) as topology,
|
||||
patch(f"{_IPC}.torch.cuda.can_device_access_peer", return_value=True) as peer,
|
||||
patch("ctypes.CDLL") as cudart,
|
||||
patch(f"{_IPC}.load_ipc_a2a_sync") as kernels,
|
||||
patch(f"{_IPC}.torch.zeros"),
|
||||
patch.object(state, "_share"),
|
||||
):
|
||||
if nvlink:
|
||||
state.init(object())
|
||||
assert state.inited
|
||||
peer.assert_called_once_with(rank, 1 - rank)
|
||||
kernels.assert_called_once()
|
||||
else:
|
||||
with pytest.raises(_Unsupported, match="NVLink"):
|
||||
state.init(object())
|
||||
assert not state.inited
|
||||
peer.assert_not_called()
|
||||
cudart.assert_not_called()
|
||||
kernels.assert_not_called()
|
||||
topology.assert_called_once_with([1, 3])
|
||||
|
||||
|
||||
def test_reinitializes_ipc_transport_for_replaced_process_group():
|
||||
state = IpcA2AState()
|
||||
old_group = object()
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Checkpoint architecture validation and native parameter-name mapping."""
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
|
||||
Kandinsky6ArchConfig,
|
||||
Kandinsky6VideoAudioConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
|
||||
Kandinsky6AudioVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides,expected",
|
||||
[
|
||||
({}, (48, 32, 128, (8, 8, 8))),
|
||||
({"ff_dim_a": 96, "axes_dims_a": (4, 4, 4)}, (48, 32, 96, (4, 4, 4))),
|
||||
({"time_dim_a": 16, "fix_modulation": True}, (48, 16, 128, (8, 8, 8))),
|
||||
({"time_dim_a": 16, "is_multimodal": False}, (48, 16, 128, (8, 8, 8))),
|
||||
],
|
||||
)
|
||||
def test_audio_dimension_inheritance_and_overrides(overrides, expected):
|
||||
arch = Kandinsky6ArchConfig(
|
||||
model_dim=48, time_dim=32, ff_dim=128, axes_dims=(8, 8, 8), **overrides
|
||||
)
|
||||
assert (
|
||||
arch.model_dim_a,
|
||||
arch.time_dim_a,
|
||||
arch.ff_dim_a,
|
||||
arch.axes_dims_a,
|
||||
) == expected
|
||||
assert (arch.hidden_size, arch.num_attention_heads) == (48, 2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides,error",
|
||||
[
|
||||
({"model_dim_a": 50, "axes_dims_a": (4, 4, 4)}, "model_dim_a"),
|
||||
({"time_dim_a": 16}, "time_dim_a"),
|
||||
],
|
||||
)
|
||||
def test_incompatible_audio_dimensions_are_rejected(overrides, error):
|
||||
with pytest.raises(ValueError, match=error):
|
||||
Kandinsky6ArchConfig(
|
||||
model_dim=48, time_dim=32, axes_dims=(8, 8, 8), **overrides
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source,target",
|
||||
[
|
||||
(
|
||||
"video_text_transformer_blocks.0.feed_forward.net.0.proj.weight",
|
||||
"video_text_transformer_blocks.0.feed_forward.mlp.fc_in.weight",
|
||||
),
|
||||
(
|
||||
"video_text_transformer_blocks.3.attn.to_q.weight",
|
||||
"video_text_transformer_blocks.3.self_attention.to_q.weight",
|
||||
),
|
||||
(
|
||||
"video_time_embeddings.timestep_embedder.linear_1.weight",
|
||||
"video_time_embeddings.in_layer.weight",
|
||||
),
|
||||
(
|
||||
"visual_transformer_blocks.0.video_dec_block.feed_forward.net.2.bias",
|
||||
"visual_transformer_blocks.0.videoT.feed_forward.mlp.fc_out.bias",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_checkpoint_name_mapping(source, target):
|
||||
map_name = get_param_names_mapping(Kandinsky6ArchConfig().param_names_mapping)
|
||||
assert map_name(source)[0] == target
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_grid", [1, 10], ids=["sft", "distilled"])
|
||||
def test_checkpoint_head_width_does_not_change_latent_width(n_grid):
|
||||
config = Kandinsky6VideoAudioConfig()
|
||||
config.update_model_arch(
|
||||
{
|
||||
"in_visual_dim": 16,
|
||||
"out_visual_dim": 16 * n_grid,
|
||||
"in_audio_dim": 40,
|
||||
"out_audio_dim": 40 * n_grid,
|
||||
"model_dim_a": 2048,
|
||||
"ff_dim_a": 7168,
|
||||
"fix_modulation": True,
|
||||
"scale_factor": [1.0, 2.0, 2.0],
|
||||
}
|
||||
)
|
||||
arch = config.arch_config
|
||||
assert (arch.in_channels, arch.out_channels, arch.num_channels_latents) == (
|
||||
16,
|
||||
16 * n_grid,
|
||||
16,
|
||||
)
|
||||
assert (arch.in_audio_dim, arch.out_audio_dim) == (40, 40 * n_grid)
|
||||
assert tuple(arch.scale_factor) == (1.0, 2.0, 2.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scaling_factor", [0.5302, 0.417])
|
||||
def test_audio_vae_scaling_factor_comes_from_the_checkpoint(scaling_factor):
|
||||
config = Kandinsky6AudioVAEConfig()
|
||||
config.update_model_arch(
|
||||
{
|
||||
"mode": "44k",
|
||||
"sample_rate": 44100,
|
||||
"downsample_factor": 1024,
|
||||
"scaling_factor": scaling_factor,
|
||||
}
|
||||
)
|
||||
assert config.arch_config.scaling_factor == scaling_factor
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""GAP 2 regression: ``Kandinsky6AudioVAE`` must not re-renormalize an
|
||||
already-loaded (final) checkpoint's audio-VAE Conv1d weights.
|
||||
|
||||
The diffusers reference calls ``MMAudioVAE.remove_weight_norm()`` exactly
|
||||
once, inside ``MMAudioVAE.__init__`` -- BEFORE ``from_pretrained`` overwrites
|
||||
those freshly-constructed (random) weights with the checkpoint's real,
|
||||
already-final values, so a real checkpoint's own weights are never
|
||||
renormalized. The pre-fix SGLang port instead triggered the same
|
||||
RMS-renormalization lazily, on the first ``decode()`` call -- i.e. AFTER a
|
||||
real checkpoint's weights were loaded -- corrupting them (measured by the
|
||||
audit at ~9.7% decoded-waveform rel-L2 error against the diffusers
|
||||
reference).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
|
||||
Kandinsky6AudioVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio import (
|
||||
Kandinsky6AudioVAE,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio_vae.mmaudio_vae import (
|
||||
MMAudioVAE,
|
||||
)
|
||||
|
||||
# A small, self-contained BigVGANV2 config (not a real checkpoint's) just so
|
||||
# Kandinsky6AudioVAE.__init__ can build the whole module tree on CPU quickly;
|
||||
# none of these tests call vocode()/wrapped_decode(), only the mel-level
|
||||
# decode() where the audited bug lives.
|
||||
_TINY_VOCODER_CONFIG = {
|
||||
"resblock": "1",
|
||||
"num_mels": 80, # matches mode="16k"'s data_dim
|
||||
"upsample_rates": [2],
|
||||
"upsample_kernel_sizes": [4],
|
||||
"upsample_initial_channel": 8,
|
||||
"resblock_kernel_sizes": [3],
|
||||
"resblock_dilation_sizes": [[1, 1, 1]],
|
||||
"activation": "snake",
|
||||
"snake_logscale": False,
|
||||
}
|
||||
|
||||
|
||||
def _tiny_audio_vae_config() -> Kandinsky6AudioVAEConfig:
|
||||
config = Kandinsky6AudioVAEConfig()
|
||||
config.arch_config.mode = "16k"
|
||||
config.arch_config.vocoder_config = dict(_TINY_VOCODER_CONFIG)
|
||||
return config
|
||||
|
||||
|
||||
def test_mmaudio_vae_remove_weight_norm_is_idempotent():
|
||||
torch.manual_seed(0)
|
||||
vae = MMAudioVAE(mode="16k", need_encoder=False)
|
||||
vae.remove_weight_norm()
|
||||
latents = torch.randn(1, vae.embed_dim, 4)
|
||||
mel_once = vae.decode(latents.clone())
|
||||
|
||||
# A second, redundant call must be a true no-op: it must not re-derive
|
||||
# (and thus change) already-normalized weights.
|
||||
vae.remove_weight_norm()
|
||||
mel_twice = vae.decode(latents.clone())
|
||||
|
||||
assert torch.equal(mel_once, mel_twice)
|
||||
|
||||
|
||||
def test_audio_vae_weights_are_not_renormalized_after_checkpoint_load():
|
||||
"""Simulates VAELoader's real flow: construct, then load_state_dict with
|
||||
a checkpoint's own (already-final) weights -- decode() must use those
|
||||
weights as-is, matching the diffusers reference's "already normalized,
|
||||
no-op at inference" contract, not silently re-derive a different set of
|
||||
weights from them.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
config = _tiny_audio_vae_config()
|
||||
vae = Kandinsky6AudioVAE(config)
|
||||
assert vae.vae._weights_normalized is True
|
||||
|
||||
# `load_state_dict(vae.state_dict())` mirrors VAELoader's real call
|
||||
# (`vae.load_state_dict(loaded, strict=strict_load)`) with the module's
|
||||
# own current weights standing in for "a checkpoint's own weights" --
|
||||
# exactly what happens when a real checkpoint's stored tensors are
|
||||
# loaded: no code path scales or otherwise disturbs them afterward.
|
||||
vae.load_state_dict(vae.state_dict())
|
||||
|
||||
latents = torch.randn(1, vae.vae.embed_dim, 4)
|
||||
mel_a = vae.decode(latents.clone())
|
||||
|
||||
# An explicit extra call (this wrapper's own public API) must also be a
|
||||
# no-op now, not a second renormalization of the loaded weights.
|
||||
vae.remove_weight_norm()
|
||||
mel_b = vae.decode(latents.clone())
|
||||
|
||||
assert torch.equal(mel_a, mel_b)
|
||||
@@ -0,0 +1,502 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""BF16 forward, attention dispatch, CUDA graph and cache lifetime regressions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
|
||||
Kandinsky6VideoAudioConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.cache.cache_dit_integration import (
|
||||
CacheDitConfig,
|
||||
disable_cache_on_transformer,
|
||||
enable_cache_on_transformer,
|
||||
refresh_context_on_transformer,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGBranch, CFGPolicy
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
global_force_attn_backend_context_manager,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
|
||||
Kandinsky6Attention,
|
||||
Kandinsky6Modulation,
|
||||
Kandinsky6QKNorm,
|
||||
Kandinsky6TimeEmbeddings,
|
||||
Kandinsky6Transformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.denoising import (
|
||||
Kandinsky6DenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.server_args import get_global_server_args
|
||||
from sglang.multimodal_gen.runtime.utils import precision
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
# Kandinsky6's feed-forward uses ColumnParallelLinear/RowParallelLinear, which
|
||||
# need a (size-1) TP group to construct -- matches test_kandinsky6_sr_stages.py's
|
||||
# fixture of the same name.
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29513",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
|
||||
|
||||
def _randomize(module: torch.nn.Module, seed: int) -> None:
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
with torch.no_grad():
|
||||
for param in module.parameters():
|
||||
param.copy_(torch.randn(param.shape, generator=generator) * 0.05)
|
||||
|
||||
|
||||
def test_time_embeddings_forward_under_bf16_linear_weights():
|
||||
# Minimal, pinpoint reproduction of the review comment: a bf16-weight
|
||||
# Kandinsky6TimeEmbeddings fed the (always-fp32) sinusoidal embedding must not
|
||||
# raise, and must return bf16 (the Linear weights' dtype).
|
||||
module = Kandinsky6TimeEmbeddings(model_dim=16, time_dim=8)
|
||||
_randomize(module, seed=0)
|
||||
module = module.to(torch.bfloat16)
|
||||
|
||||
out = module(torch.tensor([0.0, 500.0, 999.0]))
|
||||
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert torch.isfinite(out.float()).all()
|
||||
|
||||
|
||||
def test_modulation_forward_under_bf16_linear_weights():
|
||||
# Kandinsky6Modulation (the AdaLN projection every block/out-layer uses) has the
|
||||
# identical autocast(dtype=float32)-doesn't-cast-weights bug; fed an fp32 time
|
||||
# embedding with bf16 weights, it must not raise either.
|
||||
module = Kandinsky6Modulation(time_dim=8, model_dim=16, num_params=9)
|
||||
module = module.to(torch.bfloat16)
|
||||
|
||||
out = module(torch.randn(2, 8)) # fp32 input, as Kandinsky6TimeEmbeddings used to
|
||||
|
||||
assert out.dtype == torch.bfloat16
|
||||
assert torch.isfinite(out.float()).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA attention")
|
||||
@pytest.mark.parametrize("backend", ["FA", "TORCH_SDPA", "SAGE_ATTN_3"])
|
||||
@pytest.mark.parametrize("lengths", [(7, None), (129, None), (193, 65), (17, 129)])
|
||||
@torch.no_grad()
|
||||
def test_attention_backend_dispatch_and_repeated_forward(monkeypatch, backend, lengths):
|
||||
if backend == "FA" and torch.cuda.get_device_capability()[0] == 12:
|
||||
pytest.skip("the platform currently resolves FA to SDPA on SM12.x")
|
||||
if backend == "SAGE_ATTN_3":
|
||||
if importlib.util.find_spec("sageattn3") is None:
|
||||
pytest.skip("requires the optional SageAttention3 extension")
|
||||
if torch.cuda.get_device_capability() not in {(12, 0), (12, 1)}:
|
||||
pytest.skip("upstream SageAttention3 requires SM120 or SM121")
|
||||
monkeypatch.setattr(
|
||||
precision._mixed_precision_state,
|
||||
"state",
|
||||
precision.MixedPrecisionState(param_dtype=torch.bfloat16),
|
||||
)
|
||||
query_len, key_len = lengths
|
||||
selected = AttentionBackendEnum[backend]
|
||||
with global_force_attn_backend_context_manager(selected):
|
||||
layer = Kandinsky6Attention(
|
||||
256,
|
||||
128,
|
||||
Kandinsky6Transformer3DModel._supported_attention_backends,
|
||||
kv_dim=128 if key_len is not None else None,
|
||||
is_cross_attention=key_len is not None,
|
||||
).eval()
|
||||
assert layer.attention.backend == selected, "must not silently fall back"
|
||||
_randomize(layer, seed=419)
|
||||
layer.query_norm.weight.fill_(1)
|
||||
layer.key_norm.weight.fill_(1)
|
||||
layer.cuda().bfloat16()
|
||||
torch.manual_seed(420)
|
||||
query = torch.randn(1, query_len, 256, device="cuda", dtype=torch.bfloat16)
|
||||
key = (
|
||||
torch.randn(1, key_len, 128, device="cuda", dtype=torch.bfloat16)
|
||||
if key_len is not None
|
||||
else None
|
||||
)
|
||||
original_query = query.clone()
|
||||
original_key = key.clone() if key is not None else None
|
||||
implementation = layer.attention.attn_impl
|
||||
forward = implementation.forward
|
||||
calls = 0
|
||||
|
||||
def counted_forward(*args, **kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return forward(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(implementation, "forward", counted_forward)
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
first = layer(query, key)
|
||||
repeated = layer(query, key)
|
||||
assert calls == 2, "must execute the requested backend, not a masked bypass"
|
||||
assert first.shape == query.shape and first.dtype == query.dtype
|
||||
assert torch.isfinite(first).all()
|
||||
torch.testing.assert_close(repeated, first, rtol=0, atol=0)
|
||||
torch.testing.assert_close(query, original_query, rtol=0, atol=0)
|
||||
if key is not None:
|
||||
torch.testing.assert_close(key, original_key, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32])
|
||||
@pytest.mark.parametrize("width", [16, 64, 128, 256])
|
||||
@pytest.mark.parametrize("eps", [None, 1e-5])
|
||||
@torch.no_grad()
|
||||
def test_qk_norm_exact_output_and_parameter_storage(dtype, width, eps):
|
||||
torch.manual_seed(943)
|
||||
norm = Kandinsky6QKNorm(width, eps=eps, device="cuda", dtype=dtype)
|
||||
norm.weight.normal_()
|
||||
reference = torch.nn.RMSNorm(width, eps=eps, device="cuda", dtype=dtype)
|
||||
reference.load_state_dict(norm.state_dict())
|
||||
parameter = norm.weight
|
||||
pointer = parameter.data_ptr()
|
||||
for scale in (0.0, 0.001, 1.0, 100.0):
|
||||
x = (torch.randn(4096, width * 2, device="cuda", dtype=dtype) * scale)[:, ::2]
|
||||
expected = reference(x.float()).to(dtype)
|
||||
actual = norm(x.float()).to(dtype)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
assert norm.weight is parameter
|
||||
assert norm.weight.data_ptr() == pointer
|
||||
assert norm.weight.dtype == dtype
|
||||
|
||||
norm.cpu().cuda()
|
||||
torch.testing.assert_close(norm(x.float()).to(dtype), expected, rtol=0, atol=0)
|
||||
torch.testing.assert_close(norm.weight, reference.weight, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
def test_qk_norm_autograd_preserves_native_math(device):
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("requires CUDA")
|
||||
torch.manual_seed(944)
|
||||
norm = Kandinsky6QKNorm(128, device=device, dtype=torch.bfloat16)
|
||||
reference = torch.nn.RMSNorm(128, device=device, dtype=torch.bfloat16)
|
||||
with torch.no_grad():
|
||||
norm.weight.normal_()
|
||||
reference.load_state_dict(norm.state_dict())
|
||||
x = torch.randn(7, 128, device=device, requires_grad=True)
|
||||
ref_x = x.detach().clone().requires_grad_()
|
||||
expected = reference(ref_x)
|
||||
actual = norm(x)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
actual.sum().backward()
|
||||
expected.sum().backward()
|
||||
torch.testing.assert_close(norm.weight.grad, reference.weight.grad, rtol=0, atol=0)
|
||||
torch.testing.assert_close(x.grad, ref_x.grad, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA graphs")
|
||||
@torch.no_grad()
|
||||
def test_qk_norm_graph_replay_observes_weight_updates():
|
||||
norm = Kandinsky6QKNorm(128, device="cuda", dtype=torch.bfloat16)
|
||||
x = torch.randn(17, 128, device="cuda", dtype=torch.bfloat16).float()
|
||||
stream = torch.cuda.Stream()
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
with torch.cuda.stream(stream):
|
||||
for _ in range(3):
|
||||
norm(x)
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = norm(x).to(torch.bfloat16)
|
||||
for _ in range(2):
|
||||
norm.weight.add_(0.125)
|
||||
graph.replay()
|
||||
expected = torch.nn.functional.rms_norm(
|
||||
x, norm.normalized_shape, norm.weight, norm.eps
|
||||
).to(torch.bfloat16)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def _tiny_multimodal_arch_kwargs() -> dict:
|
||||
"""Small but structurally real T2VA config: is_multimodal=True (every released
|
||||
Kandinsky6 checkpoint), audio tower dims intentionally different from the video
|
||||
tower's to exercise both ``Kandinsky6TimeEmbeddings``/``Kandinsky6Modulation``
|
||||
instances the fused block owns."""
|
||||
return dict(
|
||||
model_dim=32,
|
||||
time_dim=16,
|
||||
ff_dim=64,
|
||||
axes_dims=(8, 4, 4),
|
||||
in_visual_dim=4,
|
||||
out_visual_dim=4,
|
||||
in_text_dim=8,
|
||||
in_text_dim2=8,
|
||||
in_audio_dim=4,
|
||||
num_text_blocks=1,
|
||||
num_visual_blocks=1,
|
||||
patch_size=(1, 1, 1),
|
||||
visual_cond=False,
|
||||
visual_token_type_num_embeddings=0,
|
||||
is_multimodal=True,
|
||||
model_dim_a=16,
|
||||
time_dim_a=16,
|
||||
ff_dim_a=32,
|
||||
axes_dims_a=(4, 2, 2),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA graphs")
|
||||
@pytest.mark.parametrize("use_cfg", [False, True])
|
||||
@torch.no_grad()
|
||||
def test_bcg_joint_cfg_output_lifetime_and_text_shape_fallback(monkeypatch, use_cfg):
|
||||
args = get_global_server_args()
|
||||
args.attention_backend = "fa"
|
||||
args.pipeline_config = Kandinsky6TI2VAPipelineConfig()
|
||||
config = Kandinsky6VideoAudioConfig()
|
||||
config.update_model_arch(_tiny_multimodal_arch_kwargs())
|
||||
with torch.device("meta"):
|
||||
dit = Kandinsky6Transformer3DModel(config, {}).eval()
|
||||
torch.manual_seed(7)
|
||||
dit.load_state_dict(
|
||||
{
|
||||
name: (torch.randn(param.shape, device="cuda") * 0.05).to(torch.bfloat16)
|
||||
for name, param in dit.named_parameters()
|
||||
},
|
||||
assign=True,
|
||||
)
|
||||
dit.post_load_weights()
|
||||
stage = Kandinsky6DenoisingStage(dit, scheduler=None)
|
||||
batch = SimpleNamespace(
|
||||
is_warmup=False,
|
||||
do_classifier_free_guidance=use_cfg,
|
||||
guidance_scale=5.0,
|
||||
cfg_normalization=0.0,
|
||||
guidance_rescale=0.0,
|
||||
)
|
||||
inputs = dict(
|
||||
transformer=dit,
|
||||
batch=batch,
|
||||
server_args=args,
|
||||
step_index=0,
|
||||
video_input=torch.randn(1, 2, 2, 2, 4, device="cuda", dtype=torch.bfloat16),
|
||||
audio_input=torch.randn(1, 5, 4, device="cuda", dtype=torch.bfloat16),
|
||||
t_expand=torch.tensor([500.0], device="cuda"),
|
||||
visual_rope_pos=[torch.arange(2, device="cuda") for _ in range(3)],
|
||||
scale_factor=(1.0, 1.0, 1.0),
|
||||
sparse_params=None,
|
||||
visual_token_type_ids=None,
|
||||
)
|
||||
|
||||
def policy(text_len):
|
||||
return CFGPolicy(
|
||||
branches=[
|
||||
CFGBranch(
|
||||
name,
|
||||
conditional,
|
||||
{
|
||||
"encoder_hidden_states": torch.randn(
|
||||
1, text_len, 8, device="cuda", dtype=torch.bfloat16
|
||||
),
|
||||
"pooled_projections": torch.randn(
|
||||
1, 8, device="cuda", dtype=torch.bfloat16
|
||||
),
|
||||
"text_rope_pos": torch.arange(text_len, device="cuda"),
|
||||
},
|
||||
)
|
||||
for name, conditional in (
|
||||
("conditional", True),
|
||||
("unconditional", False),
|
||||
)
|
||||
if conditional or use_cfg
|
||||
]
|
||||
)
|
||||
|
||||
policies = [policy(3), policy(3), policy(5)]
|
||||
references = [
|
||||
stage._predict_joint_velocity(cfg_policy=p, **inputs) for p in policies
|
||||
]
|
||||
args.enable_breakable_cuda_graph = True
|
||||
batch.is_warmup = True
|
||||
stage._predict_joint_velocity(cfg_policy=policies[0], **inputs)
|
||||
batch.is_warmup = False
|
||||
runner = stage._bcg_runners[id(dit)]
|
||||
replay = runner.replay
|
||||
replay_calls = 0
|
||||
|
||||
def record_replay(entry, kwargs):
|
||||
nonlocal replay_calls
|
||||
replay_calls += 1
|
||||
return replay(entry, kwargs)
|
||||
|
||||
monkeypatch.setattr(runner, "replay", record_replay)
|
||||
try:
|
||||
assert len(runner.entries) == 1, "eager fallback is not a capture test"
|
||||
assert all(entry.num_segments > 0 for entry in runner.entries.values())
|
||||
outputs = [
|
||||
stage._predict_joint_velocity(cfg_policy=p, **inputs) for p in policies
|
||||
]
|
||||
assert replay_calls == (4 if use_cfg else 2)
|
||||
# validate after every replay: neither CFG nor subsequent requests may
|
||||
# overwrite a previously returned video/audio tensor
|
||||
for actual, expected in zip(outputs, references, strict=True):
|
||||
for tensor, reference in zip(actual, expected, strict=True):
|
||||
torch.testing.assert_close(tensor, reference, rtol=0, atol=0)
|
||||
assert len(runner.entries) == 1, "serving must not capture unseen text lengths"
|
||||
finally:
|
||||
runner.reset()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason=(
|
||||
"the DiT's fused LayerNormScaleShift/RMSNormScaleShift requires a real "
|
||||
"CUDA/XPU tensor on any CUDA-capable host (see module docstring); bf16 "
|
||||
"numerics are only meaningful on real hardware anyway"
|
||||
),
|
||||
)
|
||||
def test_kandinsky6_dit_real_bf16_forward_does_not_crash():
|
||||
"""Real (non-mocked) forward pass of a tiny Kandinsky6Transformer3DModel whose
|
||||
weights are bf16 -- the exact configuration the SGLang PR review comment flagged
|
||||
as broken (a fp32 sinusoidal embedding feeding a bf16 Linear)."""
|
||||
device = torch.device("cuda")
|
||||
config = Kandinsky6VideoAudioConfig()
|
||||
arch_kwargs = _tiny_multimodal_arch_kwargs()
|
||||
config.update_model_arch(dict(arch_kwargs))
|
||||
dit = Kandinsky6Transformer3DModel(config, dict(arch_kwargs)).eval()
|
||||
_randomize(dit, seed=1)
|
||||
dit = dit.to(device=device, dtype=torch.bfloat16)
|
||||
|
||||
batch, duration, height, width = 1, 2, 2, 2
|
||||
text_len, audio_len = 3, 5
|
||||
hidden_states = torch.randn(
|
||||
batch, duration, height, width, 4, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
hidden_states_audio = torch.randn(
|
||||
batch, audio_len, 4, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
encoder_hidden_states = torch.randn(
|
||||
batch, text_len, 8, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
pooled_projections = torch.randn(batch, 8, dtype=torch.bfloat16, device=device)
|
||||
# Scheduler timesteps are plain fp32 (not cast to the model's weight dtype by the
|
||||
# caller), same as every real sampling loop -- exactly the dtype that used to
|
||||
# mismatch the bf16 Linear.
|
||||
timestep = torch.tensor([500.0], device=device)
|
||||
visual_rope_pos = (
|
||||
torch.arange(duration, device=device),
|
||||
torch.arange(height, device=device),
|
||||
torch.arange(width, device=device),
|
||||
)
|
||||
text_rope_pos = torch.arange(text_len, device=device)
|
||||
|
||||
video_out, audio_out = dit(
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_audio=hidden_states_audio,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
timestep=timestep,
|
||||
pooled_projections=pooled_projections,
|
||||
visual_rope_pos=visual_rope_pos,
|
||||
text_rope_pos=text_rope_pos,
|
||||
)
|
||||
|
||||
assert video_out.shape == (batch, duration, height, width, 4)
|
||||
assert audio_out.shape == (batch, audio_len, 4)
|
||||
assert video_out.dtype == torch.bfloat16
|
||||
assert audio_out.dtype == torch.bfloat16
|
||||
assert torch.isfinite(video_out.float()).all()
|
||||
assert torch.isfinite(audio_out.float()).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize("separate_cfg", [False, True])
|
||||
@torch.no_grad()
|
||||
def test_cache_dit_joint_streams_refresh_and_unmount(separate_cfg):
|
||||
config = Kandinsky6VideoAudioConfig()
|
||||
arch = _tiny_multimodal_arch_kwargs() | {"num_visual_blocks": 4}
|
||||
config.update_model_arch(arch)
|
||||
dit = Kandinsky6Transformer3DModel(config, {}).eval()
|
||||
_randomize(dit, seed=11)
|
||||
dit = dit.to(device="cuda", dtype=torch.bfloat16)
|
||||
torch.manual_seed(13)
|
||||
inputs = dict(
|
||||
hidden_states=torch.randn(1, 2, 2, 2, 4, device="cuda", dtype=torch.bfloat16),
|
||||
hidden_states_audio=torch.randn(1, 5, 4, device="cuda", dtype=torch.bfloat16),
|
||||
encoder_hidden_states=torch.randn(1, 3, 8, device="cuda", dtype=torch.bfloat16),
|
||||
pooled_projections=torch.randn(1, 8, device="cuda", dtype=torch.bfloat16),
|
||||
visual_rope_pos=[torch.arange(2, device="cuda") for _ in range(3)],
|
||||
text_rope_pos=torch.arange(3, device="cuda"),
|
||||
)
|
||||
|
||||
def run_steps():
|
||||
outputs = []
|
||||
for step in range(3):
|
||||
for branch in range(2 if separate_cfg else 1):
|
||||
outputs.append(
|
||||
dit(
|
||||
**(
|
||||
inputs
|
||||
| {
|
||||
"pooled_projections": inputs["pooled_projections"]
|
||||
+ branch
|
||||
}
|
||||
),
|
||||
timestep=torch.tensor([500.0 - step * 100], device="cuda"),
|
||||
)
|
||||
)
|
||||
return outputs
|
||||
|
||||
def assert_exact(actual, expected):
|
||||
for pair, reference in zip(actual, expected, strict=True):
|
||||
for tensor, target in zip(pair, reference, strict=True):
|
||||
torch.testing.assert_close(tensor, target, rtol=0, atol=0)
|
||||
|
||||
reference = run_steps()
|
||||
enable_cache_on_transformer(
|
||||
dit,
|
||||
CacheDitConfig(
|
||||
enabled=True,
|
||||
num_inference_steps=3,
|
||||
Fn_compute_blocks=1,
|
||||
Bn_compute_blocks=1,
|
||||
max_warmup_steps=1,
|
||||
steps_computation_mask=[1, 1, 1],
|
||||
steps_computation_policy="static",
|
||||
),
|
||||
has_separate_cfg=separate_cfg,
|
||||
)
|
||||
try:
|
||||
assert_exact(run_steps(), reference)
|
||||
cached_requests = []
|
||||
for _ in range(2):
|
||||
refresh_context_on_transformer(
|
||||
dit,
|
||||
3,
|
||||
steps_computation_mask=[1, 0, 1],
|
||||
steps_computation_policy="static",
|
||||
)
|
||||
outputs = run_steps()
|
||||
for video, audio in outputs:
|
||||
assert torch.isfinite(video).all()
|
||||
assert torch.isfinite(audio).all()
|
||||
assert dit._context_manager.get_cached_steps() == [1]
|
||||
if separate_cfg:
|
||||
assert dit._context_manager.get_cfg_cached_steps() == [1]
|
||||
cached_requests.append(outputs)
|
||||
assert_exact(*cached_requests)
|
||||
finally:
|
||||
disable_cache_on_transformer(dit)
|
||||
assert_exact(run_steps(), reference)
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import load_file, save_file
|
||||
from transformers import CLIPTextConfig, CLIPTextModel
|
||||
|
||||
from sglang.multimodal_gen.configs.models.encoders import (
|
||||
CLIPTextConfig as NativeCLIPTextConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.flux import FluxPipelineConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.krea2 import Krea2PipelineConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.stablediffusion3 import (
|
||||
StableDiffusion3PipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
|
||||
ComponentLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.text_encoder_loader import (
|
||||
TextEncoderLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.clip import (
|
||||
CLIPTextModel as NativeCLIPTextModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.clip import (
|
||||
CLIPTextModelWithProjection,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import (
|
||||
Kandinsky6TI2VAPipeline,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
|
||||
_text_encoder_max_length_is_fixed,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config_cls,fixed",
|
||||
[
|
||||
(FluxPipelineConfig, [True, False]),
|
||||
(Kandinsky6TI2VAPipelineConfig, [False, True]),
|
||||
(Krea2PipelineConfig, [False, False]),
|
||||
(StableDiffusion3PipelineConfig, [False, False, False]),
|
||||
],
|
||||
)
|
||||
def test_request_max_length_preserves_fixed_encoder_context(config_cls, fixed):
|
||||
config = config_cls()
|
||||
assert [
|
||||
_text_encoder_max_length_is_fixed(config, index) for index in range(len(fixed))
|
||||
] == fixed
|
||||
|
||||
|
||||
def test_clip_uses_shared_native_loader():
|
||||
assert "text_encoder_2" not in Kandinsky6TI2VAPipeline.component_loaders
|
||||
assert "text_encoder_2" in Kandinsky6TI2VAPipelineConfig().native_only_components
|
||||
for architecture in ("CLIPTextModel", "Qwen2_5_VLForConditionalGeneration"):
|
||||
loader = ComponentLoader.for_component_type(
|
||||
"text_encoder", "transformers", architecture
|
||||
)
|
||||
assert type(loader) is TextEncoderLoader
|
||||
|
||||
|
||||
@pytest.mark.parametrize("eos_token_id", [2, 31])
|
||||
@pytest.mark.parametrize("masked", [False, True])
|
||||
def test_clip_bare_and_prefixed_checkpoints_match(tmp_path, eos_token_id, masked):
|
||||
torch.manual_seed(42)
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
reference = (
|
||||
CLIPTextModel(
|
||||
CLIPTextConfig(
|
||||
vocab_size=32,
|
||||
hidden_size=16,
|
||||
intermediate_size=32,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=2,
|
||||
max_position_embeddings=8,
|
||||
bos_token_id=1,
|
||||
eos_token_id=eos_token_id,
|
||||
)
|
||||
)
|
||||
.to(device)
|
||||
.eval()
|
||||
)
|
||||
config = NativeCLIPTextConfig()
|
||||
config.update_model_arch(reference.config.to_dict())
|
||||
config.post_diffusers_config_update()
|
||||
weights = {name: value.cpu() for name, value in reference.state_dict().items()}
|
||||
save_file(weights, str(tmp_path / "prefixed.safetensors"))
|
||||
save_file(
|
||||
{name.removeprefix("text_model."): value for name, value in weights.items()},
|
||||
str(tmp_path / "bare.safetensors"),
|
||||
)
|
||||
ids = torch.tensor([[1, 5, 31, 31, 0], [1, 6, 7, 31, 0]], device=device)
|
||||
mask = (
|
||||
torch.tensor([[1, 1, 1, 0, 0], [1, 1, 1, 1, 0]], device=device)
|
||||
if masked
|
||||
else None
|
||||
)
|
||||
with get_parallel().override(
|
||||
tp_size=1, tp_rank=0, tp_group=None, attn_tp_size=1, attn_tp_rank=0
|
||||
):
|
||||
models = []
|
||||
for checkpoint in ("prefixed", "bare"):
|
||||
model = NativeCLIPTextModel(config).to(device).eval()
|
||||
loaded = model.load_weights(
|
||||
load_file(str(tmp_path / f"{checkpoint}.safetensors")).items()
|
||||
)
|
||||
assert loaded == set(dict(model.named_parameters()))
|
||||
models.append(model)
|
||||
for name, value in models[0].state_dict().items():
|
||||
torch.testing.assert_close(
|
||||
models[1].state_dict()[name], value, rtol=0, atol=0
|
||||
)
|
||||
with torch.no_grad():
|
||||
expected = reference(ids, attention_mask=mask, output_hidden_states=True)
|
||||
outputs = [
|
||||
model(ids, attention_mask=mask, output_hidden_states=True)
|
||||
for model in models
|
||||
]
|
||||
for prefixed, bare, reference_output in (
|
||||
(
|
||||
outputs[0].last_hidden_state,
|
||||
outputs[1].last_hidden_state,
|
||||
expected.last_hidden_state,
|
||||
),
|
||||
(outputs[0].pooler_output, outputs[1].pooler_output, expected.pooler_output),
|
||||
(
|
||||
tuple(outputs[0].hidden_states),
|
||||
tuple(outputs[1].hidden_states),
|
||||
expected.hidden_states,
|
||||
),
|
||||
):
|
||||
torch.testing.assert_close(prefixed, bare, rtol=0, atol=0)
|
||||
torch.testing.assert_close(prefixed, reference_output, rtol=2e-5, atol=2e-5)
|
||||
|
||||
|
||||
def test_bare_text_checkpoint_preserves_projection_weight():
|
||||
config = NativeCLIPTextConfig()
|
||||
config.update_model_arch(
|
||||
{
|
||||
"hidden_size": 16,
|
||||
"intermediate_size": 32,
|
||||
"num_hidden_layers": 1,
|
||||
"num_attention_heads": 2,
|
||||
"vocab_size": 32,
|
||||
"projection_dim": 8,
|
||||
}
|
||||
)
|
||||
with get_parallel().override(tp_size=1, tp_rank=0, attn_tp_size=1, attn_tp_rank=0):
|
||||
model = CLIPTextModelWithProjection(config)
|
||||
weight = torch.randn_like(model.text_projection.weight)
|
||||
assert model.load_weights([("text_projection.weight", weight)]) == {
|
||||
"text_projection.weight"
|
||||
}
|
||||
torch.testing.assert_close(model.text_projection.weight, weight, rtol=0, atol=0)
|
||||
@@ -0,0 +1,127 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""GAP 6 regression: the Kandinsky6 IT2VA conditioning-image preprocessor
|
||||
must aspect-preserving-resize-then-centre-crop, matching the diffusers
|
||||
reference's ``encode_i2va_first_frame``
|
||||
(``pipeline_kandinsky6_ti2va.py``, ``scale = min(src_h/height,
|
||||
src_w/width)``; resize to ``(src_h/scale, src_w/scale)``; centre-crop to
|
||||
``(height, width)``), not stretch-resize the whole source image to the
|
||||
target box (which distorts its aspect ratio whenever the source doesn't
|
||||
already match the target aspect ratio).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import PIL.Image
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6.image_encoding import (
|
||||
Kandinsky6ImageEncodingStage,
|
||||
)
|
||||
|
||||
|
||||
def _gradient_image(width: int, height: int) -> PIL.Image.Image:
|
||||
"""A deterministic, per-pixel-distinguishable RGB image (not a flat
|
||||
color), so resize/crop geometry is actually verifiable pixel-for-pixel.
|
||||
"""
|
||||
xs = np.linspace(0, 255, width, dtype=np.uint8)
|
||||
ys = np.linspace(0, 255, height, dtype=np.uint8)
|
||||
grid = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
grid[..., 0] = xs[None, :]
|
||||
grid[..., 1] = ys[:, None]
|
||||
grid[..., 2] = 128
|
||||
return PIL.Image.fromarray(grid, mode="RGB")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"src_h,src_w,height,width",
|
||||
[
|
||||
(1080, 1920, 512, 768), # 16:9 source, 3:2 target (audit's own case)
|
||||
(1920, 1080, 512, 768), # 9:16 source, 3:2 target (audit's own case)
|
||||
(1024, 1536, 512, 768), # already 3:2 -- matching aspect
|
||||
],
|
||||
)
|
||||
def test_cover_resize_dims_matches_diffusers_formula(src_h, src_w, height, width):
|
||||
# encode_i2va_first_frame: scale = min(src_h/height, src_w/width);
|
||||
# resize to (int(src_h/scale), int(src_w/scale)), then centre-crop.
|
||||
scale = min(src_h / height, src_w / width)
|
||||
expected = (int(src_h / scale), int(src_w / scale))
|
||||
assert (
|
||||
Kandinsky6ImageEncodingStage._cover_resize_dims(src_h, src_w, height, width)
|
||||
== expected
|
||||
)
|
||||
# The resize always covers the target box on both axes (never smaller).
|
||||
assert expected[0] >= height
|
||||
assert expected[1] >= width
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"src_w,src_h,min_mean_abs_diff",
|
||||
[
|
||||
(1920, 1080, 0.015), # 16:9 landscape source (audit's own case)
|
||||
(1080, 1920, 0.05), # 9:16 portrait source (audit's own case)
|
||||
],
|
||||
)
|
||||
def test_mismatched_aspect_pil_image_is_cropped_not_stretched(
|
||||
src_w, src_h, min_mean_abs_diff
|
||||
):
|
||||
# The model's documented default target resolution (512x768, 2:3)
|
||||
# against realistic phone/webcam source aspect ratios.
|
||||
height, width = 512, 768
|
||||
image = _gradient_image(src_w, src_h)
|
||||
|
||||
produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width)
|
||||
assert produced.shape == (1, 3, height, width)
|
||||
|
||||
# Independent oracle: the diffusers reference's own aspect-preserving
|
||||
# resize + centre-crop, computed directly via PIL, without calling any
|
||||
# of the code under test.
|
||||
scale = min(src_h / height, src_w / width)
|
||||
new_h, new_w = int(src_h / scale), int(src_w / scale)
|
||||
resized = image.resize((new_w, new_h), resample=PIL.Image.BILINEAR)
|
||||
top, left = (new_h - height) // 2, (new_w - width) // 2
|
||||
cropped = resized.crop((left, top, left + width, top + height))
|
||||
expected_np = np.array(cropped).astype(np.float32) / 255.0
|
||||
expected = torch.from_numpy(expected_np).permute(2, 0, 1) * 2.0 - 1.0
|
||||
|
||||
assert torch.allclose(produced[0], expected, atol=2.0 / 255.0)
|
||||
|
||||
# The deleted stretch-resize procedure (plain `.resize((width, height))`,
|
||||
# no crop) produces a materially different image whenever the source
|
||||
# aspect ratio doesn't already match the target -- proving this is a
|
||||
# real behavior change, not a no-op refactor.
|
||||
stretched = image.resize((width, height), resample=PIL.Image.BILINEAR)
|
||||
stretched_np = np.array(stretched).astype(np.float32) / 255.0
|
||||
stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0
|
||||
mean_abs_diff = (produced[0] - stretched_t).abs().mean().item()
|
||||
assert mean_abs_diff > min_mean_abs_diff
|
||||
|
||||
|
||||
def test_matching_aspect_pil_image_resize_only_agrees_with_stretch():
|
||||
# A sanity check on the oracle itself: when the source already matches
|
||||
# the target aspect ratio, cover-resize-then-crop and plain stretch
|
||||
# resize agree closely (no crop actually removes anything) -- so the
|
||||
# large the divergence asserted above is specifically an aspect-ratio
|
||||
# effect, not a general property of the two code paths.
|
||||
height, width = 64, 96
|
||||
src_w, src_h = 288, 192 # exactly 3:2, matching height:width == 64:96 == 2:3...
|
||||
# (use width:height == 96:64 == 3:2 to match src_w:src_h == 288:192 == 3:2)
|
||||
image = _gradient_image(src_w, src_h)
|
||||
|
||||
produced = Kandinsky6ImageEncodingStage._preprocess(image, height, width)
|
||||
stretched = image.resize((width, height), resample=PIL.Image.BILINEAR)
|
||||
stretched_np = np.array(stretched).astype(np.float32) / 255.0
|
||||
stretched_t = torch.from_numpy(stretched_np).permute(2, 0, 1) * 2.0 - 1.0
|
||||
|
||||
mean_abs_diff = (produced[0] - stretched_t).abs().mean().item()
|
||||
assert mean_abs_diff < 0.02
|
||||
|
||||
|
||||
def test_tensor_input_uses_cover_crop_geometry_too():
|
||||
height, width = 64, 96
|
||||
src_h, src_w = 108, 192
|
||||
tensor = torch.rand(1, 3, src_h, src_w) * 2.0 - 1.0 # already in [-1, 1]
|
||||
|
||||
produced = Kandinsky6ImageEncodingStage._preprocess(tensor, height, width)
|
||||
assert produced.shape[-2:] == (height, width)
|
||||
@@ -0,0 +1,271 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Exercise the declared offload groups, not just component selection."""
|
||||
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
|
||||
Kandinsky6VideoAudioConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_audio import (
|
||||
Kandinsky6AudioVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
Kandinsky6SRVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
global_force_attn_backend_context_manager,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.layerwise_offload import (
|
||||
LayerwiseOffloadableModuleMixin,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
|
||||
Kandinsky6Transformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
|
||||
Kandinsky6SRLatentUpscalerBank,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_audio import (
|
||||
Kandinsky6AudioVAE,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.server_args import get_global_server_args
|
||||
from sglang.multimodal_gen.runtime.utils import precision
|
||||
from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import (
|
||||
TINY_KVAE,
|
||||
TINY_LU_MODEL,
|
||||
)
|
||||
from sglang.srt.utils.network import get_free_port_below_ephemeral
|
||||
|
||||
pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def single_gpu(monkeypatch, default_global_server_args):
|
||||
if not model_parallel_is_initialized():
|
||||
port = get_free_port_below_ephemeral()
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT=str(port),
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
monkeypatch.setenv(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
monkeypatch.setattr(
|
||||
precision._mixed_precision_state,
|
||||
"state",
|
||||
precision.MixedPrecisionState(
|
||||
param_dtype=torch.bfloat16, reduce_dtype=torch.float32
|
||||
),
|
||||
)
|
||||
args = get_global_server_args()
|
||||
args.attention_backend = "fa"
|
||||
args.pipeline_config = Kandinsky6TI2VAPipelineConfig()
|
||||
with global_force_attn_backend_context_manager(AttentionBackendEnum.FA):
|
||||
yield
|
||||
|
||||
|
||||
def _dit(multimodal):
|
||||
config = Kandinsky6VideoAudioConfig()
|
||||
config.update_model_arch(
|
||||
dict(
|
||||
model_dim=128,
|
||||
time_dim=32,
|
||||
ff_dim=256,
|
||||
axes_dims=(32, 48, 48),
|
||||
in_visual_dim=4,
|
||||
out_visual_dim=4,
|
||||
in_text_dim=16,
|
||||
in_text_dim2=8,
|
||||
in_audio_dim=4,
|
||||
out_audio_dim=4,
|
||||
num_text_blocks=2,
|
||||
num_visual_blocks=2,
|
||||
patch_size=(1, 1, 1),
|
||||
visual_cond=False,
|
||||
visual_token_type_num_embeddings=0,
|
||||
is_multimodal=multimodal,
|
||||
model_dim_a=128,
|
||||
time_dim_a=32,
|
||||
ff_dim_a=256,
|
||||
axes_dims_a=(32, 48, 48),
|
||||
ca_rope=True,
|
||||
cross_gates=True,
|
||||
fix_modulation=True,
|
||||
)
|
||||
)
|
||||
model = Kandinsky6Transformer3DModel(config, {})
|
||||
# TP-aware linears can use uninitialized storage before checkpoint loading
|
||||
with torch.no_grad():
|
||||
for parameter in model.parameters():
|
||||
parameter.normal_(std=0.02)
|
||||
model.to(device="cuda", dtype=torch.bfloat16).eval()
|
||||
inputs = dict(
|
||||
hidden_states=torch.randn(1, 2, 3, 3, 4, device="cuda", dtype=torch.bfloat16),
|
||||
encoder_hidden_states=torch.randn(
|
||||
1, 7, 16, device="cuda", dtype=torch.bfloat16
|
||||
),
|
||||
pooled_projections=torch.randn(1, 8, device="cuda", dtype=torch.bfloat16),
|
||||
timestep=torch.tensor([500.0], device="cuda"),
|
||||
visual_rope_pos=[torch.arange(n, device="cuda") for n in (2, 3, 3)],
|
||||
text_rope_pos=torch.arange(7, device="cuda"),
|
||||
)
|
||||
if multimodal:
|
||||
inputs["hidden_states_audio"] = torch.randn(
|
||||
1, 5, 4, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
groups = (
|
||||
["video_text_transformer_blocks", "audio_text_transformer_blocks"]
|
||||
if multimodal
|
||||
else ["text_transformer_blocks"]
|
||||
)
|
||||
return model, lambda: model(**inputs), groups + ["visual_transformer_blocks"]
|
||||
|
||||
|
||||
def _audio_vae():
|
||||
config = Kandinsky6AudioVAEConfig()
|
||||
config.arch_config.mode = "16k"
|
||||
config.arch_config.vocoder_config = dict(
|
||||
resblock="1",
|
||||
num_mels=80,
|
||||
upsample_rates=[2],
|
||||
upsample_kernel_sizes=[4],
|
||||
upsample_initial_channel=8,
|
||||
resblock_kernel_sizes=[3],
|
||||
resblock_dilation_sizes=[[1, 1, 1]],
|
||||
activation="snake",
|
||||
snake_logscale=False,
|
||||
)
|
||||
model = Kandinsky6AudioVAE(config).to("cuda").eval()
|
||||
mel = torch.randn(1, 80, 8, device="cuda")
|
||||
|
||||
def forward():
|
||||
latent = model.vae.encode(mel).mean
|
||||
decoded = model.decode(latent)
|
||||
return latent, decoded, model.vocode(decoded)
|
||||
|
||||
groups = [f"vae.encoder.down.{i}.block" for i in range(3)]
|
||||
groups += [f"vae.decoder.up.{i}.block" for i in range(3)]
|
||||
return model, forward, groups + ["vocoder.ups.0", "vocoder.resblocks"]
|
||||
|
||||
|
||||
def _sr_vae():
|
||||
config = Kandinsky6SRVAEConfig()
|
||||
config.update_model_arch(
|
||||
dict(
|
||||
vae_type="video-kvae",
|
||||
encoder_config=dict(TINY_KVAE, in_channels=3),
|
||||
decoder_config=dict(TINY_KVAE, out_ch=3),
|
||||
)
|
||||
)
|
||||
model = Kandinsky6SRVAE(config).to("cuda").eval()
|
||||
pixels = torch.randn(1, 3, 33, 32, 32, device="cuda")
|
||||
|
||||
def forward():
|
||||
latent, segments = model.encode(pixels)
|
||||
assert segments == [17, 16]
|
||||
return latent, model.decode(latent).sample
|
||||
|
||||
groups = [f"encoder.down.{i}.block" for i in range(5)]
|
||||
groups += [f"decoder.up.{i}.block" for i in range(5)]
|
||||
return model, forward, groups
|
||||
|
||||
|
||||
def _upscaler():
|
||||
model = (
|
||||
Kandinsky6SRLatentUpscalerBank(
|
||||
[{"target_scale": scale, "model": dict(TINY_LU_MODEL)} for scale in (2, 4)]
|
||||
)
|
||||
.to("cuda")
|
||||
.eval()
|
||||
)
|
||||
latent = torch.randn(1, 4, 3, 3, 3, device="cuda")
|
||||
|
||||
def forward():
|
||||
# exercise both branches of every loaded entry, including width changes
|
||||
return tuple(
|
||||
entry(latent, scale) for entry in model._models for scale in (2, 4)
|
||||
)
|
||||
|
||||
groups = [
|
||||
f"_models.{index}.{group}"
|
||||
for index in range(2)
|
||||
for group in (
|
||||
"pre_blocks",
|
||||
"mid_blocks",
|
||||
"post_blocks",
|
||||
"x2_branch.adapter",
|
||||
"x2_branch.mid_blocks",
|
||||
"x2_branch.blocks",
|
||||
)
|
||||
]
|
||||
return model, forward, groups
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"component", ["video-dit", "joint-dit", "audio-vae", "sr-vae", "upscaler"]
|
||||
)
|
||||
@pytest.mark.parametrize("resident_layers", [0, 1])
|
||||
@torch.no_grad()
|
||||
def test_layerwise_groups_preserve_repeated_forwards(component, resident_layers):
|
||||
torch.manual_seed(13)
|
||||
if component.endswith("dit"):
|
||||
model, forward, groups = _dit(component == "joint-dit")
|
||||
elif component == "audio-vae":
|
||||
model, forward, groups = _audio_vae()
|
||||
elif component == "sr-vae":
|
||||
model, forward, groups = _sr_vae()
|
||||
else:
|
||||
model, forward, groups = _upscaler()
|
||||
assert isinstance(model, LayerwiseOffloadableModuleMixin)
|
||||
assert set(groups) <= set(model.layer_names)
|
||||
assert model.layerwise_offload_dit_group_enabled == component.endswith("dit")
|
||||
state = {name: value.cpu().clone() for name, value in model.state_dict().items()}
|
||||
with (
|
||||
torch.inference_mode(),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
):
|
||||
expected = forward()
|
||||
model.configure_layerwise_offload(
|
||||
SimpleNamespace(
|
||||
performance_mode="speed",
|
||||
pin_cpu_memory=True,
|
||||
layerwise_tuning_for=lambda *args, **kwargs: (
|
||||
1,
|
||||
resident_layers,
|
||||
"leading",
|
||||
"forward",
|
||||
),
|
||||
)
|
||||
)
|
||||
assert {
|
||||
manager.layers_attr_str for manager in model.layerwise_offload_managers
|
||||
} == set(groups)
|
||||
for _ in range(2):
|
||||
model.prepare_for_next_req()
|
||||
torch.testing.assert_close(forward(), expected, rtol=0, atol=0)
|
||||
for manager in model.layerwise_offload_managers:
|
||||
assert manager._last_forwarded_layer == manager.num_layers - 1
|
||||
manager.release_all()
|
||||
model.disable_offload()
|
||||
torch.testing.assert_close(forward(), expected, rtol=0, atol=0)
|
||||
for name, value in model.state_dict().items():
|
||||
torch.testing.assert_close(value.cpu(), state[name], rtol=0, atol=0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(pytest.main([__file__, *sys.argv[1:]]))
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Load real PEFT safetensors through Kandinsky6's native adapter mapping."""
|
||||
|
||||
from collections import defaultdict
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import (
|
||||
Kandinsky6TI2VAPipeline,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"prefix", ["", "transformer.", "base_model.model.transformer."]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"source,target",
|
||||
[
|
||||
(
|
||||
"visual_transformer_blocks.0.video_dec_block.feed_forward.net.0.proj",
|
||||
"visual_transformer_blocks.0.videoT.feed_forward.mlp.fc_in",
|
||||
),
|
||||
(
|
||||
"visual_transformer_blocks.0.audio_dec_block.feed_forward.net.2",
|
||||
"visual_transformer_blocks.0.audioT.feed_forward.mlp.fc_out",
|
||||
),
|
||||
(
|
||||
"video_text_transformer_blocks.1.feed_forward.net.0.proj",
|
||||
"video_text_transformer_blocks.1.feed_forward.mlp.fc_in",
|
||||
),
|
||||
(
|
||||
"audio_text_transformer_blocks.1.attn.to_query",
|
||||
"audio_text_transformer_blocks.1.self_attention.to_query",
|
||||
),
|
||||
(
|
||||
"video_time_embeddings.timestep_embedder.linear_1",
|
||||
"video_time_embeddings.in_layer",
|
||||
),
|
||||
(
|
||||
"audio_time_embeddings.timestep_embedder.linear_2",
|
||||
"audio_time_embeddings.out_layer",
|
||||
),
|
||||
(
|
||||
"visual_transformer_blocks.0.videoT.self_attention.to_key",
|
||||
"visual_transformer_blocks.0.videoT.self_attention.to_key",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_peft_adapter_maps_to_native_layers(tmp_path, prefix, source, target):
|
||||
pipeline = object.__new__(Kandinsky6TI2VAPipeline)
|
||||
pipeline.server_args = SimpleNamespace(
|
||||
pipeline_config=Kandinsky6TI2VAPipelineConfig(), lora_path=None
|
||||
)
|
||||
pipeline.modules = {
|
||||
"transformer": SimpleNamespace(
|
||||
param_names_mapping={}, lora_param_names_mapping={}
|
||||
)
|
||||
}
|
||||
pipeline.device = torch.device("cpu")
|
||||
pipeline.lora_adapters = defaultdict(dict)
|
||||
pipeline.loaded_adapter_paths = {}
|
||||
pipeline.loaded_adapter_alphas = {}
|
||||
values = {
|
||||
"lora_A": torch.arange(16, dtype=torch.float32).reshape(2, 8),
|
||||
"lora_B": torch.arange(24, dtype=torch.float32).reshape(12, 2),
|
||||
"alpha": torch.tensor(4.0),
|
||||
}
|
||||
path = tmp_path / "adapter.safetensors"
|
||||
save_file(
|
||||
{
|
||||
f"{prefix}{source}.{key}.default.weight"
|
||||
if key != "alpha"
|
||||
else f"{prefix}{source}.alpha": value
|
||||
for key, value in values.items()
|
||||
},
|
||||
path,
|
||||
)
|
||||
pipeline.load_lora_adapter(str(path), "probe", rank=0)
|
||||
actual = pipeline.lora_adapters["probe"]
|
||||
assert set(actual) == {f"{target}.{key}" for key in values}
|
||||
for key, expected in values.items():
|
||||
torch.testing.assert_close(actual[f"{target}.{key}"], expected, rtol=0, atol=0)
|
||||
@@ -0,0 +1,346 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Pipeline wiring, residency precision and joint video/audio denoising."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6 import (
|
||||
Kandinsky6TI2VASamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed.cfg_policy import CFGPolicy
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers import (
|
||||
component_residency_strategies,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
|
||||
Kandinsky6RoPE1D,
|
||||
Kandinsky6RoPE3D,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
|
||||
PiflowScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import (
|
||||
Kandinsky6TI2VAPipeline,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages import InputValidationStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6 import (
|
||||
Kandinsky6AudioDecodingStage,
|
||||
Kandinsky6DecodingStage,
|
||||
Kandinsky6DenoisingStage,
|
||||
Kandinsky6ImageEncodingStage,
|
||||
Kandinsky6LatentPreparationStage,
|
||||
denoising,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.text_encoding import (
|
||||
TextEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.timestep_preparation import (
|
||||
TimestepPreparationStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import get_global_server_args
|
||||
|
||||
EXPECTED_REQUIRED_MODULES = [
|
||||
"scheduler",
|
||||
"text_encoder",
|
||||
"text_encoder_2",
|
||||
"tokenizer",
|
||||
"tokenizer_2",
|
||||
"transformer",
|
||||
"vae",
|
||||
"audio_vae",
|
||||
]
|
||||
|
||||
EXPECTED_STAGE_ORDER = [
|
||||
("input_validation_stage", InputValidationStage),
|
||||
("text_encoding_stage", TextEncodingStage),
|
||||
("timestep_preparation_stage", TimestepPreparationStage),
|
||||
("latent_preparation_stage", Kandinsky6LatentPreparationStage),
|
||||
("image_encoding_stage", Kandinsky6ImageEncodingStage),
|
||||
("denoising_stage", Kandinsky6DenoisingStage),
|
||||
# Audio decodes before video, matching the diffusers reference -- see
|
||||
# Kandinsky6TI2VAPipeline.create_pipeline_stages's comment.
|
||||
("audio_decoding_stage", Kandinsky6AudioDecodingStage),
|
||||
("decoding_stage", Kandinsky6DecodingStage),
|
||||
]
|
||||
|
||||
|
||||
def _make_pipeline() -> Kandinsky6TI2VAPipeline:
|
||||
pipeline = object.__new__(Kandinsky6TI2VAPipeline)
|
||||
pipeline.modules = {
|
||||
name: MagicMock(name=name)
|
||||
for name in Kandinsky6TI2VAPipeline._required_config_modules
|
||||
}
|
||||
pipeline._stages = []
|
||||
pipeline._stage_name_mapping = {}
|
||||
pipeline._disagg_role = RoleType.MONOLITHIC
|
||||
return pipeline
|
||||
|
||||
|
||||
def test_pipeline_identity_and_required_modules():
|
||||
assert Kandinsky6TI2VAPipeline.pipeline_name == "Kandinsky6TI2VAPipeline"
|
||||
assert Kandinsky6TI2VAPipeline.is_video_pipeline is True
|
||||
assert Kandinsky6TI2VAPipeline._required_config_modules == EXPECTED_REQUIRED_MODULES
|
||||
# No separate "vocoder" module -- audio_vae bundles the mel-VAE decoder
|
||||
# and the BigVGAN vocoder in one checkpoint component.
|
||||
assert "vocoder" not in Kandinsky6TI2VAPipeline._required_config_modules
|
||||
|
||||
|
||||
def test_disagg_role_rejects_non_monolithic_deployment():
|
||||
pipeline = object.__new__(Kandinsky6TI2VAPipeline)
|
||||
|
||||
with pytest.raises(ValueError, match="monolithic"):
|
||||
pipeline.validate_disagg_role(RoleType.ENCODER)
|
||||
with pytest.raises(ValueError, match="monolithic"):
|
||||
pipeline.validate_disagg_role(RoleType.DENOISER)
|
||||
|
||||
# MONOLITHIC is the only accepted role -- no exception raised.
|
||||
pipeline.validate_disagg_role(RoleType.MONOLITHIC)
|
||||
|
||||
|
||||
def test_create_pipeline_stages_produces_documented_order():
|
||||
pipeline = _make_pipeline()
|
||||
server_args = MagicMock()
|
||||
|
||||
Kandinsky6TI2VAPipeline.create_pipeline_stages(pipeline, server_args)
|
||||
|
||||
actual_types = [type(stage) for stage in pipeline.stages]
|
||||
expected_types = [stage_cls for _, stage_cls in EXPECTED_STAGE_ORDER]
|
||||
assert actual_types == expected_types
|
||||
|
||||
actual_names = list(pipeline._stage_name_mapping.keys())
|
||||
expected_names = [name for name, _ in EXPECTED_STAGE_ORDER]
|
||||
assert actual_names == expected_names
|
||||
|
||||
# get_module wiring: each stage received the mocked module it asked for.
|
||||
denoising_stage = pipeline._stage_name_mapping["denoising_stage"]
|
||||
assert denoising_stage.transformer is pipeline.modules["transformer"]
|
||||
|
||||
audio_decoding_stage = pipeline._stage_name_mapping["audio_decoding_stage"]
|
||||
assert audio_decoding_stage.audio_vae is pipeline.modules["audio_vae"]
|
||||
|
||||
decoding_stage = pipeline._stage_name_mapping["decoding_stage"]
|
||||
assert decoding_stage.vae is pipeline.modules["vae"]
|
||||
|
||||
image_encoding_stage = pipeline._stage_name_mapping["image_encoding_stage"]
|
||||
assert image_encoding_stage.vae is pipeline.modules["vae"]
|
||||
|
||||
|
||||
def _pro_distill_scheduler() -> PiflowScheduler:
|
||||
"""The scheduler of kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers."""
|
||||
scheduler = PiflowScheduler(
|
||||
nfe=None,
|
||||
n_grid=10,
|
||||
shift=5.0,
|
||||
eps=1e-6,
|
||||
final_step_size_scale=0.5,
|
||||
num_policy_substeps=128,
|
||||
)
|
||||
scheduler.set_timesteps(10)
|
||||
return scheduler
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stage_name", ["denoising_stage", "audio_decoding_stage"])
|
||||
@pytest.mark.parametrize("precision", ["bf16", "fp32"])
|
||||
def test_component_residency_preserves_loaded_precision(stage_name, precision):
|
||||
pipeline = _make_pipeline()
|
||||
args = MagicMock()
|
||||
args.pipeline_config.dit_precision = precision
|
||||
args.pipeline_config.audio_vae_precision = precision
|
||||
args.component_precisions = {"transformer": precision, "audio_vae": precision}
|
||||
pipeline.create_pipeline_stages(args)
|
||||
use = pipeline._stage_name_mapping[stage_name].component_uses(args)[0]
|
||||
|
||||
# loaders own precision; movement must not downcast RoPE or audio buffers
|
||||
assert use.target_dtype is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA transfers")
|
||||
def test_denoising_offload_roundtrip_preserves_fp32_rope(monkeypatch):
|
||||
device = torch.device("cuda", torch.cuda.current_device())
|
||||
monkeypatch.setattr(
|
||||
component_residency_strategies, "get_local_torch_device", lambda: device
|
||||
)
|
||||
transformer = torch.nn.ModuleDict(
|
||||
{
|
||||
"projection": torch.nn.Linear(8, 8, dtype=torch.bfloat16),
|
||||
"rope_1d": Kandinsky6RoPE1D(8),
|
||||
"rope_3d": Kandinsky6RoPE3D((4, 4, 4)),
|
||||
}
|
||||
).to(device)
|
||||
buffers = {name: value.clone() for name, value in transformer.named_buffers()}
|
||||
stage = Kandinsky6DenoisingStage(transformer, scheduler=MagicMock())
|
||||
use = stage.component_uses(get_global_server_args())[0]
|
||||
|
||||
for _ in range(2):
|
||||
transformer.cpu()
|
||||
component_residency_strategies._module_to_local_device(
|
||||
transformer, dtype=use.target_dtype
|
||||
)
|
||||
assert next(transformer.parameters()).dtype == torch.bfloat16
|
||||
for name, value in transformer.named_buffers():
|
||||
assert value.device == device
|
||||
torch.testing.assert_close(value, buffers[name], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_parallel_cfg_uses_serial_arithmetic_for_video_and_audio(monkeypatch):
|
||||
config = Kandinsky6TI2VAPipelineConfig()
|
||||
batch = SimpleNamespace(
|
||||
do_classifier_free_guidance=True,
|
||||
guidance_scale=5.0,
|
||||
cfg_normalization=0.0,
|
||||
guidance_rescale=0.0,
|
||||
)
|
||||
policy = config.cfg_policy.build(batch, {}, {}, {})
|
||||
positive = torch.linspace(-1, 1, 256, dtype=torch.bfloat16)
|
||||
negative = torch.linspace(-0.8, 0.9, 256, dtype=torch.bfloat16)
|
||||
predictions = [(positive, negative), (negative, positive)]
|
||||
expected = policy.combine(predictions, batch, 5.0, config)
|
||||
legacy = CFGPolicy().combine(predictions, batch, 5.0, config, cfg_parallel=True)
|
||||
assert any(not torch.equal(a, b) for a, b in zip(expected, legacy, strict=True))
|
||||
monkeypatch.setattr(denoising, "get_classifier_free_guidance_world_size", lambda: 2)
|
||||
monkeypatch.setattr(denoising, "run_cfg_parallel", lambda *_args: predictions)
|
||||
monkeypatch.setattr(
|
||||
denoising,
|
||||
"run_two_branch_cfg_parallel",
|
||||
lambda *_args: pytest.fail("legacy all-reduce changes BF16 rounding"),
|
||||
)
|
||||
stage = Kandinsky6DenoisingStage(MagicMock(), scheduler=MagicMock())
|
||||
actual = stage._predict_joint_velocity(
|
||||
transformer=stage.transformer,
|
||||
cfg_policy=policy,
|
||||
batch=batch,
|
||||
server_args=SimpleNamespace(pipeline_config=config, enable_cfg_parallel=True),
|
||||
step_index=0,
|
||||
video_input=positive,
|
||||
audio_input=negative,
|
||||
t_expand=torch.zeros(1),
|
||||
visual_rope_pos=[],
|
||||
scale_factor=(1.0, 1.0, 1.0),
|
||||
sparse_params=None,
|
||||
visual_token_type_ids=None,
|
||||
)
|
||||
for result, reference in zip(actual, expected, strict=True):
|
||||
torch.testing.assert_close(result, reference, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"guidance_scale", [5.0, 0.5, 1.001, float("nan"), float("inf")]
|
||||
)
|
||||
def test_denoising_rejects_guidance_other_than_one_with_a_piflow_scheduler(
|
||||
guidance_scale,
|
||||
):
|
||||
"""The distilled checkpoint's ``PiflowScheduler`` has no classifier-free guidance,
|
||||
so the run must not silently ignore a guidance the caller asked for; the
|
||||
Diffusers pipeline raises the same way ("PiflowScheduler requires
|
||||
guidance_weight=1.0")."""
|
||||
scheduler = _pro_distill_scheduler()
|
||||
batch = SimpleNamespace(
|
||||
timesteps=scheduler.timesteps,
|
||||
latents=torch.zeros(1, 1, 1, 1, 1),
|
||||
audio_latents=torch.zeros(1, 1, 1),
|
||||
scheduler=scheduler,
|
||||
guidance_scale=guidance_scale,
|
||||
)
|
||||
stage = Kandinsky6DenoisingStage(transformer=MagicMock(), scheduler=scheduler)
|
||||
|
||||
with pytest.raises(ValueError, match=r"requires guidance_scale=1\.0"):
|
||||
stage.forward(batch, MagicMock())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("height,width", [(32, 32), (512, 768), (768, 512)])
|
||||
def test_distilled_denoising_calls_the_dit_once_per_step_without_guidance(
|
||||
monkeypatch, height, width
|
||||
):
|
||||
"""Pro-distill: 10 steps at guidance 1.0. The DiT head is ``n_grid`` (= 10) times as
|
||||
wide as the latents, the pi-Flow step folds it back, and the returned latents keep
|
||||
the latent width. The negative prompt is never fed to the DiT."""
|
||||
monkeypatch.setattr(
|
||||
denoising, "get_local_torch_device", lambda: torch.device("cpu")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
get_global_server_args().pipeline_config, "dit_precision", "fp32"
|
||||
)
|
||||
video_channels, audio_channels, n_grid, text_len = 4, 6, 10, 7
|
||||
text_lens = []
|
||||
|
||||
class _FakeDiT(torch.nn.Module):
|
||||
"""A real ``nn.Module`` (not a bare function): ``Kandinsky6DenoisingStage``
|
||||
now subclasses the shared ``DenoisingStage``, whose ``__init__`` walks
|
||||
``self.transformer.modules()`` to infer the attention backend -- a plain
|
||||
callable has no ``.modules()``."""
|
||||
|
||||
def forward(self, **kwargs):
|
||||
assert tuple(kwargs["scale_factor"]) == (1.0, 2.0, 2.0)
|
||||
text_lens.append(kwargs["encoder_hidden_states"].shape[1])
|
||||
video, audio = kwargs["hidden_states"], kwargs["hidden_states_audio"]
|
||||
video_velocity = torch.randn(*video.shape[:-1], n_grid * video_channels)
|
||||
audio_velocity = torch.randn(*audio.shape[:-1], n_grid * audio_channels)
|
||||
assert kwargs["return_dict"] is False
|
||||
return video_velocity, audio_velocity
|
||||
|
||||
fake_dit = _FakeDiT()
|
||||
|
||||
scheduler = _pro_distill_scheduler()
|
||||
# visual_cond: the DiT input carries the latents, a condition copy and a mask
|
||||
video = torch.randn(1, 2, 4, 4, 2 * video_channels + 1)
|
||||
batch = SimpleNamespace(
|
||||
timesteps=scheduler.timesteps,
|
||||
latents=video,
|
||||
audio_latents=torch.randn(1, 5, audio_channels),
|
||||
scheduler=scheduler,
|
||||
guidance_scale=1.0,
|
||||
do_classifier_free_guidance=False,
|
||||
extra={},
|
||||
prompt_embeds=[torch.randn(1, text_len, 8)],
|
||||
negative_prompt_embeds=[torch.randn(1, text_len + 2, 8)],
|
||||
pooled_embeds=[torch.randn(1, 3)],
|
||||
prompt_attention_mask=[torch.ones(1, text_len)],
|
||||
height=height,
|
||||
width=width,
|
||||
image_latent=None,
|
||||
is_warmup=True,
|
||||
sampling_params=Kandinsky6TI2VASamplingParams(quality="lossless"),
|
||||
)
|
||||
arch = SimpleNamespace(
|
||||
attention_engine="auto",
|
||||
in_visual_dim=video_channels,
|
||||
patch_size=(1, 2, 2),
|
||||
scale_factor=(1.0, 2.0, 2.0),
|
||||
)
|
||||
pipeline_config = SimpleNamespace(
|
||||
dit_config=SimpleNamespace(arch_config=arch),
|
||||
dit_precision="fp32",
|
||||
vae_config=SimpleNamespace(
|
||||
arch_config=SimpleNamespace(spatial_compression_ratio=8)
|
||||
),
|
||||
get_pos_prompt_embeds=lambda batch: batch.prompt_embeds[0],
|
||||
# CFG is disabled in this test (guidance_scale=1.0,
|
||||
# do_classifier_free_guidance=False), so the policy never needs a
|
||||
# negative branch, but Kandinsky6DenoisingStage still reads
|
||||
# `cfg_policy` / `get_classifier_free_guidance_scale` off the config
|
||||
# unconditionally to build the (single-branch) CFGPolicy.
|
||||
cfg_policy=CFGPolicy(),
|
||||
get_classifier_free_guidance_scale=lambda _batch, guidance_scale: (
|
||||
guidance_scale
|
||||
),
|
||||
)
|
||||
server_args = SimpleNamespace(
|
||||
pipeline_config=pipeline_config, enable_cfg_parallel=False
|
||||
)
|
||||
stage = Kandinsky6DenoisingStage(transformer=fake_dit, scheduler=scheduler)
|
||||
|
||||
result = stage.forward(batch, server_args)
|
||||
|
||||
assert text_lens == [text_len] * 10
|
||||
assert tuple(result.latents.shape) == (1, 2, 4, 4, video_channels)
|
||||
assert tuple(result.audio_latents.shape) == (1, 5, audio_channels)
|
||||
assert torch.isfinite(result.latents).all()
|
||||
assert torch.isfinite(result.audio_latents).all()
|
||||
@@ -0,0 +1,207 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Kandinsky 6 model routing and request defaults, without Hub access."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen import registry
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6 import (
|
||||
Kandinsky6TI2VADistilledSamplingParams,
|
||||
Kandinsky6TI2VASamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_pipeline import (
|
||||
Kandinsky6TI2VAPipeline,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import (
|
||||
Kandinsky6SRPipeline,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
|
||||
SFT = "kandinskylab/Kandinsky-6.0-Pro-sft-5s-Diffusers"
|
||||
DISTILL = "kandinskylab/Kandinsky-6.0-Pro-distill-5s-Diffusers"
|
||||
TI2VA = (
|
||||
Kandinsky6TI2VAPipeline,
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
Kandinsky6TI2VASamplingParams,
|
||||
)
|
||||
DISTILLED = (
|
||||
Kandinsky6TI2VAPipeline,
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
Kandinsky6TI2VADistilledSamplingParams,
|
||||
)
|
||||
SR = (Kandinsky6SRPipeline, Kandinsky6SRPipelineConfig, Kandinsky6SRSamplingParams)
|
||||
|
||||
ROUTES = [
|
||||
("kandinskylab/Kandinsky-6.0-Pro-5s-Diffusers", TI2VA),
|
||||
(SFT, TI2VA),
|
||||
(DISTILL, DISTILLED),
|
||||
("kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers", SR),
|
||||
("kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", SR),
|
||||
("/models/Kandinsky-6.0-Pro-5s-Diffusers", TI2VA),
|
||||
("/models/Kandinsky-6.0-Pro-sft-5s-Diffusers", TI2VA),
|
||||
("/models/Kandinsky-6.0-Pro-distill-5s-Diffusers/", DISTILLED),
|
||||
("/models/Kandinsky-6.0-VSR-5s-Diffusers", SR),
|
||||
("/models/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", SR),
|
||||
("/distill_runs/Kandinsky-6.0-Pro-sft-5s-Diffusers", TI2VA),
|
||||
(
|
||||
"/hub/models--kandinskylab--Kandinsky-6.0-Pro-sft-5s-Diffusers/snapshots/4f2a",
|
||||
TI2VA,
|
||||
),
|
||||
(
|
||||
"/hub/models--kandinskylab--Kandinsky-6.0-Pro-distill-5s-Diffusers/snapshots/4f2a",
|
||||
DISTILLED,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_registry_cache():
|
||||
registry.get_model_info.cache_clear()
|
||||
registry._get_config_info.cache_clear()
|
||||
yield
|
||||
registry.get_model_info.cache_clear()
|
||||
registry._get_config_info.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_path,expected", ROUTES)
|
||||
def test_model_routing(monkeypatch, model_path, expected):
|
||||
pipeline, _, _ = expected
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"maybe_download_model_index",
|
||||
lambda _: {"_class_name": pipeline.pipeline_name},
|
||||
)
|
||||
assert registry.is_registered_diffusion_model_path(model_path)
|
||||
info = registry.get_model_info(model_path, backend="sglang")
|
||||
assert (
|
||||
info.pipeline_cls,
|
||||
info.pipeline_config_cls,
|
||||
info.sampling_param_cls,
|
||||
) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name,expected",
|
||||
[
|
||||
(SFT, TI2VA),
|
||||
(DISTILL, DISTILLED),
|
||||
("/models/kandinsky6-sr-native", SR),
|
||||
("/models/Kandinsky_6.0_SR_x2", SR),
|
||||
("Kandinsky6TI2VAPipeline", TI2VA),
|
||||
("Kandinsky6T2VAPipeline", TI2VA),
|
||||
("Kandinsky6SRPipeline", SR),
|
||||
("/distill_runs/Kandinsky-6.0-Pro-sft-5s-Diffusers", TI2VA),
|
||||
("kandinskylab/Kandinsky-6.0-VSR-distilled2steps-5s-Diffusers", SR),
|
||||
],
|
||||
)
|
||||
def test_detectors_do_not_overlap(name, expected):
|
||||
claimed = [
|
||||
registry._CONFIG_REGISTRY[model_id]
|
||||
for model_id, detector in registry._MODEL_NAME_DETECTORS
|
||||
if detector(name.lower())
|
||||
]
|
||||
assert len(claimed) == 1
|
||||
assert (claimed[0].pipeline_config_cls, claimed[0].sampling_param_cls) == expected[
|
||||
1:
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("expected", [TI2VA, SR])
|
||||
def test_local_bundle_routing(tmp_path, expected):
|
||||
pipeline, _, _ = expected
|
||||
for component in ("transformer", "vae"):
|
||||
(tmp_path / component).mkdir()
|
||||
(tmp_path / "model_index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"_class_name": pipeline.pipeline_name,
|
||||
"_diffusers_version": "0.37.0",
|
||||
"transformer": ["diffusers", "X"],
|
||||
"vae": ["diffusers", "Y"],
|
||||
}
|
||||
)
|
||||
)
|
||||
info = registry.get_model_info(str(tmp_path), backend="sglang")
|
||||
assert (
|
||||
info.pipeline_cls,
|
||||
info.pipeline_config_cls,
|
||||
info.sampling_param_cls,
|
||||
) == expected
|
||||
if expected == TI2VA:
|
||||
info = registry.get_model_info(
|
||||
str(tmp_path),
|
||||
backend="sglang",
|
||||
model_id="Kandinsky-6.0-Pro-distill-5s-Diffusers",
|
||||
)
|
||||
assert info.sampling_param_cls is Kandinsky6TI2VADistilledSamplingParams
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_path,overrides,steps,guidance",
|
||||
[
|
||||
(SFT, {}, 50, 5.0),
|
||||
(DISTILL, {}, 10, 1.0),
|
||||
("/models/Kandinsky-6.0-Pro-distill-5s-Diffusers", {}, 10, 1.0),
|
||||
(DISTILL, {"num_inference_steps": 8}, 8, 1.0),
|
||||
(DISTILL, {"num_inference_steps": 50}, 50, 1.0),
|
||||
(SFT, {"num_inference_steps": 25, "guidance_scale": 3.0}, 25, 3.0),
|
||||
],
|
||||
)
|
||||
def test_request_defaults_and_overrides(
|
||||
monkeypatch, model_path, overrides, steps, guidance
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
registry,
|
||||
"maybe_download_model_index",
|
||||
lambda _: {"_class_name": "Kandinsky6TI2VAPipeline"},
|
||||
)
|
||||
args = SimpleNamespace(
|
||||
backend="sglang",
|
||||
model_id=None,
|
||||
pipeline_class_name=None,
|
||||
pipeline_config=Kandinsky6TI2VAPipelineConfig(),
|
||||
output_path=None,
|
||||
comfyui_mode=True,
|
||||
num_gpus=1,
|
||||
)
|
||||
params = SamplingParams.from_user_sampling_params_args(
|
||||
model_path, args, prompt="a cat", **overrides
|
||||
)
|
||||
assert (params.num_inference_steps, params.guidance_scale) == (steps, guidance)
|
||||
assert (params.height, params.width, params.num_frames, params.fps) == (
|
||||
512,
|
||||
768,
|
||||
121,
|
||||
24,
|
||||
)
|
||||
assert Req(sampling_params=params).do_classifier_free_guidance == (guidance > 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params_cls",
|
||||
[Kandinsky6TI2VASamplingParams, Kandinsky6TI2VADistilledSamplingParams],
|
||||
)
|
||||
@pytest.mark.parametrize("num_gpus", [1, 2, 3, 4, 8])
|
||||
def test_request_frame_count_is_independent_of_gpu_count(params_cls, num_gpus):
|
||||
params = params_cls()
|
||||
params._adjust(
|
||||
SimpleNamespace(
|
||||
pipeline_config=Kandinsky6TI2VAPipelineConfig(),
|
||||
output_path=None,
|
||||
comfyui_mode=True,
|
||||
num_gpus=num_gpus,
|
||||
)
|
||||
)
|
||||
assert params.num_frames == 121
|
||||
@@ -0,0 +1,123 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.dits import kandinsky6
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
@pytest.mark.parametrize("shape", [(1, 1, 1, 1), (2, 3, 5, 7), (1, 31, 32, 48)])
|
||||
def test_3d_rope_broadcast_matches_materialized_axes(device, shape, monkeypatch):
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("requires CUDA")
|
||||
rope = kandinsky6.Kandinsky6RoPE3D((64, 32, 32)).to(device)
|
||||
batch, frames, height, width = shape
|
||||
pos = [torch.arange(n * 2, device=device)[::2] for n in shape[1:]]
|
||||
args = torch.cat(
|
||||
[
|
||||
rope.args_0[pos[0]]
|
||||
.view(1, frames, 1, 1, -1)
|
||||
.repeat(batch, 1, height, width, 1),
|
||||
(rope.args_1[pos[1]] / 2.0)
|
||||
.view(1, 1, height, 1, -1)
|
||||
.repeat(batch, frames, 1, width, 1),
|
||||
(rope.args_2[pos[2]] / 3.16)
|
||||
.view(1, 1, 1, width, -1)
|
||||
.repeat(batch, frames, height, 1, 1),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
cosine, sine = args.cos(), args.sin()
|
||||
expected = torch.stack([cosine, -sine, sine, cosine], dim=-1)
|
||||
expected = expected.view(*expected.shape[:-1], 2, 2).unsqueeze(-4)
|
||||
|
||||
def no_repeat(*args, **kwargs):
|
||||
pytest.fail("RoPE axes should broadcast without materializing repeats")
|
||||
|
||||
monkeypatch.setattr(torch.Tensor, "repeat", no_repeat)
|
||||
actual = rope(shape, pos, (1.0, 2.0, 3.16))
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
assert actual.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("strided", [False, True])
|
||||
def test_rotary_cpu_and_autograd(strided):
|
||||
x = torch.randn(2, 17, 3, 64, requires_grad=True)
|
||||
if strided:
|
||||
x = x[:, ::2]
|
||||
rope = torch.randn(1, x.shape[1], 1, 32, 2, 2)
|
||||
expected = (rope * x.reshape(2, x.shape[1], 3, 32, 1, 2)).sum(-1).reshape_as(x)
|
||||
actual = kandinsky6._apply_rotary(x, rope)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
torch.autograd.grad(actual.sum(), x, retain_graph=True)[0],
|
||||
torch.autograd.grad(expected.sum(), x)[0],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or torch.version.hip is not None,
|
||||
reason="CUDA required",
|
||||
)
|
||||
def test_rotary_uses_fused_kernel(monkeypatch):
|
||||
original = kandinsky6.apply_matrix_rope
|
||||
calls = []
|
||||
|
||||
def record(x, rope, dtype=None):
|
||||
calls.append(x.shape)
|
||||
return original(x, rope, dtype)
|
||||
|
||||
monkeypatch.setattr(kandinsky6, "apply_matrix_rope", record)
|
||||
x = torch.randn(1, 257, 8, 128, dtype=torch.bfloat16, device="cuda")
|
||||
rope = torch.randn(1, 257, 1, 64, 2, 2, device="cuda")
|
||||
expected = (
|
||||
(rope * x.float().reshape(1, 257, 8, 64, 1, 2))
|
||||
.sum(-1)
|
||||
.reshape_as(x)
|
||||
.to(x.dtype)
|
||||
)
|
||||
assert torch.equal(kandinsky6._apply_rotary(x, rope), expected)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("strided", [False, True])
|
||||
def test_rotary_cast_preserves_rounding_and_autograd(device, dtype, strided):
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("requires CUDA")
|
||||
x = torch.randn(2, 17, 3, 64, device=device, requires_grad=True)
|
||||
if strided:
|
||||
x = x[:, ::2]
|
||||
rope = torch.randn(1, x.shape[1], 1, 32, 2, 2, device=device)
|
||||
rounded = x.to(dtype).float().reshape(2, x.shape[1], 3, 32, 1, 2)
|
||||
expected = (rope * rounded).sum(-1).reshape_as(x).to(dtype)
|
||||
actual = kandinsky6._apply_rotary(x, rope, dtype)
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
torch.autograd.grad(actual.sum(), x, retain_graph=True)[0],
|
||||
torch.autograd.grad(expected.sum(), x)[0],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
with torch.no_grad():
|
||||
torch.testing.assert_close(
|
||||
kandinsky6._apply_rotary(x, rope, dtype), expected, rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or torch.version.hip is not None,
|
||||
reason="CUDA required",
|
||||
)
|
||||
def test_gate_sum_preserves_fp32_intermediates():
|
||||
x = torch.randn(2, 257, 128, dtype=torch.bfloat16, device="cuda")
|
||||
update = torch.randn_like(x)
|
||||
gate = torch.randn(2, 1, 1152, dtype=x.dtype, device=x.device)[..., 256:384]
|
||||
expected = (x.float() + gate.float() * update.float()).to(x.dtype)
|
||||
assert not torch.equal(expected, x + gate * update)
|
||||
assert torch.equal(kandinsky6._apply_gate_sum(x, update, gate), expected)
|
||||
@@ -0,0 +1,166 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Compare native Ulysses/Ring joint video-audio forwards with a single GPU."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6 import (
|
||||
Kandinsky6VideoAudioConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6 import (
|
||||
Kandinsky6TI2VAPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
global_force_attn_backend,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6 import (
|
||||
Kandinsky6Transformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.server_args import set_global_server_args
|
||||
from sglang.multimodal_gen.runtime.utils.precision import set_mixed_precision_policy
|
||||
from sglang.srt.utils.network import get_free_port_below_ephemeral
|
||||
|
||||
|
||||
def _forward(rank, world_size, ring, tp_size, port, output_dir, model_dim):
|
||||
os.environ.update(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT=str(port),
|
||||
RANK=str(rank),
|
||||
LOCAL_RANK=str(rank),
|
||||
WORLD_SIZE=str(world_size),
|
||||
)
|
||||
torch.cuda.set_device(rank)
|
||||
set_mixed_precision_policy(torch.bfloat16, torch.float32)
|
||||
global_force_attn_backend(AttentionBackendEnum.FA)
|
||||
set_global_server_args(
|
||||
SimpleNamespace(
|
||||
attention_backend="fa",
|
||||
attention_backend_config=None,
|
||||
kv_gather_degree=1,
|
||||
sp_split_auto=False,
|
||||
pipeline_config=Kandinsky6TI2VAPipelineConfig(),
|
||||
)
|
||||
)
|
||||
maybe_init_distributed_environment_and_model_parallel(
|
||||
tp_size=tp_size,
|
||||
sp_size=world_size // tp_size,
|
||||
ulysses_degree=1 if ring else world_size // tp_size,
|
||||
ring_degree=world_size // tp_size if ring else 1,
|
||||
)
|
||||
config = Kandinsky6VideoAudioConfig()
|
||||
config.update_model_arch(
|
||||
dict(
|
||||
model_dim=model_dim,
|
||||
time_dim=32,
|
||||
ff_dim=512,
|
||||
axes_dims=(32, 48, 48),
|
||||
in_visual_dim=4,
|
||||
out_visual_dim=4,
|
||||
in_text_dim=16,
|
||||
in_text_dim2=8,
|
||||
in_audio_dim=4,
|
||||
out_audio_dim=4,
|
||||
num_text_blocks=1,
|
||||
num_visual_blocks=2,
|
||||
patch_size=(1, 1, 1),
|
||||
visual_cond=False,
|
||||
visual_token_type_num_embeddings=0,
|
||||
is_multimodal=True,
|
||||
model_dim_a=model_dim,
|
||||
time_dim_a=32,
|
||||
ff_dim_a=256,
|
||||
axes_dims_a=(32, 48, 48),
|
||||
ca_rope=True,
|
||||
cross_gates=True,
|
||||
fix_modulation=True,
|
||||
)
|
||||
)
|
||||
model = Kandinsky6Transformer3DModel(config, {}).eval()
|
||||
generator = torch.Generator().manual_seed(7)
|
||||
weights_path = Path(output_dir) / "weights.pt"
|
||||
with torch.no_grad():
|
||||
if world_size == 1:
|
||||
for parameter in model.parameters():
|
||||
parameter.copy_(
|
||||
torch.randn(parameter.shape, generator=generator) * 0.02
|
||||
)
|
||||
torch.save(model.state_dict(), weights_path)
|
||||
else:
|
||||
weights = torch.load(weights_path, weights_only=True)
|
||||
for name, parameter in model.named_parameters():
|
||||
weight_loader = parameter.__dict__.get("weight_loader")
|
||||
if weight_loader is None:
|
||||
parameter.copy_(weights[name])
|
||||
else:
|
||||
weight_loader(parameter, weights[name])
|
||||
model = model.to(device=rank, dtype=torch.bfloat16)
|
||||
|
||||
for frames in (2, 3):
|
||||
# odd visual sequence length exercises masked tail padding; audio stays odd
|
||||
generator.manual_seed(frames)
|
||||
|
||||
def tensor(*shape):
|
||||
return torch.randn(*shape, generator=generator).to(
|
||||
device=rank, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
with (
|
||||
torch.no_grad(),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
):
|
||||
video, audio = model(
|
||||
hidden_states=tensor(1, frames, 3, 3, 4),
|
||||
hidden_states_audio=tensor(1, 5, 4),
|
||||
encoder_hidden_states=tensor(1, 7, 16),
|
||||
pooled_projections=tensor(1, 8),
|
||||
timestep=torch.tensor([500.0], device=rank),
|
||||
visual_rope_pos=[torch.arange(n, device=rank) for n in (frames, 3, 3)],
|
||||
text_rope_pos=torch.arange(7, device=rank),
|
||||
)
|
||||
assert video.shape == (1, frames, 3, 3, 4)
|
||||
assert audio.shape == (1, 5, 4)
|
||||
assert torch.isfinite(video).all() and torch.isfinite(audio).all()
|
||||
path = Path(output_dir) / f"{frames}.pt"
|
||||
if world_size == 1:
|
||||
torch.save((video.cpu(), audio.cpu()), path)
|
||||
else:
|
||||
reference = torch.load(path, weights_only=True, map_location=f"cuda:{rank}")
|
||||
for actual, expected in zip((video, audio), reference, strict=True):
|
||||
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.002)
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def _launch(world_size, ring, output_dir, tp_size=1, model_dim=256):
|
||||
port = get_free_port_below_ephemeral()
|
||||
mp.spawn(
|
||||
_forward,
|
||||
args=(world_size, ring, tp_size, port, output_dir, model_dim),
|
||||
nprocs=world_size,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="requires two CUDA GPUs")
|
||||
def test_joint_video_audio_sequence_parallel(tmp_path):
|
||||
_launch(1, False, str(tmp_path))
|
||||
_launch(2, False, str(tmp_path))
|
||||
_launch(2, True, str(tmp_path))
|
||||
_launch(2, False, str(tmp_path), tp_size=2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(torch.cuda.device_count() < 4, reason="requires four CUDA GPUs")
|
||||
def test_joint_video_audio_tensor_sequence_parallel(tmp_path):
|
||||
_launch(1, False, str(tmp_path), model_dim=512)
|
||||
for tp_size in (1, 2, 4):
|
||||
_launch(4, False, str(tmp_path), tp_size=tp_size, model_dim=512)
|
||||
_launch(4, True, str(tmp_path), tp_size=2, model_dim=512)
|
||||
@@ -0,0 +1,305 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Kandinsky 6 SR checkpoint configuration and CLI request validation."""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRArchConfig,
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
Kandinsky6SRVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6 import (
|
||||
Kandinsky6TI2VASamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
|
||||
TINY_DIT = dict(
|
||||
in_visual_dim=4,
|
||||
in_text_dim=8,
|
||||
in_text_dim2=8,
|
||||
time_dim=16,
|
||||
out_visual_dim=4,
|
||||
patch_size=[1, 2, 2],
|
||||
model_dim=32,
|
||||
ff_dim=64,
|
||||
num_text_blocks=0,
|
||||
num_visual_blocks=2,
|
||||
axes_dims=[8, 4, 4],
|
||||
visual_cond=False,
|
||||
instruct_type="noise",
|
||||
attention_params={"512": {"type": "flash"}},
|
||||
use_text=False,
|
||||
)
|
||||
TINY_PIFLOW = dict(
|
||||
nfe=2,
|
||||
num_policy_substeps=8,
|
||||
final_step_size_scale=0.5,
|
||||
shift=5.0,
|
||||
n_grid=3,
|
||||
eps=1e-6,
|
||||
)
|
||||
SR_PARAMS = dict(
|
||||
scale_factor={"512": [1.0, 1.0, 1.0]},
|
||||
visual_size=[512],
|
||||
scheduler_scale=5.0,
|
||||
lq_noise_scale=0.7,
|
||||
lq_noise_type="ddpm",
|
||||
lq_channel_noise_scale=0.0,
|
||||
cap_noise_timestep=False,
|
||||
fps=24,
|
||||
)
|
||||
|
||||
|
||||
def flat_config(**changes):
|
||||
"""A converted ``transformer/config.json`` (pi-Flow DX head) with ``changes`` applied."""
|
||||
flat = dict(TINY_DIT, _class_name="Kandinsky6SRTransformer3DModel", n_grid=3)
|
||||
flat.update(
|
||||
piflow_nfe=2,
|
||||
piflow_num_policy_substeps=8,
|
||||
piflow_final_step_size_scale=0.5,
|
||||
piflow_shift=5.0,
|
||||
piflow_eps=1e-6,
|
||||
attribute_overrides={"instruct_type": "noise"},
|
||||
sr_visual_size=[512],
|
||||
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
|
||||
sr_scheduler_scale=5.0,
|
||||
sr_lq_noise_scale=0.7,
|
||||
sr_lq_noise_type="ddpm",
|
||||
sr_lq_channel_noise_scale=0.0,
|
||||
sr_cap_noise_timestep=False,
|
||||
sr_fps=24,
|
||||
)
|
||||
flat.update(changes)
|
||||
return flat
|
||||
|
||||
|
||||
def official_config(**changes):
|
||||
"""An official ``transformer/config.json``: constructor kwargs, ``sr_params``, the TOTAL DX head
|
||||
width in ``out_visual_dim`` (``base * n_grid``) and no ``n_grid`` / ``piflow_*`` keys.
|
||||
"""
|
||||
official = {
|
||||
key: value for key, value in TINY_DIT.items() if key not in ("out_visual_dim",)
|
||||
}
|
||||
official.update(
|
||||
out_visual_dim=TINY_DIT["out_visual_dim"] * TINY_PIFLOW["n_grid"],
|
||||
attribute_overrides=None,
|
||||
sr_params=dict(SR_PARAMS),
|
||||
)
|
||||
official.update(changes)
|
||||
return official
|
||||
|
||||
|
||||
def parse(flat):
|
||||
config = Kandinsky6SRDitConfig()
|
||||
config.update_model_arch(flat)
|
||||
return config.arch_config
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# DiT arch config
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_arch_config_parses_an_official_config_and_maps_checkpoint_names():
|
||||
arch = parse(official_config())
|
||||
assert arch.n_grid == 1 and not arch.is_piflow # pi-Flow lives in the scheduler
|
||||
assert arch.out_visual_dim == 12 and arch.attribute_overrides == {}
|
||||
assert arch.patch_size == (1, 2, 2) and arch.axes_dims == (8, 4, 4)
|
||||
assert (arch.hidden_size, arch.num_attention_heads) == (32, 2)
|
||||
assert arch.sr_scale_factor == {"512": [1.0, 1.0, 1.0]} # JSON keys stay strings
|
||||
assert (arch.sr_visual_size, arch.sr_scheduler_scale, arch.sr_fps) == (
|
||||
[512],
|
||||
5.0,
|
||||
24,
|
||||
)
|
||||
|
||||
map_name = get_param_names_mapping(arch.param_names_mapping)
|
||||
assert map_name("pooled_bias")[0] == "pooled_bias"
|
||||
assert (
|
||||
map_name("visual_transformer_blocks.3.feed_forward.net.0.proj.weight")[0]
|
||||
== "visual_transformer_blocks.3.feed_forward.mlp.fc_in.weight"
|
||||
)
|
||||
assert (
|
||||
map_name("visual_transformer_blocks.0.feed_forward.net.2.weight")[0]
|
||||
== "visual_transformer_blocks.0.feed_forward.mlp.fc_out.weight"
|
||||
)
|
||||
assert (
|
||||
map_name("time_embeddings.timestep_embedder.linear_1.weight")[0]
|
||||
== "time_embeddings.in_layer.weight"
|
||||
)
|
||||
# the attention output projection is not a feed-forward layer and keeps its name
|
||||
assert (
|
||||
map_name("visual_transformer_blocks.0.self_attention.out_layer.weight")[0]
|
||||
== "visual_transformer_blocks.0.self_attention.out_layer.weight"
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_sr_params_keys_are_rejected():
|
||||
with pytest.raises(ValueError, match="sr_params has unsupported keys"):
|
||||
parse(official_config(sr_params={"fps": 24, "new_knob": 1}))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"changes,error,flag",
|
||||
[
|
||||
({"use_text": True}, NotImplementedError, "use_text"),
|
||||
({"use_adapter": True}, NotImplementedError, "use_adapter"),
|
||||
({"use_lq_modulation": True}, NotImplementedError, "use_lq_modulation"),
|
||||
({"surprise_key": 1}, ValueError, "surprise_key"),
|
||||
({"attribute_overrides": {"n_grid": 2}}, ValueError, "n_grid"),
|
||||
({"attribute_overrides": {"instruct_type": "bogus"}}, ValueError, "bogus"),
|
||||
({"piflow_eps": None}, ValueError, "piflow_eps"),
|
||||
({"patch_size": [2, 2, 2]}, ValueError, "patch_size"),
|
||||
({"sr_lq_noise_type": "gauss"}, ValueError, "gauss"),
|
||||
],
|
||||
)
|
||||
def test_unsupported_or_unknown_config_fails_loudly_naming_the_key(
|
||||
changes, error, flag
|
||||
):
|
||||
"""A config the port cannot honor must never build a silently different model:
|
||||
unsupported flags raise NotImplementedError naming the flag, unknown / inconsistent
|
||||
keys raise ValueError (``update_model_arch`` alone would park unknown keys in
|
||||
``extra_attrs``)."""
|
||||
with pytest.raises(error, match=flag):
|
||||
parse(flat_config(**changes))
|
||||
|
||||
|
||||
def test_sparse_attention_request_is_detected_from_attention_params():
|
||||
assert parse(flat_config()).requested_sparse_attention() is None
|
||||
nabla = flat_config(attention_params={"512": {"type": "nabla", "P": 0.9}})
|
||||
assert parse(nabla).requested_sparse_attention() == "nabla"
|
||||
# an override can switch it off again
|
||||
overridden = flat_config(
|
||||
attention_params={"512": {"type": "nabla"}},
|
||||
attribute_overrides={"attention_params": {"512": {"type": "flash"}}},
|
||||
)
|
||||
assert parse(overridden).requested_sparse_attention() is None
|
||||
|
||||
|
||||
def test_default_arch_config_is_valid_and_text_free():
|
||||
"""Registry / ``PipelineConfig.from_kwargs`` instantiate the default config."""
|
||||
arch = Kandinsky6SRArchConfig()
|
||||
assert arch.use_text is False and not arch.is_piflow and arch.n_grid == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# VAE config
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_vae_config_parses_bundle_config_and_rejects_bad_ones():
|
||||
config = Kandinsky6SRVAEConfig()
|
||||
config.update_model_arch(
|
||||
dict(
|
||||
vae_type="video-kvae",
|
||||
encoder_config={"ch": 8},
|
||||
decoder_config={"ch": 8},
|
||||
scaling_factor=0.53,
|
||||
spatial_factor=16,
|
||||
temporal_factor=4,
|
||||
)
|
||||
)
|
||||
arch = config.arch_config
|
||||
assert arch.scaling_factor == 0.53
|
||||
assert (arch.spatial_compression_ratio, arch.temporal_compression_ratio) == (16, 4)
|
||||
assert config.get_vae_scale_factor() == 16
|
||||
|
||||
with pytest.raises(ValueError, match="video-kvae"):
|
||||
Kandinsky6SRVAEConfig().update_model_arch({"vae_type": "wan"})
|
||||
with pytest.raises(ValueError, match="surprise"):
|
||||
Kandinsky6SRVAEConfig().update_model_arch({"surprise": 1})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Pipeline config and sampling params
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_pipeline_config_is_text_free_monolithic_and_strict():
|
||||
config = Kandinsky6SRPipelineConfig()
|
||||
config.check_pipeline_config() # the four per-encoder tuples must stay consistent
|
||||
assert config.text_encoder_configs == ()
|
||||
assert config.supports_disaggregation() is False
|
||||
# These components must load strictly and re-raise (no diffusers AutoModel fallback).
|
||||
assert {"transformer", "vae", "latent_upscaler"} <= set(
|
||||
config.native_only_components
|
||||
)
|
||||
|
||||
|
||||
def test_sampling_params_validate_the_sr_knobs():
|
||||
params = Kandinsky6SRSamplingParams(video_path="clip.mp4")
|
||||
assert (params.sr_resolution_scale, params.num_inference_steps) == (2.25, 4)
|
||||
assert (params.sr_tiles_batch_size, params.sr_tile_min_overlap) == (1, 0.20)
|
||||
assert params.seed == 42 and params.guidance_scale == 1.0
|
||||
|
||||
for bad in (
|
||||
{"sr_resolution_scale": 3},
|
||||
{"num_inference_steps": 0},
|
||||
{"sr_tiles_batch_size": 0},
|
||||
{"sr_tile_min_overlap": 1.0},
|
||||
{"sr_target_resize_mode": "crop"},
|
||||
{"sr_target_resolution": "big"},
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
Kandinsky6SRSamplingParams(video_path="clip.mp4", **bad)
|
||||
for good in ("hd", "fullhd", "2k", "1280x720", "none"):
|
||||
Kandinsky6SRSamplingParams(video_path="clip.mp4", sr_target_resolution=good)
|
||||
|
||||
|
||||
def test_sr_flags_reach_the_sampling_params_through_the_generate_cli():
|
||||
"""New SamplingParams fields are only CLI flags if the base ``add_cli_args`` declares
|
||||
them; ``get_cli_args`` then filters by the resolved subclass' dataclass fields."""
|
||||
parser = argparse.ArgumentParser()
|
||||
SamplingParams.add_cli_args(parser)
|
||||
args = parser.parse_args(
|
||||
[
|
||||
"--video-path",
|
||||
"clip.mp4",
|
||||
"--sr-resolution-scale",
|
||||
"4",
|
||||
"--num-inference-steps",
|
||||
"3",
|
||||
"--sr-tiles-batch-size",
|
||||
"2",
|
||||
"--sr-tile-min-overlap",
|
||||
"0.3",
|
||||
"--sr-target-resolution",
|
||||
"fullhd",
|
||||
"--sr-target-resize-mode",
|
||||
"exact",
|
||||
]
|
||||
)
|
||||
kwargs = Kandinsky6SRSamplingParams.get_cli_args(args)
|
||||
params = Kandinsky6SRSamplingParams(**kwargs)
|
||||
assert (
|
||||
params.video_path == ["clip.mp4"] and params.source_video_path() == "clip.mp4"
|
||||
)
|
||||
assert (params.sr_resolution_scale, params.num_inference_steps) == (4.0, 3)
|
||||
assert (params.sr_tiles_batch_size, params.sr_tile_min_overlap) == (2, 0.3)
|
||||
assert (params.sr_target_resolution, params.sr_target_resize_mode) == (
|
||||
"fullhd",
|
||||
"exact",
|
||||
)
|
||||
# the sr_* knobs are ignored by other models' params instead of choking on them; the
|
||||
# shared num_inference_steps flag, unlike those, is a core field every model gets.
|
||||
assert "sr_tiles_batch_size" not in Kandinsky6TI2VASamplingParams.get_cli_args(args)
|
||||
assert "num_inference_steps" in Kandinsky6TI2VASamplingParams.get_cli_args(args)
|
||||
|
||||
|
||||
def test_request_needs_exactly_one_video_and_names_its_output_after_it():
|
||||
with pytest.raises(ValueError, match="--video-path"):
|
||||
Kandinsky6SRSamplingParams()._validate_with_pipeline_config(
|
||||
Kandinsky6SRPipelineConfig()
|
||||
)
|
||||
with pytest.raises(ValueError, match="exactly one video"):
|
||||
Kandinsky6SRSamplingParams(video_path=["a.mp4", "b.mp4"]).source_video_path()
|
||||
|
||||
params = Kandinsky6SRSamplingParams(video_path="/data/my clip.mp4", prompt=" ")
|
||||
params._set_output_file_name()
|
||||
assert re.fullmatch(r"my_clip_sr2\.25x_\d{8}-\d{6}\.mp4", params.output_file_name)
|
||||
@@ -0,0 +1,210 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""SR decode parallelism must preserve causal state and request isolation."""
|
||||
|
||||
import os
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
Kandinsky6SRVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
get_decode_parallel_group_coordinator,
|
||||
get_dp_rank,
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.parallel_conv import (
|
||||
SpatialParallelConv3d,
|
||||
disable_spatial_parallel_decode,
|
||||
gather_and_trim_height,
|
||||
split_height_for_parallel_decode,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import (
|
||||
Kandinsky6SRVAE,
|
||||
_ChunkedConv3d,
|
||||
_SpatialChunkedConv3d,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
|
||||
Kandinsky6SRDecodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_DENOISED_KEY,
|
||||
SR_TILES_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
decode_chunks,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import set_global_server_args
|
||||
from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import TINY_KVAE
|
||||
from sglang.srt.utils.network import get_free_port_below_ephemeral
|
||||
|
||||
|
||||
def _decode(rank, world_size, tp_size, dp_size, port, spatial):
|
||||
os.environ.update(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT=str(port),
|
||||
RANK=str(rank),
|
||||
LOCAL_RANK=str(rank),
|
||||
WORLD_SIZE=str(world_size),
|
||||
)
|
||||
torch.cuda.set_device(rank)
|
||||
sp_size = world_size // tp_size // dp_size
|
||||
maybe_init_distributed_environment_and_model_parallel(
|
||||
tp_size=tp_size, sp_size=sp_size, ulysses_degree=sp_size, dp_size=dp_size
|
||||
)
|
||||
group = get_decode_parallel_group_coordinator()
|
||||
assert group.world_size == world_size // dp_size
|
||||
spatial_active = spatial and group.world_size > 1
|
||||
if spatial_active:
|
||||
torch.manual_seed(3 + get_dp_rank())
|
||||
conv = _ChunkedConv3d(2, 3, 3, padding=(0, 1, 1)).to(rank)
|
||||
parallel_conv = _SpatialChunkedConv3d.from_conv(conv)
|
||||
assert parallel_conv.weight is conv.weight
|
||||
x = torch.randn(1, 2, 9, 9, 5, device=rank)
|
||||
with torch.inference_mode():
|
||||
expected_conv = conv(x)
|
||||
shard, height = split_height_for_parallel_decode(
|
||||
x, x.shape[-2], group.world_size, group.rank_in_group
|
||||
)
|
||||
# force different temporal chunk counts on uneven height shards
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae._MAX_CONV_NUMEL",
|
||||
200,
|
||||
):
|
||||
actual_conv = gather_and_trim_height(parallel_conv(shard), height)
|
||||
torch.testing.assert_close(actual_conv, expected_conv)
|
||||
with disable_spatial_parallel_decode():
|
||||
torch.testing.assert_close(parallel_conv(x), expected_conv)
|
||||
config = Kandinsky6SRVAEConfig()
|
||||
config.update_model_arch(
|
||||
dict(
|
||||
encoder_config=dict(TINY_KVAE, in_channels=3),
|
||||
decoder_config=dict(TINY_KVAE, out_ch=3),
|
||||
scaling_factor=0.5,
|
||||
spatial_factor=2 ** (len(TINY_KVAE["ch_mult"]) - 1),
|
||||
)
|
||||
)
|
||||
torch.manual_seed(17)
|
||||
reference_vae = Kandinsky6SRVAE(config).eval().to(device=rank, dtype=torch.bfloat16)
|
||||
config = replace(
|
||||
config, use_parallel_decode=spatial, parallel_decode_mode="spatial_shard"
|
||||
)
|
||||
vae = Kandinsky6SRVAE(config).eval().to(device=rank, dtype=torch.bfloat16)
|
||||
vae.load_state_dict(reference_vae.state_dict())
|
||||
assert (
|
||||
any(isinstance(m, SpatialParallelConv3d) for m in vae.decoder.modules())
|
||||
== spatial_active
|
||||
)
|
||||
pipeline_config = Kandinsky6SRPipelineConfig(vae_config=config)
|
||||
args = SimpleNamespace(pipeline_config=pipeline_config, component_precisions={})
|
||||
set_global_server_args(args)
|
||||
stage = Kandinsky6SRDecodeStage(vae)
|
||||
stage.server_args = args
|
||||
|
||||
with torch.inference_mode():
|
||||
for offload in (False, True):
|
||||
if offload:
|
||||
vae.configure_layerwise_offload(
|
||||
SimpleNamespace(
|
||||
performance_mode="speed",
|
||||
pin_cpu_memory=True,
|
||||
layerwise_tuning_for=lambda *args, **kwargs: (
|
||||
1,
|
||||
0,
|
||||
"leading",
|
||||
"forward",
|
||||
),
|
||||
)
|
||||
)
|
||||
first = None
|
||||
# Uneven batches, idle ranks, two temporal segments, then a repeat.
|
||||
for sizes, frames, height in (
|
||||
((2, 2, 1), 9, 5),
|
||||
((1,), 1, 1),
|
||||
((2, 1), 5, 4),
|
||||
((2, 2, 1), 9, 5),
|
||||
):
|
||||
generator = torch.Generator().manual_seed(7 + get_dp_rank())
|
||||
chunks = [
|
||||
torch.randn(size, frames, height, 3, 4, generator=generator)
|
||||
for size in sizes
|
||||
]
|
||||
reference_vae.prepare_for_next_req()
|
||||
expected = decode_chunks(
|
||||
chunks,
|
||||
reference_vae,
|
||||
scaling_factor=reference_vae.scaling_factor,
|
||||
device=torch.device("cuda", rank),
|
||||
)
|
||||
for parallel in (False, True):
|
||||
config.use_parallel_tiling = parallel
|
||||
vae.prepare_for_next_req()
|
||||
request = Req(sampling_params=Kandinsky6SRSamplingParams())
|
||||
request.extra[SR_DENOISED_KEY] = chunks
|
||||
with (
|
||||
patch.object(vae, "decode", wraps=vae.decode) as decode,
|
||||
patch.object(
|
||||
vae.decoder, "forward", wraps=vae.decoder.forward
|
||||
) as decoder,
|
||||
):
|
||||
actual = stage.forward(request, args).extra[SR_TILES_KEY]
|
||||
count = len(
|
||||
range(group.rank_in_group, sum(sizes), group.world_size)
|
||||
)
|
||||
assert decode.call_count == (
|
||||
count
|
||||
if parallel and not spatial_active and sum(sizes) > 1
|
||||
else sum(sizes)
|
||||
)
|
||||
local_height = height
|
||||
if spatial_active and height >= group.world_size:
|
||||
local_height = height // group.world_size + (
|
||||
group.rank_in_group < height % group.world_size
|
||||
)
|
||||
assert all(
|
||||
call.args[0].shape[-2] == local_height
|
||||
for call in decoder.call_args_list
|
||||
)
|
||||
assert len(actual) == len(expected)
|
||||
for tile, reference in zip(actual, expected):
|
||||
assert tile.device.type == "cpu" and tile.dtype == torch.uint8
|
||||
torch.testing.assert_close(
|
||||
tile, reference, rtol=0, atol=2 if spatial_active else 0
|
||||
)
|
||||
if frames == 9:
|
||||
if first is None:
|
||||
first = actual
|
||||
else:
|
||||
for tile, reference in zip(actual, first):
|
||||
torch.testing.assert_close(tile, reference, rtol=0, atol=0)
|
||||
if offload:
|
||||
vae.disable_offload()
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"world_size,tp_size,dp_size",
|
||||
[(1, 1, 1), (2, 2, 1), (2, 1, 1), (4, 2, 1), (4, 1, 2)],
|
||||
)
|
||||
@pytest.mark.parametrize("spatial", [False, True], ids=["tiles", "spatial"])
|
||||
def test_sr_parallel_decode(world_size, tp_size, dp_size, spatial):
|
||||
if torch.cuda.device_count() < world_size:
|
||||
pytest.skip(f"requires {world_size} CUDA GPUs")
|
||||
mp.spawn(
|
||||
_decode,
|
||||
args=(world_size, tp_size, dp_size, get_free_port_below_ephemeral(), spatial),
|
||||
nprocs=world_size,
|
||||
)
|
||||
@@ -0,0 +1,290 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Check SR prompt admission and request fields without running a GPU scheduler.
|
||||
|
||||
JSON uses TestClient; multipart tests call the parsed endpoint directly. The
|
||||
full-checkpoint test_server_kandinsky6_sr.py also exercises actual file uploads.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai import video_api
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import VideoResponse
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.utils import (
|
||||
resolve_sampling_params_cls,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.video_api import (
|
||||
_build_video_sampling_params,
|
||||
_multipart_video_extras,
|
||||
)
|
||||
|
||||
_SR_EXTRA_FIELDS = frozenset(
|
||||
{
|
||||
"sr_resolution_scale",
|
||||
"sr_tiles_batch_size",
|
||||
"sr_tile_min_overlap",
|
||||
"sr_target_resolution",
|
||||
"sr_target_resize_mode",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _fake_server_args(**overrides) -> SimpleNamespace:
|
||||
base = dict(
|
||||
pipeline_config=Kandinsky6SRPipelineConfig(),
|
||||
pipeline_class_name="Kandinsky6SRPipeline",
|
||||
backend="auto",
|
||||
model_id=None,
|
||||
model_path="kandinskylab/Kandinsky-6.0-VSR-5s-Diffusers",
|
||||
served_model_name="kandinsky6-sr-test",
|
||||
input_save_path=os.path.expanduser(
|
||||
"~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/uploads"
|
||||
),
|
||||
output_path=os.path.expanduser(
|
||||
"~/.cache/k6_agent_sg-sr-gaps-fix/http_request_test/output"
|
||||
),
|
||||
attention_backend_config={},
|
||||
enable_trace=False,
|
||||
comfyui_mode=False,
|
||||
num_gpus=1,
|
||||
)
|
||||
base.update(overrides)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
|
||||
def _patched_global_server_args(server_args):
|
||||
return patch(
|
||||
"sglang.multimodal_gen.runtime.server_args.server_args._global_server_args",
|
||||
server_args,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The two declared pieces (SamplingParams-level)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_prompt_optional_defaults_false_and_sr_overrides_it():
|
||||
"""A generic pipeline keeps requiring ``prompt``; only the SR params opt out."""
|
||||
assert SamplingParams.prompt_optional is False
|
||||
assert Kandinsky6SRSamplingParams.prompt_optional is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"sampling_params_cls, status",
|
||||
[(Kandinsky6SRSamplingParams, 200), (SamplingParams, 400)],
|
||||
)
|
||||
def test_json_prompt_requirement_matches_pipeline(
|
||||
tmp_path, sampling_params_cls, status
|
||||
):
|
||||
app = FastAPI()
|
||||
app.include_router(video_api.router)
|
||||
server_args = _fake_server_args(
|
||||
input_save_path=str(tmp_path / "uploads"), output_path=str(tmp_path / "outputs")
|
||||
)
|
||||
dispatch = AsyncMock()
|
||||
with (
|
||||
_patched_global_server_args(server_args),
|
||||
patch.object(
|
||||
video_api, "resolve_sampling_params_cls", return_value=sampling_params_cls
|
||||
),
|
||||
patch.object(video_api, "_dispatch_job_async", dispatch),
|
||||
TestClient(app) as client,
|
||||
):
|
||||
response = client.post(
|
||||
"/v1/videos",
|
||||
json={"video_path": "unused.mp4", "sr_resolution_scale": 4},
|
||||
)
|
||||
assert response.status_code == status, response.text
|
||||
if status == 200:
|
||||
assert response.json()["status"] == "queued"
|
||||
dispatch.assert_awaited_once()
|
||||
_, batch = dispatch.call_args.args
|
||||
assert batch.prompt == ""
|
||||
assert batch.sampling_params.sr_resolution_scale == 4
|
||||
else:
|
||||
dispatch.assert_not_called()
|
||||
|
||||
|
||||
def test_sr_declares_its_cli_flags_as_multipart_extra_fields():
|
||||
assert Kandinsky6SRSamplingParams.video_request_extra_fields() == _SR_EXTRA_FIELDS
|
||||
# The base default (what every other model still gets) stays empty.
|
||||
assert SamplingParams.video_request_extra_fields() == frozenset()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The real multipart-field allowlist (video_api._multipart_video_extras)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_multipart_extras_keep_sr_fields_only_for_the_sr_params_class():
|
||||
raw_form = {
|
||||
"video_path": "unused.mp4",
|
||||
"sr_resolution_scale": "4",
|
||||
"sr_tiles_batch_size": "3",
|
||||
"not_a_declared_field": "dropped",
|
||||
}
|
||||
sr_extras = _multipart_video_extras(
|
||||
raw_form,
|
||||
extra_body=None,
|
||||
extra_params=None,
|
||||
sampling_params_cls=Kandinsky6SRSamplingParams,
|
||||
)
|
||||
assert sr_extras["sr_resolution_scale"] == 4 # JSON-decoded, not the raw "4"
|
||||
assert sr_extras["sr_tiles_batch_size"] == 3
|
||||
assert "not_a_declared_field" not in sr_extras
|
||||
|
||||
# A model without the override (the pre-fix behaviour) drops the sr_* fields too.
|
||||
base_extras = _multipart_video_extras(
|
||||
raw_form,
|
||||
extra_body=None,
|
||||
extra_params=None,
|
||||
sampling_params_cls=SamplingParams,
|
||||
)
|
||||
assert "sr_resolution_scale" not in base_extras
|
||||
assert "sr_tiles_batch_size" not in base_extras
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The registry-backed class resolution the admission gate reads
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_video_sampling_params_cls_resolves_to_sr_for_the_sr_pipeline():
|
||||
server_args = _fake_server_args()
|
||||
assert resolve_sampling_params_cls(server_args) is Kandinsky6SRSamplingParams
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# End-to-end request parsing: a prompt-less request field set -> sampling params
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_build_video_sampling_params_accepts_no_prompt_and_carries_sr_fields():
|
||||
"""The real kwargs-building code (``_build_video_sampling_params``), given a
|
||||
request built the way the multipart branch builds one for a prompt-optional
|
||||
pipeline (``prompt=prompt or ""`` + the surviving sr_* extras), must construct a
|
||||
real ``Kandinsky6SRSamplingParams`` with those fields set and no prompt error."""
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai.protocol import (
|
||||
VideoGenerationsRequest,
|
||||
)
|
||||
|
||||
server_args = _fake_server_args()
|
||||
raw_form = {
|
||||
"video_path": "unused.mp4",
|
||||
"sr_resolution_scale": "4",
|
||||
"sr_tiles_batch_size": "3",
|
||||
}
|
||||
extras = _multipart_video_extras(
|
||||
raw_form,
|
||||
extra_body=None,
|
||||
extra_params=None,
|
||||
sampling_params_cls=Kandinsky6SRSamplingParams,
|
||||
)
|
||||
req = VideoGenerationsRequest(
|
||||
prompt="", # what video_api.py substitutes for a missing prompt
|
||||
video_path=extras.pop("video_path"),
|
||||
**extras,
|
||||
)
|
||||
|
||||
with _patched_global_server_args(server_args):
|
||||
sampling_params = _build_video_sampling_params("gap1-request-id", req)
|
||||
|
||||
assert isinstance(sampling_params, Kandinsky6SRSamplingParams)
|
||||
assert sampling_params.prompt == ""
|
||||
assert sampling_params.sr_resolution_scale == 4
|
||||
assert sampling_params.sr_tiles_batch_size == 3
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# The full endpoint, called directly (see module docstring for why not TestClient)
|
||||
# --------------------------------------------------------------------------- #
|
||||
class _FakeMultipartRequest:
|
||||
"""Duck-types the slice of ``fastapi.Request`` ``create_video`` reads directly."""
|
||||
|
||||
def __init__(self, form: dict):
|
||||
self.headers = {"content-type": "multipart/form-data; boundary=x"}
|
||||
self._form = form
|
||||
|
||||
async def form(self):
|
||||
return self._form
|
||||
|
||||
|
||||
def test_multipart_sr_request_without_prompt_is_accepted():
|
||||
"""A multipart SR request with no ``prompt`` field must be admitted (no 400), the
|
||||
``sr_*`` fields must reach the queued request, and it must fail on the pre-fix
|
||||
code: the old unconditional ``if not prompt: raise HTTPException(...)`` gate would
|
||||
reject this before any of the rest of ``create_video`` ran."""
|
||||
server_args = _fake_server_args()
|
||||
request = _FakeMultipartRequest(
|
||||
{"sr_resolution_scale": "4", "sr_tiles_batch_size": "3"}
|
||||
)
|
||||
dispatched: list[tuple] = []
|
||||
|
||||
async def _fake_dispatch(job_id, batch, **kwargs):
|
||||
dispatched.append((job_id, batch, kwargs))
|
||||
|
||||
async def run():
|
||||
with (
|
||||
_patched_global_server_args(server_args),
|
||||
patch.object(video_api, "_dispatch_job_async", side_effect=_fake_dispatch),
|
||||
):
|
||||
response = await video_api.create_video(
|
||||
request,
|
||||
prompt=None, # <-- the field under test: no prompt supplied
|
||||
enhance_prompt=None,
|
||||
task_type=None,
|
||||
x264_preset=None,
|
||||
perf_dump_path=None,
|
||||
input_reference=None,
|
||||
reference_url=None,
|
||||
video_reference=None,
|
||||
video_url=None,
|
||||
video_path="unused.mp4",
|
||||
model=None,
|
||||
n=1,
|
||||
num_outputs_per_prompt=None,
|
||||
seconds=None,
|
||||
size=None,
|
||||
fps=None,
|
||||
num_frames=None,
|
||||
seed=None,
|
||||
generator_device="cuda",
|
||||
negative_prompt=None,
|
||||
guidance_scale=None,
|
||||
guidance_scale_2=None,
|
||||
true_cfg_scale=None,
|
||||
num_inference_steps=None,
|
||||
max_sequence_length=None,
|
||||
flow_shift=None,
|
||||
enable_teacache=None,
|
||||
enable_frame_interpolation=None,
|
||||
frame_interpolation_exp=None,
|
||||
frame_interpolation_scale=None,
|
||||
frame_interpolation_model_path=None,
|
||||
enable_upscaling=None,
|
||||
upscaling_model_path=None,
|
||||
upscaling_scale=None,
|
||||
output_quality=None,
|
||||
output_compression=None,
|
||||
output_path=None,
|
||||
extra_params=None,
|
||||
extra_body=None,
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
return response
|
||||
|
||||
response = asyncio.run(run())
|
||||
|
||||
assert isinstance(response, VideoResponse)
|
||||
assert response.status == "queued"
|
||||
assert len(dispatched) == 1
|
||||
_, batch, _ = dispatched[0]
|
||||
assert batch.prompt == ""
|
||||
assert batch.sampling_params.sr_resolution_scale == 4
|
||||
assert batch.sampling_params.sr_tiles_batch_size == 3
|
||||
@@ -0,0 +1,467 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Strict loading, dtype and config contracts for tiny Kandinsky 6 SR components."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
Kandinsky6SRVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders import (
|
||||
component_loader as component_loader_module,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.latent_upscaler_loader import (
|
||||
LatentUpscalerLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.vae_loader import VAELoader
|
||||
from sglang.multimodal_gen.runtime.loader.fsdp_load import (
|
||||
load_model_from_full_model_state_dict,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import get_param_names_mapping
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRTransformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
|
||||
Kandinsky6SRLatentUpscalerBank,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
|
||||
|
||||
TINY_DIT = dict(
|
||||
in_visual_dim=4,
|
||||
in_text_dim=8,
|
||||
in_text_dim2=8,
|
||||
time_dim=16,
|
||||
out_visual_dim=4,
|
||||
patch_size=[1, 2, 2],
|
||||
model_dim=32,
|
||||
ff_dim=64,
|
||||
num_text_blocks=0,
|
||||
num_visual_blocks=2,
|
||||
axes_dims=[8, 4, 4],
|
||||
visual_cond=False,
|
||||
instruct_type="noise",
|
||||
use_text=False,
|
||||
n_grid=3,
|
||||
attribute_overrides={"instruct_type": "noise"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29508",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cpu_only(monkeypatch):
|
||||
"""Load on CPU; the forward test moves its DiT to CUDA for fused kernels."""
|
||||
monkeypatch.setattr(
|
||||
component_loader_module.ComponentLoader,
|
||||
"target_device",
|
||||
staticmethod(lambda component_starts_on_cpu: torch.device("cpu")),
|
||||
)
|
||||
|
||||
|
||||
def _dit_config(**changes):
|
||||
flat = dict(TINY_DIT, **changes)
|
||||
config = Kandinsky6SRDitConfig()
|
||||
config.update_model_arch(flat)
|
||||
return config, flat
|
||||
|
||||
|
||||
def _dit(**changes):
|
||||
"""Tiny DiT with deterministic random weights (the linears start uninitialized)."""
|
||||
config, flat = _dit_config(**changes)
|
||||
model = Kandinsky6SRTransformer3DModel(config, flat)
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
with torch.no_grad():
|
||||
for param in model.parameters():
|
||||
param.copy_(torch.randn(param.shape, generator=generator) * 0.05)
|
||||
return model
|
||||
|
||||
|
||||
def _official_state_dict(model):
|
||||
"""State dict as the current Diffusers checkpoint stores it."""
|
||||
return {
|
||||
name.replace(".feed_forward.mlp.fc_in.", ".feed_forward.net.0.proj.")
|
||||
.replace(".feed_forward.mlp.fc_out.", ".feed_forward.net.2.")
|
||||
.replace(
|
||||
"time_embeddings.in_layer.",
|
||||
"time_embeddings.timestep_embedder.linear_1.",
|
||||
)
|
||||
.replace(
|
||||
"time_embeddings.out_layer.",
|
||||
"time_embeddings.timestep_embedder.linear_2.",
|
||||
): tensor
|
||||
for name, tensor in model.state_dict().items()
|
||||
}
|
||||
|
||||
|
||||
def _load_through_the_real_loader(checkpoint, **changes):
|
||||
"""Meta-init the DiT and load ``checkpoint`` like ``maybe_load_fsdp_model`` does."""
|
||||
config, flat = _dit_config(**changes)
|
||||
with torch.device("meta"):
|
||||
model = Kandinsky6SRTransformer3DModel(config, flat)
|
||||
load_model_from_full_model_state_dict(
|
||||
model,
|
||||
model.preprocess_loaded_state_dict(iter(checkpoint.items())),
|
||||
torch.device("cpu"),
|
||||
torch.float32,
|
||||
strict=False,
|
||||
param_names_mapping=get_param_names_mapping(model.param_names_mapping),
|
||||
)
|
||||
return model
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# DiT
|
||||
# --------------------------------------------------------------------------- #
|
||||
def test_complete_checkpoint_loads_and_meta_buffers_are_rebuilt():
|
||||
"""Positive control for the failure tests below, and the meta-device path: RoPE
|
||||
tables and time-embedding frequencies are not in the checkpoint and must be rebuilt
|
||||
by ``post_load_weights`` so that a loaded model actually runs."""
|
||||
reference = _dit()
|
||||
checkpoint = _official_state_dict(reference)
|
||||
model = _load_through_the_real_loader(checkpoint)
|
||||
model.post_load_weights()
|
||||
assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()])
|
||||
assert not model.time_embeddings.freqs.is_meta
|
||||
torch.testing.assert_close(model.pooled_bias, reference.pooled_bias)
|
||||
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
model.to(device)
|
||||
x = torch.randn(1, 2, 4, 4, 4, device=device)
|
||||
rope_pos = [torch.arange(2, device=device) for _ in range(3)]
|
||||
out = model(x, torch.tensor([500.0], device=device), rope_pos)
|
||||
assert out.shape == (1, 2, 4, 4, 12) and torch.isfinite(out).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"missing",
|
||||
[
|
||||
"pooled_bias",
|
||||
"visual_transformer_blocks.0.visual_modulation.out_layer.bias",
|
||||
],
|
||||
)
|
||||
def test_missing_checkpoint_tensor_fails_the_load(missing):
|
||||
"""The generic loader zero-fills missing parameters whose names contain ``bias`` (and
|
||||
would only warn about others). For this model every parameter is required, so a
|
||||
partial checkpoint must raise instead of yielding a silently different network."""
|
||||
checkpoint = _official_state_dict(_dit())
|
||||
del checkpoint[missing]
|
||||
with pytest.raises(ValueError, match="was not loaded"):
|
||||
_load_through_the_real_loader(checkpoint)
|
||||
|
||||
|
||||
def test_unexpected_checkpoint_tensor_fails_the_load():
|
||||
checkpoint = _official_state_dict(_dit())
|
||||
checkpoint["text_embeddings.in_layer.weight"] = torch.zeros(2, 2)
|
||||
with pytest.raises(ValueError, match="unexpected keys"):
|
||||
_load_through_the_real_loader(checkpoint)
|
||||
|
||||
|
||||
def test_official_config_builds_the_head_from_the_total_width_and_loads_official_keys():
|
||||
"""An official config stores the TOTAL DX head width in ``out_visual_dim`` (n_grid lives
|
||||
in the scheduler config) plus nested ``sr_params`` and ``attribute_overrides: null``.
|
||||
"""
|
||||
official = {
|
||||
key: value
|
||||
for key, value in TINY_DIT.items()
|
||||
if key not in ("n_grid", "attribute_overrides")
|
||||
}
|
||||
official.update(
|
||||
out_visual_dim=12,
|
||||
attribute_overrides=None,
|
||||
sr_params=dict(
|
||||
scale_factor={"512": [1.0, 1.0, 1.0]},
|
||||
visual_size=[512],
|
||||
scheduler_scale=5.0,
|
||||
lq_noise_scale=0.7,
|
||||
lq_noise_type="ddpm",
|
||||
lq_channel_noise_scale=0.0,
|
||||
cap_noise_timestep=False,
|
||||
fps=24,
|
||||
),
|
||||
)
|
||||
config = Kandinsky6SRDitConfig()
|
||||
config.update_model_arch(official)
|
||||
with torch.device("meta"):
|
||||
model = Kandinsky6SRTransformer3DModel(config, official)
|
||||
# head width = prod(patch_size) * the TOTAL out_visual_dim
|
||||
assert model.state_dict()["out_layer.out_layer.weight"].shape[0] == 4 * 12
|
||||
assert model.base_out_visual_dim == 4 and model.n_grid == 1
|
||||
|
||||
load_model_from_full_model_state_dict(
|
||||
model,
|
||||
model.preprocess_loaded_state_dict(
|
||||
iter(
|
||||
_official_state_dict(_dit()).items()
|
||||
) # same shapes as the legacy DX head
|
||||
),
|
||||
torch.device("cpu"),
|
||||
torch.float32,
|
||||
strict=False,
|
||||
param_names_mapping=get_param_names_mapping(model.param_names_mapping),
|
||||
)
|
||||
model.post_load_weights()
|
||||
assert not any(t.is_meta for t in [*model.parameters(), *model.buffers()])
|
||||
|
||||
|
||||
def test_wide_input_checkpoint_needs_visual_cond_to_run_under_noise():
|
||||
"""A hybrid_anchor checkpoint has a ``2 * in + 1`` input layer; the default ``noise``
|
||||
override would feed ``in`` channels. That mismatch must be reported, and the
|
||||
documented ``visual_cond`` override must make it run."""
|
||||
wide = dict(
|
||||
instruct_type="hybrid_anchor", attribute_overrides={"instruct_type": "noise"}
|
||||
)
|
||||
with pytest.raises(ValueError, match="visual_cond"):
|
||||
_dit(**wide)
|
||||
|
||||
model = _dit(
|
||||
instruct_type="hybrid_anchor",
|
||||
attribute_overrides={"instruct_type": "noise", "visual_cond": True},
|
||||
)
|
||||
assert model.instruct_type == "noise" and model.visual_cond is True
|
||||
assert model.visual_embed_dim == 9
|
||||
assert model.config.instruct_type == "hybrid_anchor" # the trained value is kept
|
||||
|
||||
|
||||
def test_rope_angle_tables_stay_fp32_when_the_module_is_cast():
|
||||
"""The residency manager casts a module to its target dtype (bf16) on every move to
|
||||
the GPU; the RoPE angle tables must not be rounded to bf16 with the weights."""
|
||||
model = _dit().to(torch.bfloat16)
|
||||
assert model.pooled_bias.dtype == torch.bfloat16
|
||||
for name, buffer in model.visual_rope_embeddings.named_buffers():
|
||||
assert buffer.dtype == torch.float32, name
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fake server args for the component loaders
|
||||
# --------------------------------------------------------------------------- #
|
||||
class FakeServerArgs:
|
||||
def __init__(self):
|
||||
self.pipeline_config = Kandinsky6SRPipelineConfig()
|
||||
self.model_paths = {}
|
||||
self.component_paths = {}
|
||||
self.component_precisions = {}
|
||||
self.component_weights_paths = {}
|
||||
self.component_quantizations = {}
|
||||
self.num_gpus = 1
|
||||
self.disable_autocast = False
|
||||
self.revision = None
|
||||
self.trust_remote_code = False
|
||||
|
||||
def should_start_component_on_cpu(self, component_name):
|
||||
return False
|
||||
|
||||
def should_direct_gpu_weight_load_component(self, component_name):
|
||||
return False
|
||||
|
||||
def should_use_fsdp_for_component(self, component_name):
|
||||
return False
|
||||
|
||||
def requested_component_attention_backend(self, component_name):
|
||||
return None
|
||||
|
||||
def resolve_component_attention_backend(self, *names):
|
||||
return None, None
|
||||
|
||||
|
||||
def _write_component(directory, config, tensors):
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
(directory / "config.json").write_text(json.dumps(config))
|
||||
save_file(tensors, str(directory / "diffusion_pytorch_model.safetensors"))
|
||||
|
||||
|
||||
def _tiny_vae_bundle(tmp_path):
|
||||
raw_config = dict(
|
||||
_class_name="Kandinsky6SRVAE",
|
||||
vae_type="video-kvae",
|
||||
encoder_config=dict(TINY_KVAE, in_channels=3),
|
||||
decoder_config=dict(TINY_KVAE, out_ch=3),
|
||||
scaling_factor=0.5,
|
||||
spatial_factor=16,
|
||||
temporal_factor=4,
|
||||
)
|
||||
config = Kandinsky6SRVAEConfig()
|
||||
config.update_model_arch(raw_config)
|
||||
wrapper = Kandinsky6SRVAE(config)
|
||||
directory = tmp_path / "vae"
|
||||
_write_component(
|
||||
directory,
|
||||
raw_config,
|
||||
{k: v.contiguous() for k, v in wrapper.state_dict().items()},
|
||||
)
|
||||
return directory
|
||||
|
||||
|
||||
def test_vae_loader_loads_the_wrapper_strictly(tmp_path):
|
||||
directory = _tiny_vae_bundle(tmp_path)
|
||||
server_args = FakeServerArgs()
|
||||
vae = VAELoader().load_customized(str(directory), server_args, "vae")
|
||||
assert isinstance(vae, Kandinsky6SRVAE) and vae.scaling_factor == 0.5
|
||||
assert next(vae.parameters()).dtype == torch.bfloat16 # vae_precision
|
||||
assert (vae.spatial_factor, vae.temporal_factor) == (16, 4)
|
||||
|
||||
# one missing tensor must fail: non-strict VAE loading would only log a warning
|
||||
weights_path = directory / "diffusion_pytorch_model.safetensors"
|
||||
weights = load_file(str(weights_path))
|
||||
torch.testing.assert_close(
|
||||
vae.state_dict(),
|
||||
{name: value.bfloat16() for name, value in weights.items()},
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
weights.pop(next(iter(weights)))
|
||||
save_file(weights, str(weights_path))
|
||||
with pytest.raises(RuntimeError, match="Missing key"):
|
||||
VAELoader().load_customized(str(directory), FakeServerArgs(), "vae")
|
||||
|
||||
|
||||
def _tiny_lu_bundle(tmp_path, models=None, scales=(2, 4), include_scales=True):
|
||||
models = models or [
|
||||
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
|
||||
for scale in ("2x", "4x")
|
||||
]
|
||||
bank = Kandinsky6SRLatentUpscalerBank(models, 0.5, scales=scales)
|
||||
directory = tmp_path / "latent_upscaler"
|
||||
config = {
|
||||
"_class_name": "Kandinsky6SRLatentUpscalerBank",
|
||||
"models": models,
|
||||
"scaling_factor": 0.5,
|
||||
}
|
||||
if include_scales:
|
||||
config["scales"] = list(scales)
|
||||
_write_component(
|
||||
directory,
|
||||
config,
|
||||
{k: v.contiguous() for k, v in bank.state_dict().items()},
|
||||
)
|
||||
return directory
|
||||
|
||||
|
||||
def test_latent_upscaler_loader_loads_bank_in_bf16_eval(tmp_path):
|
||||
directory = _tiny_lu_bundle(tmp_path)
|
||||
bank, _ = LatentUpscalerLoader().load(
|
||||
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
|
||||
)
|
||||
assert bank.scales == (2, 4)
|
||||
assert not bank.training
|
||||
assert all(
|
||||
p.dtype == torch.bfloat16 and not p.requires_grad for p in bank.parameters()
|
||||
)
|
||||
z = torch.randn(1, 4, 3, 4, 6, dtype=torch.bfloat16)
|
||||
assert bank.upscale(z, scale=2).shape == (1, 4, 3, 8, 12)
|
||||
assert bank.upscale(z, scale=4).shape == (1, 4, 3, 16, 24)
|
||||
with pytest.raises(ValueError, match="no x8 entry"):
|
||||
bank.upscale(z, scale=8)
|
||||
|
||||
|
||||
def test_latent_upscaler_legacy_config_uses_current_default_scale_order(tmp_path):
|
||||
models = [
|
||||
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
|
||||
for scale in ("4x", "2x")
|
||||
]
|
||||
directory = _tiny_lu_bundle(
|
||||
tmp_path,
|
||||
models=models,
|
||||
include_scales=False,
|
||||
)
|
||||
bank, _ = LatentUpscalerLoader().load(
|
||||
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
|
||||
)
|
||||
assert bank.scales == (2, 4)
|
||||
assert bank._models[0] is bank.for_scale(2)
|
||||
assert bank._models[1] is bank.for_scale(4)
|
||||
|
||||
|
||||
def test_latent_upscaler_may_be_declared_with_the_kandinsky6_library():
|
||||
"""The official ``save_pretrained`` writes ["kandinsky6", ...], the Hub repo ["diffusers", ...]."""
|
||||
for library in ("diffusers", "kandinsky6"):
|
||||
loader = component_loader_module.ComponentLoader.for_component_type(
|
||||
"latent_upscaler", library
|
||||
)
|
||||
assert isinstance(loader, LatentUpscalerLoader)
|
||||
with pytest.raises(AssertionError, match="latent_upscaler must be loaded from"):
|
||||
component_loader_module.ComponentLoader.for_component_type(
|
||||
"latent_upscaler", "transformers"
|
||||
)
|
||||
|
||||
|
||||
def test_latent_upscaler_load_errors_are_raised_not_swallowed(tmp_path):
|
||||
"""``ComponentLoader.load`` falls back to a diffusers ``AutoModel`` when the
|
||||
customized loader raises, which hides real errors. This component must re-raise,
|
||||
with the offending config key or tensor named."""
|
||||
directory = _tiny_lu_bundle(tmp_path)
|
||||
weights_path = directory / "diffusion_pytorch_model.safetensors"
|
||||
weights = load_file(str(weights_path))
|
||||
weights.pop(next(iter(weights)))
|
||||
save_file(weights, str(weights_path))
|
||||
with pytest.raises(RuntimeError, match="native fallback is disabled") as error:
|
||||
LatentUpscalerLoader().load(
|
||||
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
|
||||
)
|
||||
assert "Missing key" in str(error.value.__cause__)
|
||||
|
||||
bad = copy.deepcopy(TINY_LU_MODEL)
|
||||
bad["bogus_field"] = 1
|
||||
directory = _tiny_lu_bundle(
|
||||
tmp_path / "bad",
|
||||
models=[{"target_scale": "2x", "model": TINY_LU_MODEL}],
|
||||
scales=(2,),
|
||||
)
|
||||
config_path = directory / "config.json"
|
||||
config = json.loads(config_path.read_text())
|
||||
config["models"][0]["model"] = bad
|
||||
config_path.write_text(json.dumps(config))
|
||||
with pytest.raises(RuntimeError, match="native fallback is disabled") as error:
|
||||
LatentUpscalerLoader().load(
|
||||
str(directory), FakeServerArgs(), "latent_upscaler", "diffusers"
|
||||
)
|
||||
assert "bogus_field" in str(error.value.__cause__)
|
||||
|
||||
|
||||
def test_motion_attention_is_not_a_supported_entry_key():
|
||||
"""The consolidated architecture (mirroring FastVideo's ``kandinsky6_sr.py`` /
|
||||
commit 950a5edb) only implements the released checkpoint, which never trains
|
||||
natten/shifted-window motion attention: a config that asks for it must fail loudly
|
||||
at construction instead of silently building an unsupported backend."""
|
||||
with_motion = {
|
||||
**copy.deepcopy(TINY_LU_MODEL),
|
||||
"enable_x2_entry": False,
|
||||
"motion_attention": {
|
||||
"spatial_kernel_size": 3,
|
||||
"temporal_offsets": [-1, 1],
|
||||
"after_mid_blocks": [1],
|
||||
"num_heads": 2,
|
||||
},
|
||||
}
|
||||
with pytest.raises(ValueError, match="motion_attention"):
|
||||
Kandinsky6SRLatentUpscalerBank(
|
||||
[{"target_scale": "4x", "model": with_motion}], 0.5
|
||||
)
|
||||
+369
@@ -0,0 +1,369 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Validate release config shapes and component loading without downloading weights."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from collections import Counter
|
||||
from types import SimpleNamespace
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from kandinsky6_sr_release_configs import (
|
||||
DISTILLED_MODEL_INDEX,
|
||||
DISTILLED_SCHEDULER_CONFIG,
|
||||
DISTILLED_TRANSFORMER_CONFIG,
|
||||
FLOW_MODEL_INDEX,
|
||||
FLOW_SCHEDULER_CONFIG,
|
||||
FLOW_SR_CONFIG,
|
||||
FLOW_TRANSFORMER_CONFIG,
|
||||
LATENT_UPSCALER_CONFIG,
|
||||
VAE_CONFIG,
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
Kandinsky6SRVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.component_loader import (
|
||||
PipelineComponentLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.component_loaders.scheduler_loader import (
|
||||
SchedulerLoader,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRTransformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
|
||||
PiflowScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
|
||||
Kandinsky6SRLatentUpscalerBank,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
|
||||
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import (
|
||||
Kandinsky6SRPipeline,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
build_sampling_spec,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
PiflowParams,
|
||||
)
|
||||
|
||||
|
||||
class Bundle(NamedTuple):
|
||||
"""One official repo: its configs and what they must resolve to."""
|
||||
|
||||
model_index: dict
|
||||
transformer: dict
|
||||
scheduler: dict
|
||||
sr_config: dict | None
|
||||
scheduler_cls: type
|
||||
head_width: int # ``out_visual_dim`` of the transformer: the TOTAL head width
|
||||
n_grid: int # grids of the head: 1 (flow-matching) or the scheduler's ``n_grid``
|
||||
scheduler_scale: float # ``sr_params.scheduler_scale``
|
||||
calls_per_tile: int # DiT calls per tile at the default ``num_inference_steps`` 5
|
||||
|
||||
|
||||
FLOW = Bundle(
|
||||
model_index=FLOW_MODEL_INDEX,
|
||||
transformer=FLOW_TRANSFORMER_CONFIG,
|
||||
scheduler=FLOW_SCHEDULER_CONFIG,
|
||||
sr_config=FLOW_SR_CONFIG,
|
||||
scheduler_cls=FlowMatchEulerDiscreteScheduler,
|
||||
head_width=64,
|
||||
n_grid=1,
|
||||
scheduler_scale=5.0,
|
||||
calls_per_tile=5,
|
||||
)
|
||||
DISTILLED = Bundle(
|
||||
model_index=DISTILLED_MODEL_INDEX,
|
||||
transformer=DISTILLED_TRANSFORMER_CONFIG,
|
||||
scheduler=DISTILLED_SCHEDULER_CONFIG,
|
||||
sr_config=None,
|
||||
scheduler_cls=PiflowScheduler,
|
||||
head_width=640,
|
||||
n_grid=10,
|
||||
scheduler_scale=3.5,
|
||||
calls_per_tile=2,
|
||||
)
|
||||
both_repos = pytest.mark.parametrize(
|
||||
"bundle", [FLOW, DISTILLED], ids=["flow_matching", "distilled_pi_flow"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29509",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
|
||||
|
||||
def _prefixes(keys, depth=2):
|
||||
return Counter(".".join(key.split(".")[:depth]) for key in keys)
|
||||
|
||||
|
||||
def _dit_config(bundle):
|
||||
config = Kandinsky6SRDitConfig()
|
||||
config.update_model_arch(copy.deepcopy(bundle.transformer))
|
||||
return config
|
||||
|
||||
|
||||
def _load_scheduler(bundle, tmp_path):
|
||||
"""The scheduler component through SGLang's own loader (a ``scheduler/`` dir)."""
|
||||
directory = tmp_path / "scheduler"
|
||||
directory.mkdir()
|
||||
(directory / "scheduler_config.json").write_text(json.dumps(bundle.scheduler))
|
||||
server_args = SimpleNamespace(pipeline_config=Kandinsky6SRPipelineConfig())
|
||||
return SchedulerLoader().load_customized(str(directory), server_args)
|
||||
|
||||
|
||||
@both_repos
|
||||
def test_real_transformer_config_builds_the_release_dit(bundle):
|
||||
config = _dit_config(bundle)
|
||||
with torch.device("meta"):
|
||||
model = Kandinsky6SRTransformer3DModel(
|
||||
config, copy.deepcopy(bundle.transformer)
|
||||
)
|
||||
state = model.state_dict()
|
||||
per_block = Counter(
|
||||
key.split(".")[1]
|
||||
for key in state
|
||||
if key.startswith("visual_transformer_blocks.")
|
||||
)
|
||||
assert (
|
||||
len(state) == 459 and len(per_block) == 32 and set(per_block.values()) == {14}
|
||||
)
|
||||
non_block = [k for k in state if not k.startswith("visual_transformer_blocks.")]
|
||||
assert len(non_block) == 11
|
||||
# ``out_visual_dim`` is the TOTAL head width (64 * n_grid); a pi-Flow repo keeps its
|
||||
# n_grid in the scheduler config
|
||||
assert state["out_layer.out_layer.weight"].shape == (bundle.head_width, 1792)
|
||||
assert model.base_out_visual_dim == 64 and model.n_grid == 1
|
||||
assert bundle.n_grid * model.base_out_visual_dim == bundle.head_width
|
||||
assert model.instruct_type == "hybrid_anchor" and model.visual_cond
|
||||
assert model.visual_embed_dim == 2 * 64 + 1 # wide [x | cond | mask] input
|
||||
arch = config.arch_config
|
||||
assert arch.sr_scale_factor == {"512": [1.0, 2.0, 2.0]}
|
||||
assert arch.sr_scheduler_scale == bundle.scheduler_scale and not arch.is_piflow
|
||||
|
||||
|
||||
@both_repos
|
||||
def test_real_scheduler_config_loads_through_the_scheduler_loader(bundle, tmp_path):
|
||||
scheduler = _load_scheduler(bundle, tmp_path)
|
||||
# PiflowScheduler subclasses the flow-matching scheduler: compare exact types
|
||||
assert type(scheduler) is bundle.scheduler_cls
|
||||
assert scheduler.config["shift"] == bundle.scheduler["shift"]
|
||||
if bundle is DISTILLED:
|
||||
assert (scheduler.config["nfe"], scheduler.config["n_grid"]) == (2, 10)
|
||||
|
||||
|
||||
@both_repos
|
||||
def test_real_scheduler_selects_the_sampler_of_its_repo(bundle, tmp_path):
|
||||
"""The distilled repo resolves to pi-Flow (``nfe`` = 2 calls per tile, whatever
|
||||
``num_inference_steps`` is), the flow-matching repo to flow-Euler
|
||||
(``num_inference_steps`` calls per tile directly -- this repo's own convention, not
|
||||
the upstream Diffusers pipeline's timestep-grid-point count).
|
||||
"""
|
||||
scheduler = _load_scheduler(bundle, tmp_path)
|
||||
arch = _dit_config(bundle).arch_config
|
||||
|
||||
def spec(num_steps):
|
||||
return build_sampling_spec(
|
||||
arch=arch,
|
||||
tiling_scale=2,
|
||||
seed=42,
|
||||
num_steps=num_steps,
|
||||
tiles_batch_size=1,
|
||||
tile_min_overlap=0.2,
|
||||
scheduler=scheduler,
|
||||
)
|
||||
|
||||
assert spec(5).steps_per_chunk == bundle.calls_per_tile
|
||||
assert spec(5).scheduler_scale == bundle.scheduler_scale
|
||||
if bundle is DISTILLED:
|
||||
assert spec(5).piflow == PiflowParams(
|
||||
nfe=2,
|
||||
num_policy_substeps=128,
|
||||
final_step_size_scale=0.5,
|
||||
shift=3.5,
|
||||
n_grid=10,
|
||||
eps=1e-6,
|
||||
)
|
||||
assert spec(3).steps_per_chunk == spec(9).steps_per_chunk == 2
|
||||
else:
|
||||
assert spec(5).piflow is None
|
||||
assert (spec(3).steps_per_chunk, spec(9).steps_per_chunk) == (3, 9)
|
||||
|
||||
|
||||
def test_real_vae_config_builds_the_release_kvae_and_maps_official_keys():
|
||||
config = Kandinsky6SRVAEConfig()
|
||||
config.update_model_arch(dict(VAE_CONFIG))
|
||||
with torch.device("meta"):
|
||||
vae = Kandinsky6SRVAE(config)
|
||||
# `self.encoder` / `self.decoder` are direct attributes of the wrapper (no intermediate
|
||||
# `model.*` nesting), so its state_dict keys already are the official checkpoint's own
|
||||
# `encoder.*` / `decoder.*` keys -- no stripping needed.
|
||||
official = dict(vae.state_dict())
|
||||
assert len(official) == 378
|
||||
assert _prefixes(official) == Counter(
|
||||
{
|
||||
"decoder.up": 238,
|
||||
"encoder.down": 86,
|
||||
"decoder.mid": 28,
|
||||
"encoder.mid": 12,
|
||||
"decoder.norm_out": 5,
|
||||
"decoder.conv_in": 2,
|
||||
"decoder.conv_out": 2,
|
||||
"encoder.conv_in": 2,
|
||||
"encoder.conv_out": 2,
|
||||
"encoder.norm_out": 1,
|
||||
}
|
||||
)
|
||||
assert vae.scaling_factor == pytest.approx(0.910344004631042)
|
||||
assert vae.spatial_factor == 16
|
||||
# the official keys load strictly, with no remapping
|
||||
vae.load_state_dict(official, strict=True, assign=True)
|
||||
|
||||
|
||||
def test_real_latent_upscaler_config_builds_the_release_bank():
|
||||
with torch.device("meta"):
|
||||
bank = Kandinsky6SRLatentUpscalerBank(
|
||||
LATENT_UPSCALER_CONFIG["models"],
|
||||
LATENT_UPSCALER_CONFIG["scaling_factor"],
|
||||
scales=(2, 4),
|
||||
)
|
||||
state = bank.state_dict()
|
||||
assert len(state) == 489
|
||||
assert _prefixes(state) == Counter({"_models.0": 311, "_models.1": 178})
|
||||
x2 = _prefixes((key.split(".", 2)[2] for key in state if "_models.0." in key))
|
||||
assert x2["x2_branch.blocks"] == 44 and x2["x2_branch.adapter"] == 28
|
||||
assert bank.scales == (2, 4)
|
||||
|
||||
|
||||
@both_repos
|
||||
@pytest.mark.parametrize("upscaler", ["present", "missing", "null"])
|
||||
def test_real_model_index_is_accepted_by_the_module_loading_path(
|
||||
bundle, tmp_path, monkeypatch, upscaler
|
||||
):
|
||||
"""``ComposedPipelineBase.load_modules`` sees a model_index.json whose extra
|
||||
``_kandinsky6_sr`` entry is a dict (not a ``[library, class]`` pair), next to a root
|
||||
``sr_config.json``: only the four components of the pipeline may be requested, with
|
||||
the scheduler class of the repo."""
|
||||
model_index = dict(bundle.model_index)
|
||||
if upscaler == "missing":
|
||||
model_index.pop("latent_upscaler")
|
||||
elif upscaler == "null":
|
||||
model_index["latent_upscaler"] = None
|
||||
(tmp_path / "model_index.json").write_text(json.dumps(model_index))
|
||||
if bundle.sr_config is not None:
|
||||
(tmp_path / "sr_config.json").write_text(json.dumps(bundle.sr_config))
|
||||
components = ["transformer", "vae", "scheduler"]
|
||||
if upscaler == "present":
|
||||
components.append("latent_upscaler")
|
||||
for name in components:
|
||||
(tmp_path / name).mkdir()
|
||||
if name != "scheduler": # the scheduler is config-only
|
||||
(tmp_path / name / "diffusion_pytorch_model.safetensors").touch()
|
||||
|
||||
requested = {}
|
||||
|
||||
def record_component(
|
||||
component_name,
|
||||
component_model_path,
|
||||
transformers_or_diffusers,
|
||||
server_args,
|
||||
component_architecture=None,
|
||||
component_type=None,
|
||||
loader_cls=None,
|
||||
component_attn_backend=None,
|
||||
component_attn_name=None,
|
||||
):
|
||||
assert component_type == component_name
|
||||
assert loader_cls is None and component_attn_backend is None
|
||||
assert component_attn_name == component_name
|
||||
requested[component_name] = (transformers_or_diffusers, component_architecture)
|
||||
return SimpleNamespace(name=component_name), 0.0
|
||||
|
||||
monkeypatch.setattr(
|
||||
PipelineComponentLoader, "load_component", staticmethod(record_component)
|
||||
)
|
||||
server_args = SimpleNamespace(
|
||||
model_subfolder=None,
|
||||
revision=None,
|
||||
comfyui_mode=False,
|
||||
component_paths={},
|
||||
component_direct_gpu_weight_loading={},
|
||||
pipeline_config=Kandinsky6SRPipelineConfig(),
|
||||
resolve_component_attention_backend=lambda *names: (None, None),
|
||||
)
|
||||
# the state ``ComposedPipelineBase.__init__`` sets up before it calls load_modules
|
||||
pipeline = object.__new__(Kandinsky6SRPipeline)
|
||||
pipeline.server_args = server_args
|
||||
pipeline.model_path = str(tmp_path)
|
||||
pipeline._required_config_modules = list(
|
||||
Kandinsky6SRPipeline._required_config_modules
|
||||
)
|
||||
pipeline._extra_config_module_map = {}
|
||||
pipeline._disagg_role = RoleType.MONOLITHIC
|
||||
pipeline.memory_usages = {}
|
||||
|
||||
modules = pipeline.load_modules(server_args)
|
||||
|
||||
expected = {
|
||||
"transformer": ("diffusers", "Kandinsky6SRTransformer3DModel"),
|
||||
"vae": ("diffusers", "Kandinsky6SRVAE"),
|
||||
"latent_upscaler": ("diffusers", "Kandinsky6SRLatentUpscalerBank"),
|
||||
"scheduler": ("diffusers", bundle.scheduler_cls.__name__),
|
||||
}
|
||||
if upscaler != "present":
|
||||
expected.pop("latent_upscaler")
|
||||
assert set(modules) == set(expected)
|
||||
assert requested == expected
|
||||
assert "latent_upscaler" in Kandinsky6SRPipeline._required_config_modules
|
||||
|
||||
|
||||
def test_legacy_bundle_requires_official_diffusers_layout(tmp_path):
|
||||
(tmp_path / "model_index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"_class_name": "Kandinsky6SRPipeline",
|
||||
"_diffusers_version": "0.37.0",
|
||||
"dit": ["x", "Kandinsky6SRDiT"],
|
||||
"vae": ["x", "Kandinsky6SRVAE"],
|
||||
}
|
||||
)
|
||||
)
|
||||
for name in ("dit", "vae"):
|
||||
(tmp_path / name).mkdir()
|
||||
(tmp_path / name / "diffusion_pytorch_model.safetensors").touch()
|
||||
pipeline = object.__new__(Kandinsky6SRPipeline)
|
||||
pipeline.server_args = SimpleNamespace(model_subfolder=None, revision=None)
|
||||
pipeline.model_path = str(tmp_path)
|
||||
with pytest.raises(
|
||||
ValueError, match="k6_video SR bundle.*kandinskylab/Kandinsky-6.0-VSR"
|
||||
):
|
||||
pipeline.load_modules(pipeline.server_args)
|
||||
@@ -0,0 +1,829 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""CPU parity of the Kandinsky 6 SR port against the k6_video reference (``kandinsky_sr``).
|
||||
|
||||
The reference package is not part of this repository. Point
|
||||
``KANDINSKY_SR_REFERENCE_SRC`` at the directory that contains ``kandinsky_sr`` (for
|
||||
example ``k6_video/src``); without it the whole module is skipped. The reference needs
|
||||
``flash_attn`` and ``loguru``, which are stubbed here (per-sequence SDPA for the dense
|
||||
varlen attention), and its base resolutions are shrunk so that tiles stay tiny. All
|
||||
components use tiny random weights (every parameter re-randomized: the reference
|
||||
zero-initializes most modulation / output layers, which would hide bugs) and float32
|
||||
math on CPU. The port loads the reference ``state_dict()`` (unprefixed, i.e. exactly the official
|
||||
Diffusers layout) with ``strict=True`` through its own name mapping.
|
||||
"""
|
||||
|
||||
import copy
|
||||
import importlib.machinery
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import types
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from kandinsky6_sr_tiny_components import TINY_KVAE, TINY_LU_MODEL, super_resolve
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.kandinsky6_sr import (
|
||||
Kandinsky6SRVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.loader.utils import (
|
||||
get_param_names_mapping,
|
||||
hf_to_custom_state_dict,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRTransformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
|
||||
PiflowScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
|
||||
Kandinsky6SRLatentUpscalerBank,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.kandinsky6_sr_vae import Kandinsky6SRVAE
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr import (
|
||||
tiling,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
build_dit_spec,
|
||||
build_sampling_spec,
|
||||
effective_scheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
denoise_with_scheduler,
|
||||
euler_start_timestep,
|
||||
make_dit_fn,
|
||||
module_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
plan_tiles,
|
||||
stitch_tiles,
|
||||
)
|
||||
|
||||
REFERENCE_ENV = "KANDINSKY_SR_REFERENCE_SRC"
|
||||
_reference_dir = os.environ.get(REFERENCE_ENV)
|
||||
if not _reference_dir or not os.path.isdir(
|
||||
os.path.join(_reference_dir, "kandinsky_sr")
|
||||
):
|
||||
pytest.skip(
|
||||
f"set {REFERENCE_ENV} to the directory containing the kandinsky_sr package",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Reference harness (test-only): stubs, tiny resolutions, tiny random components
|
||||
# --------------------------------------------------------------------------- #
|
||||
TINY_RESOLUTIONS = {512: [(64, 64), (64, 96), (96, 64)]}
|
||||
TINY_KVAE_ENC = dict(TINY_KVAE, in_channels=3)
|
||||
TINY_KVAE_DEC = dict(TINY_KVAE, out_ch=3)
|
||||
TINY_SCALING_FACTOR = 0.5
|
||||
TINY_DIT_CFG = dict(
|
||||
in_visual_dim=4,
|
||||
in_text_dim=8,
|
||||
in_text_dim2=8,
|
||||
time_dim=16,
|
||||
out_visual_dim=4,
|
||||
patch_size=(1, 2, 2),
|
||||
model_dim=32,
|
||||
ff_dim=64,
|
||||
num_text_blocks=0,
|
||||
num_visual_blocks=2,
|
||||
axes_dims=(8, 4, 4),
|
||||
visual_cond=False,
|
||||
instruct_type="noise",
|
||||
attention_params={"512": {"type": "flash"}},
|
||||
use_text=False,
|
||||
)
|
||||
TINY_PIFLOW = dict(
|
||||
nfe=2,
|
||||
num_policy_substeps=8,
|
||||
final_step_size_scale=0.5,
|
||||
shift=5.0,
|
||||
n_grid=3,
|
||||
eps=1e-6,
|
||||
)
|
||||
TINY_SR_PARAMS = dict(
|
||||
scale_factor={512: [1.0, 1.0, 1.0]},
|
||||
visual_size=[512],
|
||||
scheduler_scale=5.0,
|
||||
lq_noise_scale=0.7,
|
||||
lq_noise_type="ddpm",
|
||||
lq_channel_noise_scale=0.0,
|
||||
cap_noise_timestep=False,
|
||||
fps=24,
|
||||
)
|
||||
|
||||
|
||||
def _varlen_sdpa(q, k, v, cu_q, cu_k):
|
||||
outs = []
|
||||
for i in range(len(cu_q) - 1):
|
||||
qs, qe = int(cu_q[i]), int(cu_q[i + 1])
|
||||
ks, ke = int(cu_k[i]), int(cu_k[i + 1])
|
||||
qi = q[qs:qe].transpose(0, 1).unsqueeze(0)
|
||||
ki = k[ks:ke].transpose(0, 1).unsqueeze(0)
|
||||
vi = v[ks:ke].transpose(0, 1).unsqueeze(0)
|
||||
out = torch.nn.functional.scaled_dot_product_attention(qi, ki, vi)
|
||||
outs.append(out.squeeze(0).transpose(0, 1))
|
||||
return torch.cat(outs, dim=0)
|
||||
|
||||
|
||||
def _flash_attn_stub() -> types.ModuleType:
|
||||
module = types.ModuleType("flash_attn")
|
||||
|
||||
def varlen(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
cu_seqlens_q,
|
||||
cu_seqlens_k,
|
||||
max_seqlen_q,
|
||||
max_seqlen_k,
|
||||
return_attn_probs=False,
|
||||
**_,
|
||||
):
|
||||
out = _varlen_sdpa(q, k, v, cu_seqlens_q, cu_seqlens_k)
|
||||
return (out, torch.zeros(1), None) if return_attn_probs else out
|
||||
|
||||
def qkvpacked(qkv, cu_seqlens, max_seqlen, return_attn_probs=False, **_):
|
||||
q, k, v = qkv.unbind(dim=1)
|
||||
out = _varlen_sdpa(q, k, v, cu_seqlens, cu_seqlens)
|
||||
return (out, torch.zeros(1), None) if return_attn_probs else out
|
||||
|
||||
module.flash_attn_varlen_func = varlen
|
||||
module.flash_attn_varlen_qkvpacked_func = qkvpacked
|
||||
# transformers probes flash_attn with importlib.util.find_spec
|
||||
module.__spec__ = importlib.machinery.ModuleSpec("flash_attn", loader=None)
|
||||
return module
|
||||
|
||||
|
||||
def _loguru_stub() -> types.ModuleType:
|
||||
module = types.ModuleType("loguru")
|
||||
|
||||
class _Logger:
|
||||
def __getattr__(self, name):
|
||||
return lambda *args, **kwargs: self
|
||||
|
||||
module.logger = _Logger()
|
||||
module.__spec__ = importlib.machinery.ModuleSpec("loguru", loader=None)
|
||||
return module
|
||||
|
||||
|
||||
def _install_reference() -> None:
|
||||
os.environ.setdefault("TORCH_COMPILE_DISABLE", "1")
|
||||
os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
|
||||
if _reference_dir not in sys.path:
|
||||
sys.path.insert(0, _reference_dir)
|
||||
if "flash_attn" not in sys.modules:
|
||||
sys.modules["flash_attn"] = _flash_attn_stub()
|
||||
if "loguru" not in sys.modules:
|
||||
sys.modules["loguru"] = _loguru_stub()
|
||||
from kandinsky_sr import constants
|
||||
|
||||
constants.RESOLUTIONS.clear()
|
||||
constants.RESOLUTIONS.update(TINY_RESOLUTIONS)
|
||||
|
||||
|
||||
_install_reference()
|
||||
|
||||
from kandinsky_sr.core.components.model.dit import get_dit # noqa: E402
|
||||
from kandinsky_sr.core.components.model.dx_dit import DXDiTWrapper # noqa: E402
|
||||
from kandinsky_sr.core.components.video_kvae.cached_model import ( # noqa: E402
|
||||
CachedCausalVAE,
|
||||
)
|
||||
from omegaconf import OmegaConf # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
"""The K6 feed-forward uses TP-aware linears, which need a (size-1) TP group."""
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29507",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
|
||||
|
||||
def randomize(module: torch.nn.Module, seed: int, std: float = 0.05) -> None:
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
with torch.no_grad():
|
||||
for param in module.parameters():
|
||||
param.copy_(torch.randn(param.shape, generator=generator) * std)
|
||||
|
||||
|
||||
def build_reference_dit(*, piflow: dict | None, seed: int = 2, cfg: dict | None = None):
|
||||
cfg = dict(cfg or TINY_DIT_CFG)
|
||||
if piflow is not None:
|
||||
dit = DXDiTWrapper(
|
||||
cfg, out_visual_dim=cfg["out_visual_dim"], n_grid=piflow["n_grid"]
|
||||
)
|
||||
dit.piflow_params = dict(piflow)
|
||||
else:
|
||||
dit = get_dit(cfg)
|
||||
dit.eval()
|
||||
randomize(dit, seed)
|
||||
return dit
|
||||
|
||||
|
||||
def flat_transformer_config(*, cfg: dict, piflow: dict | None, overrides: dict) -> dict:
|
||||
"""The flat legacy dict form of a transformer config (n_grid / piflow_* / sr_* fields)."""
|
||||
flat = dict(cfg)
|
||||
flat["patch_size"] = list(flat["patch_size"])
|
||||
flat["axes_dims"] = list(flat["axes_dims"])
|
||||
flat["n_grid"] = piflow["n_grid"] if piflow is not None else 1
|
||||
if piflow is not None:
|
||||
flat.update(
|
||||
piflow_nfe=piflow["nfe"],
|
||||
piflow_num_policy_substeps=piflow["num_policy_substeps"],
|
||||
piflow_final_step_size_scale=piflow["final_step_size_scale"],
|
||||
piflow_shift=piflow["shift"],
|
||||
piflow_eps=piflow["eps"],
|
||||
)
|
||||
flat["attribute_overrides"] = dict(overrides)
|
||||
flat.update(
|
||||
sr_visual_size=[512],
|
||||
sr_scale_factor={"512": [1.0, 1.0, 1.0]},
|
||||
sr_scheduler_scale=TINY_SR_PARAMS["scheduler_scale"],
|
||||
sr_lq_noise_scale=TINY_SR_PARAMS["lq_noise_scale"],
|
||||
sr_lq_noise_type=TINY_SR_PARAMS["lq_noise_type"],
|
||||
sr_lq_channel_noise_scale=TINY_SR_PARAMS["lq_channel_noise_scale"],
|
||||
sr_cap_noise_timestep=TINY_SR_PARAMS["cap_noise_timestep"],
|
||||
sr_fps=TINY_SR_PARAMS["fps"],
|
||||
)
|
||||
return flat
|
||||
|
||||
|
||||
def build_port_dit(
|
||||
reference_dit, *, cfg: dict | None = None, piflow: dict | None, overrides: dict
|
||||
) -> Kandinsky6SRTransformer3DModel:
|
||||
"""Port DiT loaded with the reference weights (``model.`` prefix), strictly."""
|
||||
flat = flat_transformer_config(
|
||||
cfg=dict(cfg or TINY_DIT_CFG), piflow=piflow, overrides=overrides
|
||||
)
|
||||
dit_config = Kandinsky6SRDitConfig()
|
||||
dit_config.update_model_arch(flat)
|
||||
model = Kandinsky6SRTransformer3DModel(dit_config, flat)
|
||||
official = dict(reference_dit.state_dict()) # unprefixed = the official layout
|
||||
mapped, _ = hf_to_custom_state_dict(
|
||||
official,
|
||||
get_param_names_mapping(model.param_names_mapping),
|
||||
valid_target_names=set(model.state_dict()),
|
||||
)
|
||||
model.load_state_dict(mapped, strict=True)
|
||||
return model.eval()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# DiT forward parity
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _reference_forward(dit, x, time, *, scale_factor, motion_score=None):
|
||||
"""Run the packed reference DiT on a batched ``[B, T, H, W, C]`` input."""
|
||||
batch, frames, height, width, channels = x.shape
|
||||
packed = x.reshape(batch * frames, height, width, channels)
|
||||
visual_cu = frames * torch.arange(batch + 1, dtype=torch.int32)
|
||||
text_cu = torch.zeros(batch + 1, dtype=torch.int32)
|
||||
rope_pos = [
|
||||
torch.cat([torch.arange(frames) for _ in range(batch)]),
|
||||
torch.arange(height // dit.patch_size[1]),
|
||||
torch.arange(width // dit.patch_size[2]),
|
||||
]
|
||||
out = dit(
|
||||
packed,
|
||||
torch.zeros(0, 1),
|
||||
torch.zeros(batch, 1),
|
||||
time,
|
||||
visual_cu,
|
||||
text_cu,
|
||||
rope_pos,
|
||||
torch.zeros(0, dtype=torch.long),
|
||||
scale_factor=scale_factor,
|
||||
motion_score=motion_score,
|
||||
)
|
||||
return out.reshape(batch, frames, *out.shape[1:]) if out.dim() == 4 else out
|
||||
|
||||
|
||||
def _port_forward(model, x, time, *, scale_factor, motion_score=None):
|
||||
batch, frames, height, width, _ = x.shape
|
||||
rope_pos = [
|
||||
torch.arange(frames),
|
||||
torch.arange(height // model.patch_size[1]),
|
||||
torch.arange(width // model.patch_size[2]),
|
||||
]
|
||||
return model(
|
||||
x,
|
||||
time,
|
||||
rope_pos,
|
||||
scale_factor=scale_factor,
|
||||
motion_score=motion_score,
|
||||
)
|
||||
|
||||
|
||||
def _random_latent(
|
||||
width_channels: int, *, batch=2, frames=3, height=4, width=6, seed=0
|
||||
):
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
return torch.randn(
|
||||
batch, frames, height, width, width_channels, generator=generator
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("piflow", [TINY_PIFLOW, None], ids=["dx_head", "plain_head"])
|
||||
def test_dit_forward_matches_reference(piflow):
|
||||
"""Guards the block math, RoPE, modulation, time bias and the DX head layout.
|
||||
|
||||
The port is batched ``[B, T, H, W, C]`` while the reference packs samples with
|
||||
``cu_seqlens``; different per-sample times make a wrong batching/broadcast visible.
|
||||
"""
|
||||
reference = build_reference_dit(piflow=piflow)
|
||||
model = build_port_dit(
|
||||
reference, piflow=piflow, overrides={"instruct_type": "noise"}
|
||||
)
|
||||
x = _random_latent(4)
|
||||
time = torch.tensor([300.0, 850.0])
|
||||
scale = (1.0, 2.0, 2.0)
|
||||
with torch.no_grad():
|
||||
expected = _reference_forward(reference, x, time, scale_factor=scale)
|
||||
actual = _port_forward(model, x, time, scale_factor=scale)
|
||||
n_grid = 1 if piflow is None else piflow["n_grid"]
|
||||
if piflow is not None:
|
||||
# reference DX wrapper already splits the grids: [B*T, n_grid, H, W, C]
|
||||
expected = expected.movedim(1, -2).reshape(*actual.shape)
|
||||
assert actual.shape[-1] == 4 * n_grid
|
||||
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
|
||||
|
||||
|
||||
def test_dit_forward_matches_reference_with_motion_score_and_wide_input():
|
||||
"""Guards the optional motion embedding and the ``2 * in + 1`` conditioned input."""
|
||||
cfg = dict(
|
||||
TINY_DIT_CFG,
|
||||
use_motion_score=True,
|
||||
instruct_type="hybrid_anchor",
|
||||
use_lq_noise_cond=True,
|
||||
)
|
||||
reference = build_reference_dit(piflow=TINY_PIFLOW, cfg=cfg)
|
||||
model = build_port_dit(
|
||||
reference,
|
||||
cfg=cfg,
|
||||
piflow=TINY_PIFLOW,
|
||||
overrides={"instruct_type": "noise", "visual_cond": True},
|
||||
)
|
||||
reference.instruct_type = "noise"
|
||||
reference.visual_cond = True
|
||||
x = _random_latent(2 * 4 + 1)
|
||||
time = torch.tensor([120.0, 999.0])
|
||||
motion = torch.full((1,), 900.0)
|
||||
with torch.no_grad():
|
||||
expected = _reference_forward(
|
||||
reference, x, time, scale_factor=(1.0, 1.0, 1.0), motion_score=motion
|
||||
)
|
||||
actual = _port_forward(
|
||||
model, x, time, scale_factor=(1.0, 1.0, 1.0), motion_score=motion
|
||||
)
|
||||
without_motion = _port_forward(model, x, time, scale_factor=(1.0, 1.0, 1.0))
|
||||
expected = expected.movedim(1, -2).reshape(*actual.shape)
|
||||
torch.testing.assert_close(actual, expected, rtol=1e-5, atol=1e-5)
|
||||
assert not torch.allclose(actual, without_motion)
|
||||
|
||||
|
||||
def test_state_dict_keys_equal_mapped_reference_keys():
|
||||
"""Guards the module tree / key mapping: extra or missing keys would break real checkpoints."""
|
||||
reference = build_reference_dit(piflow=TINY_PIFLOW)
|
||||
model = build_port_dit(reference, piflow=TINY_PIFLOW, overrides={})
|
||||
mapped, _ = hf_to_custom_state_dict(
|
||||
dict(reference.state_dict()),
|
||||
get_param_names_mapping(model.param_names_mapping),
|
||||
)
|
||||
assert set(mapped) == set(model.state_dict())
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Sampler loops (pi-Flow / Euler) against the reference generate functions
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _reference_loop_inputs(x):
|
||||
batch, frames, height, width, _ = x.shape
|
||||
visual_cu = frames * torch.arange(batch + 1, dtype=torch.int32)
|
||||
text = {"text_embeds": torch.zeros(0, 1), "pooled_embed": torch.zeros(batch, 1)}
|
||||
text_cu = torch.zeros(batch + 1, dtype=torch.int32)
|
||||
rope_pos = [
|
||||
torch.cat([torch.arange(frames) for _ in range(batch)]),
|
||||
torch.arange(height // 2),
|
||||
torch.arange(width // 2),
|
||||
]
|
||||
return visual_cu, text, text_cu, rope_pos, torch.zeros(0, dtype=torch.long)
|
||||
|
||||
|
||||
def _port_dit_fn(model, x, scale_factor, use_motion_score=False):
|
||||
return make_dit_fn(
|
||||
model,
|
||||
latent_frames_hw=tuple(x.shape[1:4]),
|
||||
patch_size=model.patch_size,
|
||||
scale_factor=scale_factor,
|
||||
use_motion_score=use_motion_score,
|
||||
)
|
||||
|
||||
|
||||
def test_piflow_loop_matches_reference_piflow_generate():
|
||||
"""Guards the segment schedule, DXPolicy grid layout and rollout on a batch of tiles.
|
||||
|
||||
The reference packs the tiles along time with one per-frame sigma; the port batches
|
||||
``[B, T, H, W, C]``. Same weights and same start latent, fp32 on CPU.
|
||||
|
||||
The port no longer re-derives the pi-Flow segment schedule itself (``latents.piflow_schedule``
|
||||
/ ``sampling.denoise_piflow`` are gone): it drives a real ``PiflowScheduler`` object through
|
||||
``denoise_with_scheduler`` instead, which the analysis in commit d8d0e79f's port (and the
|
||||
``PiflowScheduler._policy_step`` implementation itself) shows reproduces the exact same
|
||||
per-segment ``(raw_src, seg, raw_dst, sigma_src)`` schedule as the removed pure function did.
|
||||
"""
|
||||
from kandinsky_sr.core.algo.piflow_sampler import piflow_generate
|
||||
|
||||
reference = build_reference_dit(piflow=TINY_PIFLOW)
|
||||
model = build_port_dit(
|
||||
reference, piflow=TINY_PIFLOW, overrides={"instruct_type": "noise"}
|
||||
)
|
||||
x = _random_latent(4, seed=5)
|
||||
scale = (1.0, 2.0, 2.0)
|
||||
visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x)
|
||||
packed = x.reshape(-1, *x.shape[2:]).clone()
|
||||
scheduler = PiflowScheduler(**TINY_PIFLOW)
|
||||
scheduler.set_timesteps(TINY_PIFLOW["nfe"], device="cpu")
|
||||
with torch.no_grad():
|
||||
expected = piflow_generate(
|
||||
packed,
|
||||
reference,
|
||||
text,
|
||||
visual_cu,
|
||||
text_cu,
|
||||
rope_pos,
|
||||
text_rope,
|
||||
scale,
|
||||
out_dim=4,
|
||||
start_timestep=1.0,
|
||||
device="cpu",
|
||||
**TINY_PIFLOW,
|
||||
)
|
||||
actual = denoise_with_scheduler(
|
||||
x.clone(),
|
||||
_port_dit_fn(model, x, scale),
|
||||
scheduler,
|
||||
channels=4,
|
||||
is_piflow=True,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"capped", [False, True], ids=["full_range", "capped_noise_start"]
|
||||
)
|
||||
def test_euler_loop_matches_reference_generate(capped):
|
||||
"""Guards the warped timesteps, per-step deltas and (for a wide input) the in-place
|
||||
update of the first channels only, incl. the ``cap_noise_timestep`` start time.
|
||||
|
||||
The reference's ``generate(..., 5, ...)`` counts timestep *grid points* (5 points -> 4
|
||||
Euler steps); the port's own convention counts DiT *calls* directly, so the equivalent
|
||||
port call is ``num_inference_steps=4``. Both grids are the same 5-point warped-linspace
|
||||
array (``FlowMatchEulerDiscreteScheduler.set_timesteps`` applies the configured ``shift``
|
||||
warp to the 4 unwarped sigmas this test passes in, then appends the terminal 0 itself --
|
||||
algebraically the same 5th point the reference's own ``warp(0) == 0`` produces), so the 4
|
||||
step-to-step deltas ``denoise_with_scheduler`` consumes via ``scheduler.step`` are bit-for-
|
||||
bit the 4 deltas the old, removed ``denoise_euler`` consumed from that same array.
|
||||
"""
|
||||
from kandinsky_sr.core.algo.utils import generate
|
||||
|
||||
cfg = dict(TINY_DIT_CFG, use_motion_score=True, instruct_type="hybrid_anchor")
|
||||
reference = build_reference_dit(piflow=None, cfg=cfg)
|
||||
model = build_port_dit(
|
||||
reference,
|
||||
cfg=cfg,
|
||||
piflow=None,
|
||||
overrides={"instruct_type": "noise", "visual_cond": True},
|
||||
)
|
||||
reference.instruct_type, reference.visual_cond = "noise", True
|
||||
x = _random_latent(2 * 4 + 1, seed=9)
|
||||
scale = (1.0, 1.5, 1.5)
|
||||
start = euler_start_timestep(
|
||||
cap_noise_timestep=capped, lq_noise_scale=0.7, instruct_type="noise"
|
||||
)
|
||||
assert start == (0.7 if capped else 1.0)
|
||||
visual_cu, text, text_cu, rope_pos, text_rope = _reference_loop_inputs(x)
|
||||
packed = x.reshape(-1, *x.shape[2:]).clone()
|
||||
num_inference_steps = 4 # the reference's 5 grid points == 4 actual Euler steps
|
||||
scheduler = FlowMatchEulerDiscreteScheduler(shift=5.0)
|
||||
sigmas = torch.linspace(start, 0.0, num_inference_steps + 1)[:-1].tolist()
|
||||
scheduler.set_timesteps(sigmas=sigmas, device="cpu")
|
||||
with torch.no_grad():
|
||||
expected = generate(
|
||||
packed,
|
||||
reference,
|
||||
"cpu",
|
||||
5,
|
||||
text,
|
||||
text,
|
||||
visual_cu,
|
||||
text_cu,
|
||||
text_cu,
|
||||
rope_pos,
|
||||
text_rope,
|
||||
text_rope,
|
||||
scale,
|
||||
1.0,
|
||||
5.0,
|
||||
start_timestep=start,
|
||||
)
|
||||
actual = denoise_with_scheduler(
|
||||
x.clone(),
|
||||
_port_dit_fn(model, x, scale, use_motion_score=True),
|
||||
scheduler,
|
||||
channels=4,
|
||||
is_piflow=False,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
actual, expected.reshape(*x.shape[:4], 4), rtol=1e-5, atol=1e-5
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Tiling / stitching against the reference helpers
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize(
|
||||
"frame_hw,scale", [((64, 96), 2), ((64, 64), 4), ((96, 64), 2), ((80, 144), 2)]
|
||||
)
|
||||
def test_tile_plan_and_stitch_match_reference(frame_hw, scale):
|
||||
"""Guards the tile grid (aspect -> base resolution -> even grid) and that the
|
||||
frame-chunked stitch equals one reference stitch over the whole clip (T=19, not a
|
||||
multiple of the chunk size)."""
|
||||
from kandinsky_sr.core.algo.tiling_utils import stitch_tiles_hanning
|
||||
from kandinsky_sr.pipeline.sr_pipeline import _tile_geometry
|
||||
|
||||
height, width = frame_hw
|
||||
plan = plan_tiles(
|
||||
frame_hw=frame_hw,
|
||||
visual_size=512,
|
||||
tiling_scale=scale,
|
||||
tile_min_overlap=0.2,
|
||||
resolutions=TINY_RESOLUTIONS,
|
||||
)
|
||||
base, tile_hw, grid = _tile_geometry(
|
||||
height, width, 512, scale, 0.25, 16, "even", 0.2
|
||||
)
|
||||
assert (plan.base_hw, plan.tile_hw) == (base, tile_hw)
|
||||
assert tuple(plan.pixel_grid) == tuple(grid)
|
||||
generator = torch.Generator().manual_seed(3)
|
||||
tiles = [
|
||||
torch.randint(0, 256, (3, 19, *base), dtype=torch.uint8, generator=generator)
|
||||
for _ in range(grid.total_tiles)
|
||||
]
|
||||
expected = (
|
||||
stitch_tiles_hanning(
|
||||
[t.float() for t in tiles], grid, height, width, scale=scale
|
||||
)
|
||||
.clamp(0, 255)
|
||||
.to(torch.uint8)
|
||||
)
|
||||
assert torch.equal(stitch_tiles(tiles, plan), expected)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# End to end: pure orchestration vs kandinsky_sr.pipeline.stages.run_tiled_sr*
|
||||
# --------------------------------------------------------------------------- #
|
||||
def build_reference_lu_bank(target_scales=("2x", "4x"), seed=0):
|
||||
"""Reference LU bank with random weights and its bundle-style ``models`` config."""
|
||||
import pydantic
|
||||
from kandinsky_sr.core.algo.latent_upscaler import LatentUpscalerBank
|
||||
from kandinsky_sr.core.components.latent_upscaler.config import ModelConfig
|
||||
from kandinsky_sr.core.components.latent_upscaler.model.factory import (
|
||||
build_upsampler,
|
||||
)
|
||||
|
||||
models = [
|
||||
{"target_scale": scale, "model": copy.deepcopy(TINY_LU_MODEL)}
|
||||
for scale in target_scales
|
||||
]
|
||||
adapter = pydantic.TypeAdapter(ModelConfig)
|
||||
bank = LatentUpscalerBank()
|
||||
for index, spec in enumerate(models):
|
||||
upsampler = build_upsampler(adapter.validate_python(spec["model"])).eval()
|
||||
randomize(upsampler, seed + 10 + index)
|
||||
upsampler.target_scale = spec["target_scale"]
|
||||
upsampler.scaling_factor = TINY_SCALING_FACTOR
|
||||
bank[spec["target_scale"]] = upsampler
|
||||
return bank, models
|
||||
|
||||
|
||||
def build_stacks(*, piflow, use_lu):
|
||||
"""(reference SRComponents, port dict) sharing identical random weights."""
|
||||
from kandinsky_sr.pipeline.sr_pipeline import SRComponents, SRParams
|
||||
|
||||
torch.manual_seed(0)
|
||||
ref_vae = CachedCausalVAE(
|
||||
encoder_conf=OmegaConf.create(dict(TINY_KVAE_ENC)),
|
||||
decoder_conf=OmegaConf.create(dict(TINY_KVAE_DEC)),
|
||||
).eval()
|
||||
randomize(ref_vae, 1, std=0.1)
|
||||
ref_vae.config = SimpleNamespace(scaling_factor=TINY_SCALING_FACTOR)
|
||||
ref_dit = build_reference_dit(piflow=piflow, seed=2)
|
||||
ref_bank, models = build_reference_lu_bank() if use_lu else (None, None)
|
||||
reference = SRComponents(
|
||||
dit=ref_dit,
|
||||
vae=ref_vae,
|
||||
latent_upscaler=ref_bank,
|
||||
sr_params=SRParams(**TINY_SR_PARAMS),
|
||||
spatial_factor=16,
|
||||
)
|
||||
|
||||
vae_config = Kandinsky6SRVAEConfig()
|
||||
vae_config.update_model_arch(
|
||||
dict(
|
||||
vae_type="video-kvae",
|
||||
encoder_config=dict(TINY_KVAE_ENC),
|
||||
decoder_config=dict(TINY_KVAE_DEC),
|
||||
scaling_factor=TINY_SCALING_FACTOR,
|
||||
spatial_factor=16,
|
||||
temporal_factor=4,
|
||||
)
|
||||
)
|
||||
vae = Kandinsky6SRVAE(vae_config)
|
||||
# unprefixed reference keys = the official layout, mapped by the wrapper's load hook
|
||||
vae.load_state_dict(dict(ref_vae.state_dict()), strict=True)
|
||||
port = {
|
||||
"vae": vae.eval(),
|
||||
"dit": build_port_dit(
|
||||
ref_dit, piflow=piflow, overrides={"instruct_type": "noise"}
|
||||
),
|
||||
"bank": None,
|
||||
}
|
||||
if use_lu:
|
||||
bank = Kandinsky6SRLatentUpscalerBank(models, TINY_SCALING_FACTOR)
|
||||
converted = {}
|
||||
scale_indices = {"2x": 0, "4x": 1}
|
||||
for key, value in ref_bank.state_dict().items():
|
||||
scale, suffix = key.split(".", 1)
|
||||
suffix = suffix.replace("private_", "")
|
||||
suffix = re.sub(r"(output_proj\.)0\.", r"\1norm.", suffix)
|
||||
suffix = re.sub(r"(output_proj\.)2\.", r"\1conv.", suffix)
|
||||
converted[f"_models.{scale_indices[scale]}.{suffix}"] = value
|
||||
bank.load_state_dict(
|
||||
converted,
|
||||
strict=True,
|
||||
)
|
||||
port["bank"] = bank.eval()
|
||||
return reference, port
|
||||
|
||||
|
||||
def run_reference(video, reference, *, scale, seed, tiles_batch_size, num_steps):
|
||||
from kandinsky_sr.core.algo.latent_upscaler import latent_upscaler_for_scale
|
||||
from kandinsky_sr.pipeline.sr_pipeline import (
|
||||
RunConfig,
|
||||
encode_lq_video_to_lr_latent,
|
||||
)
|
||||
from kandinsky_sr.pipeline.stages import run_tiled_sr, run_tiled_sr_from_pixels
|
||||
from kandinsky_sr.pipeline.upscale_utils import (
|
||||
pre_upscale_video,
|
||||
resolve_scale_request,
|
||||
)
|
||||
|
||||
tiling_scale, pre = resolve_scale_request(float(scale))
|
||||
if pre != 1.0:
|
||||
video = pre_upscale_video(video, pre, 16)
|
||||
run_config = RunConfig(
|
||||
device="cpu",
|
||||
num_steps=num_steps,
|
||||
seed=seed,
|
||||
overlap=0.25,
|
||||
tiles_batch_size=tiles_batch_size,
|
||||
resolution_scale=tiling_scale,
|
||||
)
|
||||
if latent_upscaler_for_scale(reference.latent_upscaler, tiling_scale) is not None:
|
||||
lr_latent = encode_lq_video_to_lr_latent(video, reference.vae, "cpu")
|
||||
return run_tiled_sr(lr_latent, reference, run_config)
|
||||
return run_tiled_sr_from_pixels(video, reference, run_config)
|
||||
|
||||
|
||||
def run_port(video, port, *, scale, seed, tiles_batch_size, num_steps):
|
||||
"""The stage logic (scale request, pre-upscale, specs) around the pure function.
|
||||
|
||||
``num_steps`` here is the reference's convention (timestep grid points); for a
|
||||
flow-matching (non pi-Flow) bundle the port's own ``num_inference_steps`` is
|
||||
``num_steps - 1`` (DiT calls), so it is translated once, here, at the one remaining
|
||||
boundary between the two conventions. A pi-Flow bundle ignores it either way (its
|
||||
scheduler's own ``nfe`` always wins), so no translation is needed for that case.
|
||||
"""
|
||||
tiling_scale, pre = tiling.resolve_scale_request(float(scale))
|
||||
if pre != 1.0:
|
||||
video = tiling.pre_upscale_video(video, pre, 16)
|
||||
arch = port["dit"].config
|
||||
port_num_steps = num_steps if arch.is_piflow else num_steps - 1
|
||||
spec = build_sampling_spec(
|
||||
arch=arch,
|
||||
tiling_scale=tiling_scale,
|
||||
seed=seed,
|
||||
num_steps=port_num_steps,
|
||||
tiles_batch_size=tiles_batch_size,
|
||||
tile_min_overlap=0.2,
|
||||
)
|
||||
bank = port["bank"]
|
||||
use_lu = bank is not None and bank.for_scale(tiling_scale) is not None
|
||||
return super_resolve(
|
||||
video,
|
||||
vae=port["vae"],
|
||||
scaling_factor=port["vae"].scaling_factor,
|
||||
dit=port["dit"],
|
||||
dit_spec=build_dit_spec(port["dit"]),
|
||||
spec=spec,
|
||||
scheduler=effective_scheduler(spec, None),
|
||||
device="cpu",
|
||||
upscale_fn=partial(bank.upscale, scale=tiling_scale) if use_lu else None,
|
||||
lu_dtype=module_dtype(bank) if use_lu else None,
|
||||
resolutions=TINY_RESOLUTIONS,
|
||||
)
|
||||
|
||||
|
||||
def _random_video(frames, height, width, seed=0):
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
return torch.randint(
|
||||
0, 256, (frames, 3, height, width), dtype=torch.uint8, generator=generator
|
||||
)
|
||||
|
||||
|
||||
E2E_CASES = [
|
||||
# scale, (H, W), tiles_batch_size
|
||||
(2, (64, 96), 1),
|
||||
(2, (64, 96), 3),
|
||||
(4, (64, 64), 2),
|
||||
(2.25, (64, 96), 1),
|
||||
]
|
||||
|
||||
|
||||
def _assert_uint8_close(actual, expected, *, max_diff: int = 1):
|
||||
assert actual.shape == expected.shape and actual.dtype == expected.dtype
|
||||
diff = (actual.int() - expected.int()).abs()
|
||||
exact = (diff == 0).float().mean().item()
|
||||
assert diff.max().item() <= max_diff and exact > 0.999, (diff.max().item(), exact)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_lu", [False, True], ids=["pixel_path", "lu_path"])
|
||||
@pytest.mark.parametrize("scale,hw,tiles_batch_size", E2E_CASES)
|
||||
def test_tiled_piflow_super_resolution_matches_reference(
|
||||
use_lu, scale, hw, tiles_batch_size
|
||||
):
|
||||
"""Guards the whole orchestration: scale request + pre-upscale, tile geometry, LU or
|
||||
tile encode, chunk seeding (``seed + first tile index``), pi-Flow loop, decode with
|
||||
truncation to uint8 and Hann stitching, against ``run_tiled_sr`` /
|
||||
``run_tiled_sr_from_pixels`` with identical random weights.
|
||||
|
||||
``max_diff=2`` (not the default 1): the reference's ``piflow_generate`` is the
|
||||
training-only inline schedule, which clamps the *final* segment's ``raw_dst`` to
|
||||
0.0 (dead code on every real checkpoint - see GAP 2 / ``latents.piflow_schedule``).
|
||||
The port now deliberately matches the real ``PiflowScheduler._policy_step``
|
||||
convention (``raw_dst=eps``) instead, which real checkpoints always run through.
|
||||
That is a ~1e-6 raw-timestep difference in the last segment only, so it shows up
|
||||
here as at most a 1-LSB-wider uint8 rounding tie on top of the pre-existing
|
||||
rounding-boundary noise (still >99.99% exact pixels), not as a structural bug.
|
||||
"""
|
||||
reference, port = build_stacks(piflow=TINY_PIFLOW, use_lu=use_lu)
|
||||
video = _random_video(9, *hw)
|
||||
kwargs = dict(scale=scale, seed=42, tiles_batch_size=tiles_batch_size, num_steps=5)
|
||||
expected = run_reference(video, reference, **kwargs)
|
||||
actual = run_port(video, port, **kwargs)
|
||||
_assert_uint8_close(actual, expected, max_diff=2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_lu", [False, True], ids=["pixel_path", "lu_path"])
|
||||
def test_tiled_euler_super_resolution_matches_reference(use_lu):
|
||||
"""Same as above for a plain (non pi-Flow) DiT: the Euler loop with ``num_steps``."""
|
||||
reference, port = build_stacks(piflow=None, use_lu=use_lu)
|
||||
video = _random_video(9, 64, 96, seed=1)
|
||||
kwargs = dict(scale=2, seed=7, tiles_batch_size=2, num_steps=4)
|
||||
expected = run_reference(video, reference, **kwargs)
|
||||
actual = run_port(video, port, **kwargs)
|
||||
_assert_uint8_close(actual, expected)
|
||||
|
||||
|
||||
def test_seed_and_chunking_change_the_result_like_the_reference():
|
||||
"""The initial noise of chunk k is seeded with ``seed + chunk start``: another seed or
|
||||
another chunk size must change the output (a port that ignored them would not)."""
|
||||
_, port = build_stacks(piflow=TINY_PIFLOW, use_lu=False)
|
||||
video = _random_video(9, 64, 96, seed=2)
|
||||
base = run_port(video, port, scale=2, seed=42, tiles_batch_size=1, num_steps=5)
|
||||
assert not torch.equal(
|
||||
base, run_port(video, port, scale=2, seed=43, tiles_batch_size=1, num_steps=5)
|
||||
)
|
||||
assert not torch.equal(
|
||||
base, run_port(video, port, scale=2, seed=42, tiles_batch_size=4, num_steps=5)
|
||||
)
|
||||
@@ -0,0 +1,253 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Validate scheduler selection, DiT call counts and incompatible head rejection."""
|
||||
|
||||
import os
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
from kandinsky6_sr_release_configs import (
|
||||
DISTILLED_SCHEDULER_CONFIG,
|
||||
DISTILLED_TRANSFORMER_CONFIG,
|
||||
FLOW_SCHEDULER_CONFIG,
|
||||
FLOW_TRANSFORMER_CONFIG,
|
||||
)
|
||||
from kandinsky6_sr_tiny_components import (
|
||||
TINY_DIT,
|
||||
build_components,
|
||||
make_request,
|
||||
make_stage,
|
||||
random_video,
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.kandinsky6_piflow import (
|
||||
PiflowScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.schedulers.scheduling_flow_match_euler_discrete import (
|
||||
FlowMatchEulerDiscreteScheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
|
||||
Kandinsky6SRDenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
|
||||
Kandinsky6SRLatentPrepStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_VIDEO_KEY,
|
||||
build_sampling_spec,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
RESOLUTIONS,
|
||||
)
|
||||
|
||||
LATENT_WIDTH = TINY_DIT["in_visual_dim"]
|
||||
N_GRID = (
|
||||
3 # the distilled bundle's own grid width, independent of TINY_DIT's legacy fields
|
||||
)
|
||||
NUM_TILES = 9 # the 64x96 clip below, cut into 32x48 tiles
|
||||
|
||||
|
||||
def official_dit(head_width):
|
||||
"""Tiny transformer config in the official layout: ``out_visual_dim`` is the TOTAL
|
||||
head width, and ``n_grid`` / ``piflow_*`` live in the scheduler config."""
|
||||
arch = {
|
||||
key: value
|
||||
for key, value in TINY_DIT.items()
|
||||
if key != "out_visual_dim" and not key.startswith(("n_grid", "piflow_", "sr_"))
|
||||
}
|
||||
return dict(
|
||||
arch,
|
||||
out_visual_dim=head_width,
|
||||
sr_params=dict(scale_factor={"512": [1.0, 1.0, 1.0]}, visual_size=[512]),
|
||||
)
|
||||
|
||||
|
||||
FLOW_DIT = official_dit(LATENT_WIDTH) # one latent per token: flow-matching head
|
||||
DX_DIT = official_dit(N_GRID * LATENT_WIDTH) # N_GRID latents per token: pi-Flow head
|
||||
|
||||
|
||||
def flow_scheduler():
|
||||
return FlowMatchEulerDiscreteScheduler(shift=5.0)
|
||||
|
||||
|
||||
def pi_flow_scheduler(*, nfe=2, n_grid=N_GRID):
|
||||
return PiflowScheduler(
|
||||
nfe=nfe,
|
||||
n_grid=n_grid,
|
||||
shift=5.0,
|
||||
eps=1e-6,
|
||||
final_step_size_scale=0.5,
|
||||
num_policy_substeps=8,
|
||||
)
|
||||
|
||||
|
||||
def release_scheduler(scheduler_cls, config):
|
||||
return scheduler_cls(**{k: v for k, v in config.items() if not k.startswith("_")})
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29510",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def tiny_resolutions(monkeypatch):
|
||||
monkeypatch.setitem(RESOLUTIONS, 512, [(64, 64), (64, 96), (96, 64)])
|
||||
|
||||
|
||||
class SamplingRun(NamedTuple):
|
||||
tiles: int
|
||||
dit_calls: int
|
||||
progress_totals: list
|
||||
|
||||
|
||||
def run_denoising(dit_arch, scheduler, *, num_steps=5):
|
||||
"""Run latent-prep + denoising (the LU-less pixel path) on the 9-tile clip and count
|
||||
the DiT calls next to the totals the denoising progress bar was opened with."""
|
||||
vae, dit, _, server_args = build_components((), dit_arch)
|
||||
dit_calls = []
|
||||
dit.register_forward_hook(lambda module, args, output: dit_calls.append(1))
|
||||
latent_prep = make_stage(
|
||||
Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, scheduler
|
||||
)
|
||||
denoise = make_stage(Kandinsky6SRDenoisingStage, server_args, dit, scheduler)
|
||||
progress_totals = []
|
||||
progress_bar = denoise.progress_bar
|
||||
|
||||
def spy_on_progress_bar(**kwargs):
|
||||
progress_totals.append(kwargs["total"])
|
||||
return progress_bar(**kwargs)
|
||||
|
||||
denoise.progress_bar = spy_on_progress_bar
|
||||
|
||||
batch = make_request(random_video(9, 64, 96), num_steps=num_steps)
|
||||
batch = latent_prep.forward(batch, server_args)
|
||||
tiles = len(batch.extra["kandinsky6_sr_chunks"])
|
||||
denoise.forward(batch, server_args)
|
||||
return SamplingRun(tiles, len(dit_calls), progress_totals)
|
||||
|
||||
|
||||
def sampling_spec(dit_arch, scheduler):
|
||||
config = Kandinsky6SRDitConfig()
|
||||
config.update_model_arch(dict(dit_arch))
|
||||
return build_sampling_spec(
|
||||
arch=config.arch_config,
|
||||
tiling_scale=2,
|
||||
seed=42,
|
||||
num_steps=5,
|
||||
tiles_batch_size=1,
|
||||
tile_min_overlap=0.2,
|
||||
scheduler=scheduler,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# DiT calls per tile
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize("nfe,num_steps", [(2, 3), (2, 5), (2, 9), (3, 5)])
|
||||
def test_pi_flow_scheduler_makes_nfe_calls_per_tile_whatever_num_inference_steps(
|
||||
nfe, num_steps
|
||||
):
|
||||
run = run_denoising(DX_DIT, pi_flow_scheduler(nfe=nfe), num_steps=num_steps)
|
||||
assert run.tiles == NUM_TILES
|
||||
assert run.dit_calls == NUM_TILES * nfe
|
||||
assert run.progress_totals == [NUM_TILES * nfe]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_steps,calls_per_tile", [(3, 3), (5, 5), (9, 9)])
|
||||
def test_flow_euler_scheduler_makes_num_inference_steps_calls_per_tile(
|
||||
num_steps, calls_per_tile
|
||||
):
|
||||
run = run_denoising(FLOW_DIT, flow_scheduler(), num_steps=num_steps)
|
||||
assert run.tiles == NUM_TILES
|
||||
assert run.dit_calls == NUM_TILES * calls_per_tile
|
||||
assert run.progress_totals == [NUM_TILES * calls_per_tile]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_steps", [3, 9])
|
||||
def test_legacy_config_with_its_own_pi_flow_fields_keeps_its_sampler(num_steps):
|
||||
"""A flat DiT config carries n_grid / piflow_* and needs no scheduler component: the
|
||||
denoising stage synthesizes a matching PiflowScheduler from those arch fields
|
||||
(run_spec.effective_scheduler) instead of leaving nothing to step."""
|
||||
run = run_denoising(TINY_DIT, None, num_steps=num_steps)
|
||||
assert run.dit_calls == NUM_TILES * TINY_DIT["piflow_nfe"]
|
||||
assert run.progress_totals == [run.dit_calls]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Samplers that do not fit the DiT head
|
||||
# --------------------------------------------------------------------------- #
|
||||
@pytest.mark.parametrize(
|
||||
"scheduler",
|
||||
[flow_scheduler(), PiflowScheduler(n_grid=N_GRID, nfe=None), None],
|
||||
ids=["flow_euler_scheduler", "pi_flow_scheduler_without_nfe", "no_scheduler"],
|
||||
)
|
||||
def test_dx_head_never_runs_flow_euler(scheduler):
|
||||
with pytest.raises(ValueError, match="flow-Euler"):
|
||||
sampling_spec(DX_DIT, scheduler)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dit_arch,n_grid,needed",
|
||||
[
|
||||
(FLOW_DIT, N_GRID, "3 x 4 = 12"),
|
||||
(DX_DIT, 2, "2 x 4 = 8"),
|
||||
(DX_DIT, 4, "4 x 4 = 16"),
|
||||
],
|
||||
ids=["single_head", "too_few_grids", "too_many_grids"],
|
||||
)
|
||||
def test_pi_flow_needs_a_head_of_n_grid_latents(dit_arch, n_grid, needed):
|
||||
with pytest.raises(ValueError, match=f"needs a {needed} channel DX head"):
|
||||
sampling_spec(dit_arch, pi_flow_scheduler(n_grid=n_grid))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("nfe", [0, -1])
|
||||
def test_pi_flow_needs_at_least_one_call_per_tile(nfe):
|
||||
with pytest.raises(ValueError, match="nfe >= 1"):
|
||||
sampling_spec(DX_DIT, pi_flow_scheduler(nfe=nfe))
|
||||
with pytest.raises(ValueError, match="nfe >= 1"):
|
||||
sampling_spec(dict(TINY_DIT, piflow_nfe=nfe), None)
|
||||
|
||||
|
||||
def test_release_transformer_and_scheduler_of_different_repos_are_rejected():
|
||||
"""The real configs of the two official repos: their transformers differ only in
|
||||
the head width, so one repo's transformer with the other's scheduler must fail."""
|
||||
flow = release_scheduler(FlowMatchEulerDiscreteScheduler, FLOW_SCHEDULER_CONFIG)
|
||||
distilled = release_scheduler(PiflowScheduler, DISTILLED_SCHEDULER_CONFIG)
|
||||
|
||||
assert sampling_spec(FLOW_TRANSFORMER_CONFIG, flow).steps_per_chunk == 5
|
||||
assert sampling_spec(DISTILLED_TRANSFORMER_CONFIG, distilled).steps_per_chunk == 2
|
||||
with pytest.raises(ValueError, match="640 channels wide.*flow-Euler"):
|
||||
sampling_spec(DISTILLED_TRANSFORMER_CONFIG, flow)
|
||||
with pytest.raises(ValueError, match="needs a 10 x 64 = 640 channel DX head"):
|
||||
sampling_spec(FLOW_TRANSFORMER_CONFIG, distilled)
|
||||
|
||||
|
||||
def test_a_mismatched_bundle_fails_before_any_tile_is_processed():
|
||||
vae, dit, _, server_args = build_components((), DX_DIT)
|
||||
dit_calls = []
|
||||
dit.register_forward_hook(lambda module, args, output: dit_calls.append(1))
|
||||
latent_prep = make_stage(
|
||||
Kandinsky6SRLatentPrepStage, server_args, vae, dit, None, flow_scheduler()
|
||||
)
|
||||
batch = make_request(random_video(9, 64, 96))
|
||||
|
||||
with pytest.raises(ValueError, match="flow-Euler"):
|
||||
latent_prep.forward(batch, server_args)
|
||||
assert dit_calls == []
|
||||
assert SR_VIDEO_KEY in batch.extra # the clip was never cut into tiles
|
||||
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Verify SR token sharding, TP weight loading, and repeated distributed forwards."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from sglang.multimodal_gen.configs.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRDitConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.kandinsky6_sr import (
|
||||
Kandinsky6SRPipelineConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.attention.selector import (
|
||||
global_force_attn_backend,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.models.dits.kandinsky6_sr import (
|
||||
Kandinsky6SRTransformer3DModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import AttentionBackendEnum
|
||||
from sglang.multimodal_gen.runtime.server_args import set_global_server_args
|
||||
from sglang.multimodal_gen.runtime.utils.precision import set_mixed_precision_policy
|
||||
from sglang.multimodal_gen.test.unit.kandinsky6_sr_tiny_components import TINY_DIT
|
||||
from sglang.srt.utils.network import get_free_port_below_ephemeral
|
||||
|
||||
|
||||
def _forward(rank, world_size, tp_size, ring, port, output_dir):
|
||||
os.environ.update(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT=str(port),
|
||||
RANK=str(rank),
|
||||
LOCAL_RANK=str(rank),
|
||||
WORLD_SIZE=str(world_size),
|
||||
)
|
||||
torch.cuda.set_device(rank)
|
||||
set_mixed_precision_policy(torch.bfloat16, torch.float32)
|
||||
global_force_attn_backend(AttentionBackendEnum.FA)
|
||||
set_global_server_args(
|
||||
SimpleNamespace(
|
||||
attention_backend="fa",
|
||||
attention_backend_config=None,
|
||||
kv_gather_degree=1,
|
||||
sp_split_auto=False,
|
||||
pipeline_config=Kandinsky6SRPipelineConfig(),
|
||||
)
|
||||
)
|
||||
sp_size = world_size // tp_size
|
||||
maybe_init_distributed_environment_and_model_parallel(
|
||||
tp_size=tp_size,
|
||||
sp_size=sp_size,
|
||||
ulysses_degree=1 if ring else sp_size,
|
||||
ring_degree=sp_size if ring else 1,
|
||||
)
|
||||
config = Kandinsky6SRDitConfig()
|
||||
config.update_model_arch(
|
||||
TINY_DIT | dict(model_dim=512, time_dim=32, ff_dim=1024, axes_dims=[32, 48, 48])
|
||||
)
|
||||
model = Kandinsky6SRTransformer3DModel(config, {}).eval()
|
||||
generator = torch.Generator().manual_seed(7)
|
||||
directory = Path(output_dir)
|
||||
with torch.no_grad():
|
||||
if world_size == 1:
|
||||
for parameter in model.parameters():
|
||||
parameter.copy_(
|
||||
torch.randn(parameter.shape, generator=generator) * 0.02
|
||||
)
|
||||
torch.save(model.state_dict(), directory / "weights.pt")
|
||||
else:
|
||||
weights = torch.load(directory / "weights.pt", weights_only=True)
|
||||
for name, parameter in model.named_parameters():
|
||||
loader = parameter.__dict__.get("weight_loader")
|
||||
if loader is None:
|
||||
parameter.copy_(weights[name])
|
||||
else:
|
||||
loader(parameter, weights[name])
|
||||
model.to(device=rank, dtype=torch.bfloat16)
|
||||
assert model.visual_transformer_blocks[0].self_attention.local_num_heads == (
|
||||
4 // tp_size
|
||||
)
|
||||
|
||||
observed = []
|
||||
|
||||
def record_shard(module, args):
|
||||
visual, _, rope, _ = args
|
||||
observed.append((visual.shape[0], visual.shape[1], rope.shape[1]))
|
||||
|
||||
handle = model.visual_transformer_blocks[0].register_forward_pre_hook(record_shard)
|
||||
first = None
|
||||
for batch_size, frames in ((1, 2), (2, 3), (1, 2)):
|
||||
generator.manual_seed(frames)
|
||||
inputs = torch.randn(batch_size, frames, 6, 6, 4, generator=generator).to(
|
||||
device=rank, dtype=torch.bfloat16
|
||||
)
|
||||
with (
|
||||
torch.no_grad(),
|
||||
set_forward_context(current_timestep=0, attn_metadata=None),
|
||||
):
|
||||
actual = model(
|
||||
inputs,
|
||||
torch.full((batch_size,), 500.0, device=rank),
|
||||
[torch.arange(n, device=rank) for n in (frames, 3, 3)],
|
||||
)
|
||||
local_len = (frames * 3 * 3 + sp_size - 1) // sp_size
|
||||
assert observed[-1] == (batch_size, local_len, local_len)
|
||||
assert actual.shape == (batch_size, frames, 6, 6, 12)
|
||||
assert torch.isfinite(actual).all()
|
||||
if frames == 2:
|
||||
if first is None:
|
||||
first = actual.clone()
|
||||
else:
|
||||
torch.testing.assert_close(actual, first, rtol=0, atol=0)
|
||||
path = directory / f"{batch_size}-{frames}.pt"
|
||||
if world_size == 1:
|
||||
torch.save(actual.cpu(), path)
|
||||
else:
|
||||
expected = torch.load(path, weights_only=True, map_location=f"cuda:{rank}")
|
||||
torch.testing.assert_close(actual, expected, rtol=0.02, atol=0.002)
|
||||
handle.remove()
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def _launch(world_size, tp_size, ring, output_dir):
|
||||
mp.spawn(
|
||||
_forward,
|
||||
args=(world_size, tp_size, ring, get_free_port_below_ephemeral(), output_dir),
|
||||
nprocs=world_size,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(torch.cuda.device_count() < 2, reason="requires two CUDA GPUs")
|
||||
def test_sr_two_gpus(tmp_path):
|
||||
_launch(1, 1, False, str(tmp_path))
|
||||
_launch(2, 1, False, str(tmp_path))
|
||||
_launch(2, 1, True, str(tmp_path))
|
||||
_launch(2, 2, False, str(tmp_path))
|
||||
|
||||
|
||||
@pytest.mark.skipif(torch.cuda.device_count() < 4, reason="requires four CUDA GPUs")
|
||||
def test_sr_four_gpus(tmp_path):
|
||||
_launch(1, 1, False, str(tmp_path))
|
||||
for tp_size in (1, 2, 4):
|
||||
_launch(4, tp_size, False, str(tmp_path))
|
||||
_launch(4, 2, True, str(tmp_path))
|
||||
@@ -0,0 +1,353 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Native SR stage-chain parity, component residency and video/audio hand-offs."""
|
||||
|
||||
import copy
|
||||
import os
|
||||
from functools import partial
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from kandinsky6_sr_tiny_components import (
|
||||
TINY_LU_MODEL,
|
||||
build_components,
|
||||
make_request,
|
||||
make_stage,
|
||||
random_video,
|
||||
super_resolve,
|
||||
)
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.disaggregation.roles import RoleType
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
|
||||
maybe_init_distributed_environment_and_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.upsampler.kandinsky6_sr_latent_upscaler import (
|
||||
Kandinsky6SRLatentUpscalerBank,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines.kandinsky6_sr_pipeline import (
|
||||
Kandinsky6SRPipeline,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.decode_stage import (
|
||||
Kandinsky6SRDecodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.denoising_stage import (
|
||||
Kandinsky6SRDenoisingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.encode_stage import (
|
||||
Kandinsky6SREncodeStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
|
||||
WARMUP_CLIP_FRAMES,
|
||||
WARMUP_CLIP_HW,
|
||||
Kandinsky6SRInputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.latent_prep_stage import (
|
||||
Kandinsky6SRLatentPrepStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.output_stage import (
|
||||
Kandinsky6SROutputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_LR_LATENT_KEY,
|
||||
SR_PLAN_KEY,
|
||||
SR_REQUESTED_HW_KEY,
|
||||
SR_TILES_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
build_dit_spec,
|
||||
build_sampling_spec,
|
||||
effective_scheduler,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
module_dtype,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
TilePlan,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
RESOLUTIONS,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def single_process_model_parallel():
|
||||
if not model_parallel_is_initialized():
|
||||
for key, value in dict(
|
||||
MASTER_ADDR="127.0.0.1",
|
||||
MASTER_PORT="29509",
|
||||
RANK="0",
|
||||
LOCAL_RANK="0",
|
||||
WORLD_SIZE="1",
|
||||
).items():
|
||||
os.environ.setdefault(key, value)
|
||||
maybe_init_distributed_environment_and_model_parallel(tp_size=1, sp_size=1)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def tiny_resolutions(monkeypatch):
|
||||
monkeypatch.setitem(RESOLUTIONS, 512, [(64, 64), (64, 96), (96, 64)])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bank_scales",
|
||||
[("2x", "4x"), (), ("4x",)],
|
||||
ids=["lu_path", "pixel_path", "pixel_fallback"],
|
||||
)
|
||||
def test_stage_chain_reproduces_the_pure_orchestration(bank_scales):
|
||||
"""encode -> latent-prep -> denoise -> decode -> output must give what
|
||||
``super_resolve`` gives on the same clip, for the LU path, the pixel path, and a bank
|
||||
that has no entry for the requested scale (which must fall back to the pixel path, not
|
||||
fail). The 64x128 clip holds 12 overlapping 32x48 tiles, denoised in chunks of 5, 5 and
|
||||
2."""
|
||||
vae, dit, bank, server_args = build_components(bank_scales)
|
||||
video = random_video(9, 64, 128)
|
||||
batch = make_request(video, tiles_batch_size=5)
|
||||
|
||||
pipeline = object.__new__(Kandinsky6SRPipeline)
|
||||
pipeline.modules = dict(vae=vae, transformer=dit, scheduler=None)
|
||||
if bank is not None:
|
||||
pipeline.modules["latent_upscaler"] = bank
|
||||
pipeline._stages, pipeline._stage_name_mapping = [], {}
|
||||
pipeline._disagg_role = RoleType.MONOLITHIC
|
||||
pipeline.create_pipeline_stages(server_args)
|
||||
for stage in pipeline.stages:
|
||||
stage.server_args = server_args
|
||||
input_stage, encode, latent_prep, denoise, decode, output = pipeline.stages
|
||||
assert [type(stage) for stage in pipeline.stages] == [
|
||||
Kandinsky6SRInputStage,
|
||||
Kandinsky6SREncodeStage,
|
||||
Kandinsky6SRLatentPrepStage,
|
||||
Kandinsky6SRDenoisingStage,
|
||||
Kandinsky6SRDecodeStage,
|
||||
Kandinsky6SROutputStage,
|
||||
]
|
||||
assert [
|
||||
(u.component_name, u.start_at_stage_entry)
|
||||
for u in encode.component_uses(server_args)
|
||||
] == [("vae", False)]
|
||||
expected_phases = [("vae", "encode_tiles")]
|
||||
if bank is not None:
|
||||
expected_phases.insert(0, ("latent_upscaler", "upscale_tiles"))
|
||||
assert [
|
||||
(u.component_name, u.phase) for u in latent_prep.component_uses(server_args)
|
||||
] == expected_phases
|
||||
(dit_use,) = denoise.component_uses(server_args)
|
||||
assert (dit_use.component_name, dit_use.phase) == ("transformer", "denoise_tiles")
|
||||
assert dit_use.memory_intensive and dit_use.target_dtype == torch.bfloat16
|
||||
assert [
|
||||
(u.component_name, u.phase) for u in decode.component_uses(server_args)
|
||||
] == [("vae", "decode_tiles")]
|
||||
|
||||
batch = encode.forward(batch, server_args)
|
||||
assert (SR_LR_LATENT_KEY in batch.extra) == ("2x" in bank_scales)
|
||||
batch = latent_prep.forward(batch, server_args)
|
||||
assert SR_LR_LATENT_KEY not in batch.extra and SR_VIDEO_KEY not in batch.extra
|
||||
assert len(batch.extra["kandinsky6_sr_chunks"]) == 3 # 12 tiles / 5 per chunk
|
||||
assert isinstance(batch.extra[SR_PLAN_KEY], TilePlan)
|
||||
batch = denoise.forward(batch, server_args)
|
||||
batch = decode.forward(batch, server_args)
|
||||
assert len(batch.extra[SR_TILES_KEY]) == 12
|
||||
result = output.forward(batch, server_args)
|
||||
|
||||
arch = dit.config
|
||||
use_lu = "2x" in bank_scales
|
||||
spec = build_sampling_spec(
|
||||
arch=arch,
|
||||
tiling_scale=2,
|
||||
seed=42,
|
||||
num_steps=5,
|
||||
tiles_batch_size=5,
|
||||
tile_min_overlap=0.2,
|
||||
)
|
||||
expected = super_resolve(
|
||||
video,
|
||||
vae=vae,
|
||||
scaling_factor=vae.scaling_factor,
|
||||
dit=dit,
|
||||
dit_spec=build_dit_spec(dit),
|
||||
spec=spec,
|
||||
scheduler=effective_scheduler(spec, None),
|
||||
device=next(dit.parameters()).device,
|
||||
upscale_fn=partial(bank.upscale, scale=2) if use_lu else None,
|
||||
lu_dtype=module_dtype(bank) if use_lu else None,
|
||||
)
|
||||
assert isinstance(result, OutputBatch)
|
||||
assert result.output.dtype == torch.float16
|
||||
assert result.output.shape == (1, 3, 9, 128, 256)
|
||||
restored = (result.output[0].float() * 255).to(torch.uint8)
|
||||
assert torch.equal(restored, expected)
|
||||
assert (batch.height, batch.width) == (128, 256)
|
||||
assert SR_TILES_KEY not in batch.extra and SR_PLAN_KEY not in batch.extra
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"role", [RoleType.ENCODER, RoleType.DENOISER, RoleType.DECODER]
|
||||
)
|
||||
def test_sr_rejects_disaggregation(role):
|
||||
pipeline = object.__new__(Kandinsky6SRPipeline)
|
||||
with pytest.raises(ValueError, match="monolithic"):
|
||||
pipeline.validate_disagg_role(role)
|
||||
pipeline.validate_disagg_role(RoleType.MONOLITHIC)
|
||||
|
||||
|
||||
def test_encode_stage_leaves_the_vae_alone_on_the_pixel_path():
|
||||
"""No LU entry for the scale -> the whole-video encode is skipped, and the VAE (which
|
||||
a CPU-offloaded run would otherwise move to the GPU for nothing) is never touched.
|
||||
"""
|
||||
vae = MagicMock(name="vae")
|
||||
bank = Kandinsky6SRLatentUpscalerBank(
|
||||
[{"target_scale": "4x", "model": copy.deepcopy(TINY_LU_MODEL)}],
|
||||
0.5,
|
||||
scales=(4,),
|
||||
)
|
||||
_, _, _, server_args = build_components(())
|
||||
video = random_video(9, 64, 64)
|
||||
batch = make_request(video)
|
||||
stage = make_stage(Kandinsky6SREncodeStage, server_args, vae, bank)
|
||||
assert stage.forward(batch, server_args) is batch
|
||||
assert torch.equal(batch.extra[SR_VIDEO_KEY], video)
|
||||
assert SR_LR_LATENT_KEY not in batch.extra
|
||||
assert vae.method_calls == [] and not vae.called
|
||||
|
||||
|
||||
def test_input_stage_runs_on_a_synthetic_clip_for_warmup_requests():
|
||||
"""Warmup requests carry no video (synthetic warmup) or a real one that must not be
|
||||
decoded twice (request warmup): both run the real path on a small synthetic clip,
|
||||
without audio, and without needing a prompt."""
|
||||
params = Kandinsky6SRSamplingParams(video_path="missing_on_purpose.mp4")
|
||||
batch = Req(sampling_params=params)
|
||||
batch.is_warmup = True
|
||||
Kandinsky6SRInputStage().forward(batch, MagicMock())
|
||||
|
||||
video = batch.extra[SR_VIDEO_KEY]
|
||||
height, width = WARMUP_CLIP_HW
|
||||
# default scale 2.25 = x1.125 pre-upscale (aligned to 16) then tiling scale 2
|
||||
assert video.shape[0] == WARMUP_CLIP_FRAMES and video.dtype == torch.uint8
|
||||
assert video.shape[2:] == (288, 432)
|
||||
# 288x432 is already a multiple of 16, so alignment padding is a no-op here, and the
|
||||
# requested size the output stage will crop back to equals the padded (encoded) size.
|
||||
assert batch.extra[SR_REQUESTED_HW_KEY] == (288, 432)
|
||||
assert batch.extra["kandinsky6_sr_tiling_scale"] == 2
|
||||
assert (batch.fps, batch.num_frames) == (24, WARMUP_CLIP_FRAMES)
|
||||
assert (batch.height, batch.width) == (288 * 2, 432 * 2)
|
||||
assert batch.audio is None and height == 256
|
||||
|
||||
|
||||
def test_input_stage_pads_a_misaligned_source_to_the_spatial_factor():
|
||||
"""A source whose pre-upscaled size is not already a multiple of 16 (every integer
|
||||
scale, which runs no pixel resize at all) must still be padded before whole-video KVAE
|
||||
encoding, and the *requested* (pre-pad) size recorded for the output stage to crop back
|
||||
to -- the fix for "whole-video KVAE encoding fails at the fourth PixelUnshuffle" for
|
||||
scales other than 2.25."""
|
||||
params = Kandinsky6SRSamplingParams(
|
||||
video_path="missing_on_purpose.mp4", sr_resolution_scale=2
|
||||
)
|
||||
batch = Req(sampling_params=params)
|
||||
batch.is_warmup = True
|
||||
Kandinsky6SRInputStage().forward(batch, MagicMock())
|
||||
|
||||
# WARMUP_CLIP_HW is already 16-aligned, so exercise a genuinely misaligned size by
|
||||
# checking the padding helper directly against the same video the stage produced, cut
|
||||
# down to a misaligned size.
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
pad_to_spatial_factor,
|
||||
)
|
||||
|
||||
misaligned = batch.extra[SR_VIDEO_KEY][:, :, :100, :70]
|
||||
padded, requested_hw = pad_to_spatial_factor(misaligned, 16)
|
||||
assert requested_hw == (100, 70)
|
||||
assert padded.shape[-2] % 16 == 0 and padded.shape[-1] % 16 == 0
|
||||
assert torch.equal(padded[:, :, :100, :70], misaligned)
|
||||
|
||||
|
||||
def test_input_stage_rejects_a_missing_video_file():
|
||||
batch = Req(sampling_params=Kandinsky6SRSamplingParams(video_path="nope.mp4"))
|
||||
with pytest.raises(ValueError, match="not found"):
|
||||
Kandinsky6SRInputStage().forward(batch, MagicMock())
|
||||
|
||||
|
||||
def test_output_stage_stitches_crops_resizes_and_carries_the_source_audio():
|
||||
"""The output stage blends tiles, restores the originally requested (pre-alignment-pad)
|
||||
size, applies the delivery resize (fit keeps the aspect ratio and never upscales),
|
||||
publishes the *final* size and fps on the request / result, and hands the source audio
|
||||
through unchanged."""
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
compute_tile_grid_even,
|
||||
)
|
||||
|
||||
grid = compute_tile_grid_even(32, 48, (16, 24), 0.2, 16)
|
||||
plan = TilePlan(
|
||||
tiling_scale=2,
|
||||
frame_hw=(32, 48),
|
||||
base_hw=(32, 48),
|
||||
tile_hw=(16, 24),
|
||||
pixel_grid=grid,
|
||||
)
|
||||
tiles = [
|
||||
torch.full((3, 9, 32, 48), 100 + i, dtype=torch.uint8)
|
||||
for i in range(grid.total_tiles)
|
||||
]
|
||||
audio = torch.linspace(-1, 1, 1000)
|
||||
|
||||
params = Kandinsky6SRSamplingParams(
|
||||
video_path="unused.mp4",
|
||||
sr_target_resolution="48x32",
|
||||
sr_target_resize_mode="fit",
|
||||
)
|
||||
batch = Req(sampling_params=params)
|
||||
batch.extra[SR_TILES_KEY], batch.extra[SR_PLAN_KEY] = tiles, plan
|
||||
batch.extra[SR_REQUESTED_HW_KEY] = (32, 48) # no alignment padding happened
|
||||
batch.audio, batch.audio_sample_rate = audio, 44100
|
||||
batch.fps = 12
|
||||
|
||||
result = Kandinsky6SROutputStage().forward(batch, MagicMock())
|
||||
|
||||
assert result.output.shape == (1, 3, 9, 32, 48) # 64x96 fitted into 48x32
|
||||
assert (batch.height, batch.width) == (32, 48)
|
||||
assert result.audio is audio and result.audio_sample_rate == 44100
|
||||
assert result.fps == 12 # the worker's effective fps reaches OutputBatch
|
||||
plain = Kandinsky6SRSamplingParams(video_path="unused.mp4")
|
||||
batch = Req(sampling_params=plain)
|
||||
batch.extra[SR_TILES_KEY], batch.extra[SR_PLAN_KEY] = tiles, plan
|
||||
assert Kandinsky6SROutputStage().forward(batch, MagicMock()).output.shape == (
|
||||
1,
|
||||
3,
|
||||
9,
|
||||
64,
|
||||
96,
|
||||
)
|
||||
|
||||
|
||||
def test_output_stage_crops_the_alignment_padding_back_off():
|
||||
"""A plan whose stitched frame is larger than what was actually requested (the input
|
||||
stage padded for alignment) must be cropped down to the requested size before any
|
||||
delivery resize -- not left at the padded size."""
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
compute_tile_grid_even,
|
||||
)
|
||||
|
||||
# Padded frame is 32x48 (aligned), but only 25x40 pixels were actually requested.
|
||||
grid = compute_tile_grid_even(32, 48, (16, 24), 0.2, 16)
|
||||
plan = TilePlan(
|
||||
tiling_scale=2,
|
||||
frame_hw=(32, 48),
|
||||
base_hw=(32, 48),
|
||||
tile_hw=(16, 24),
|
||||
pixel_grid=grid,
|
||||
)
|
||||
tiles = [
|
||||
torch.full((3, 9, 32, 48), 100 + i, dtype=torch.uint8)
|
||||
for i in range(grid.total_tiles)
|
||||
]
|
||||
batch = Req(sampling_params=Kandinsky6SRSamplingParams(video_path="unused.mp4"))
|
||||
batch.extra[SR_TILES_KEY], batch.extra[SR_PLAN_KEY] = tiles, plan
|
||||
batch.extra[SR_REQUESTED_HW_KEY] = (25, 40)
|
||||
|
||||
result = Kandinsky6SROutputStage().forward(batch, MagicMock())
|
||||
|
||||
assert result.output.shape == (1, 3, 9, 50, 80) # (25, 40) * tiling_scale(2)
|
||||
assert (batch.height, batch.width) == (50, 80)
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Tile blending and initial latent precision."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.sampling import (
|
||||
DitSpec,
|
||||
SamplingSpec,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiled import (
|
||||
build_chunk_latent,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.tiling import (
|
||||
compute_tile_grid_even,
|
||||
stitch_tiles_hanning,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"height,width,tile_hw",
|
||||
[
|
||||
(512, 512, (512, 512)),
|
||||
(512, 768, (512, 768)),
|
||||
(768, 512, (768, 512)),
|
||||
(768, 1536, (512, 768)),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("value", [0.0, 1.0, 255.0])
|
||||
def test_hann_blending_preserves_constant_color_including_corners(
|
||||
height, width, tile_hw, value
|
||||
):
|
||||
# tiny but nonzero Hann corner weights must not be clamped before normalization
|
||||
grid = compute_tile_grid_even(height, width, tile_hw, 0.2, 16)
|
||||
tiles = [torch.full((3, 1, *tile_hw), value) for _ in range(grid.total_tiles)]
|
||||
result = stitch_tiles_hanning(tiles, grid, height, width, scale=1)
|
||||
torch.testing.assert_close(
|
||||
result, torch.full_like(result, value), rtol=1e-5, atol=1e-2
|
||||
)
|
||||
|
||||
|
||||
def test_uncovered_pixels_are_zero_not_nan():
|
||||
grid = compute_tile_grid_even(64, 64, (64, 64), 0.2, 16)
|
||||
tile = torch.full((3, 1, 64, 64), 100.0)
|
||||
result = stitch_tiles_hanning([tile], grid, 64, 128, scale=1)
|
||||
assert result.shape == (3, 1, 64, 128)
|
||||
torch.testing.assert_close(result[..., :64], tile, rtol=1e-5, atol=1e-2)
|
||||
assert torch.equal(result[..., 64:], torch.zeros_like(result[..., 64:]))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dit_dtype", [torch.bfloat16, torch.float32, None])
|
||||
def test_initial_latent_uses_dit_dtype_or_preserves_input(dit_dtype):
|
||||
dit_spec = DitSpec(
|
||||
instruct_type="noise",
|
||||
visual_cond=False,
|
||||
in_visual_dim=4,
|
||||
use_motion_score=False,
|
||||
patch_size=(1, 1, 1),
|
||||
dtype=dit_dtype,
|
||||
)
|
||||
spec = SamplingSpec(
|
||||
tiling_scale=2,
|
||||
tiles_batch_size=1,
|
||||
seed=1,
|
||||
num_steps=5,
|
||||
tile_min_overlap=0.2,
|
||||
visual_size=512,
|
||||
scale_factor=(1.0, 1.0, 1.0),
|
||||
scheduler_scale=5.0,
|
||||
lq_noise_scale=0.7,
|
||||
lq_noise_type="linear",
|
||||
lq_channel_noise_scale=0.0,
|
||||
cap_noise_timestep=False,
|
||||
piflow=None,
|
||||
)
|
||||
tile = torch.randn(4, 4, 4, 4, dtype=torch.float64)
|
||||
result = build_chunk_latent(
|
||||
[tile],
|
||||
seed=1,
|
||||
dit_spec=dit_spec,
|
||||
spec=spec,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
assert result.dtype == (dit_dtype or tile.dtype)
|
||||
@@ -0,0 +1,240 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Streaming input / output helpers of Kandinsky 6 video SR (fps and frame budget).
|
||||
|
||||
The reference decodes every frame and then applies ``resample_to_target_fps`` and
|
||||
``clip_to_aligned_frames``; the port streams. These tests pin that both give the same
|
||||
frames, and that the output tensor survives the framework's uint8 conversion.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.kandinsky6_sr import (
|
||||
Kandinsky6SRSamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import _sample_to_uint8_frames
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.input_stage import (
|
||||
Kandinsky6SRInputStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.run_spec import (
|
||||
SR_TILING_SCALE_KEY,
|
||||
SR_VIDEO_KEY,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_io import (
|
||||
FrameSelector,
|
||||
decode_clip,
|
||||
extract_source_audio,
|
||||
to_output_video,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.kandinsky6_sr.video_utils import (
|
||||
clip_to_aligned_frames,
|
||||
resample_to_target_fps,
|
||||
)
|
||||
|
||||
FRAME_COUNTS = [1, 2, 9, 24, 50, 121, 125, 200, 301, 302, 400, 1000]
|
||||
SOURCE_FPS = [12, 23.976, 24, 25, 29.97, 30, 48, 50, 59.94, 60, 120]
|
||||
|
||||
|
||||
def _stream_select(total: int, fps: float) -> tuple[list[int], int]:
|
||||
"""Run the streaming selector over frames ``0 .. total - 1`` (early stop included)."""
|
||||
selector = FrameSelector(fps)
|
||||
kept: list[int] = []
|
||||
seen = 0
|
||||
for index in range(total):
|
||||
if selector.keep(index):
|
||||
kept.append(index)
|
||||
selector.mark_kept()
|
||||
seen += 1
|
||||
if selector.enough(seen):
|
||||
break
|
||||
return kept[: selector.limit(seen, len(kept))], selector.effective_fps
|
||||
|
||||
|
||||
def _reference_select(total: int, fps: float) -> tuple[list[int], int] | None:
|
||||
ids = torch.arange(total).view(total, 1, 1, 1)
|
||||
resampled, effective_fps = resample_to_target_fps(ids, fps)
|
||||
try:
|
||||
clipped = clip_to_aligned_frames(resampled)
|
||||
except ValueError:
|
||||
return None
|
||||
return clipped.flatten().tolist(), effective_fps
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fps", SOURCE_FPS)
|
||||
def test_streaming_selection_equals_reference_resample_and_clip(fps):
|
||||
"""The stream may stop early once 121 frames are kept, but must return exactly the
|
||||
frames (and the effective fps) of decode-all -> resample_to_target_fps ->
|
||||
clip_to_aligned_frames, including sources too short for the 121-frame budget and
|
||||
``int(total / step)`` truncation of the last selectable frame."""
|
||||
for total in FRAME_COUNTS:
|
||||
expected = _reference_select(total, fps)
|
||||
kept, effective_fps = _stream_select(total, fps)
|
||||
if expected is None:
|
||||
# the reference raises "no readable frames"; nothing usable was selected
|
||||
assert not kept, (total, fps)
|
||||
continue
|
||||
ids = torch.tensor(kept).view(-1, 1, 1, 1)
|
||||
actual = clip_to_aligned_frames(ids).flatten().tolist()
|
||||
assert (actual, effective_fps) == expected, (total, fps)
|
||||
|
||||
|
||||
def test_effective_fps_is_an_int_by_the_reference_rule():
|
||||
"""Downsampled sources play at exactly 24, in-tolerance / slow sources keep
|
||||
``round(source fps)`` (never a float: SamplingParams.fps must be a positive int)."""
|
||||
assert FrameSelector(60.0).effective_fps == 24
|
||||
assert FrameSelector(25.0).effective_fps == 25
|
||||
assert FrameSelector(23.976).effective_fps == 24
|
||||
assert FrameSelector(12.0).effective_fps == 12
|
||||
assert all(isinstance(FrameSelector(fps).effective_fps, int) for fps in SOURCE_FPS)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Real files (PyAV)
|
||||
# --------------------------------------------------------------------------- #
|
||||
av = pytest.importorskip("av")
|
||||
|
||||
|
||||
def write_test_video(path, *, frames, fps, size=(48, 64), audio_seconds=None):
|
||||
"""Tiny mp4 whose frame ``i`` is a flat colour ``i`` (so index errors are visible)."""
|
||||
from fractions import Fraction
|
||||
|
||||
height, width = size
|
||||
with av.open(str(path), mode="w") as container:
|
||||
video = container.add_stream("mpeg4", rate=fps)
|
||||
video.width, video.height, video.pix_fmt = width, height, "yuv420p"
|
||||
audio = None
|
||||
if audio_seconds is not None:
|
||||
audio = container.add_stream("aac", rate=22050)
|
||||
audio.layout = "mono"
|
||||
for index in range(frames):
|
||||
gray = (index * 5) % 250
|
||||
array = np.full((height, width, 3), gray, dtype=np.uint8)
|
||||
frame = av.VideoFrame.from_ndarray(array, format="rgb24")
|
||||
frame.pts, frame.time_base = index, Fraction(1, fps)
|
||||
for packet in video.encode(frame):
|
||||
container.mux(packet)
|
||||
if audio is not None:
|
||||
t = np.arange(int(22050 * audio_seconds)) / 22050
|
||||
samples = (0.3 * np.sin(2 * np.pi * 440 * t)).astype(np.float32)[None]
|
||||
frame = av.AudioFrame.from_ndarray(samples, format="fltp", layout="mono")
|
||||
frame.sample_rate, frame.pts = 22050, 0
|
||||
frame.time_base = Fraction(1, 22050)
|
||||
for packet in audio.encode(frame):
|
||||
container.mux(packet)
|
||||
for packet in audio.encode():
|
||||
container.mux(packet)
|
||||
for packet in video.encode():
|
||||
container.mux(packet)
|
||||
|
||||
|
||||
def _decode_everything(path):
|
||||
with av.open(str(path)) as container:
|
||||
stream = container.streams.video[0]
|
||||
fps = float(stream.average_rate)
|
||||
frames = [
|
||||
torch.from_numpy(f.to_ndarray(format="rgb24")).permute(2, 0, 1)
|
||||
for f in container.decode(stream)
|
||||
]
|
||||
return torch.stack(frames), fps
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fps,frames", [(30, 40), (24, 26), (12, 20), (60, 80)])
|
||||
def test_decode_clip_equals_reference_cli_pipeline(tmp_path, fps, frames):
|
||||
"""decode_clip == read_video_tchw_uint8 -> resample_to_target_fps ->
|
||||
clip_to_aligned_frames: same pixels, same frame count (1 + 8k), same output fps."""
|
||||
path = tmp_path / "clip.mp4"
|
||||
write_test_video(path, frames=frames, fps=fps)
|
||||
full, source_fps = _decode_everything(path)
|
||||
expected, expected_fps = resample_to_target_fps(full, source_fps)
|
||||
expected = clip_to_aligned_frames(expected)
|
||||
|
||||
clip = decode_clip(str(path))
|
||||
|
||||
assert clip.frames.dtype == torch.uint8 and clip.frames.shape[1] == 3
|
||||
assert clip.frames.shape[0] % 8 == 1
|
||||
assert torch.equal(clip.frames, expected)
|
||||
assert clip.fps == expected_fps and isinstance(clip.fps, int)
|
||||
|
||||
|
||||
def test_extract_source_audio_is_mono_float_trimmed_and_optional(tmp_path):
|
||||
"""Audio is decoded to mono float32 at 44.1 kHz, trimmed to the processed duration
|
||||
(the reference muxes the whole track, leaving audio longer than the video), and a
|
||||
file without an audio stream yields None instead of a silent track."""
|
||||
with_audio = tmp_path / "with_audio.mp4"
|
||||
write_test_video(with_audio, frames=10, fps=10, audio_seconds=1.0)
|
||||
full = extract_source_audio(str(with_audio))
|
||||
trimmed = extract_source_audio(str(with_audio), max_seconds=0.25)
|
||||
assert full.dtype == np.float32 and full.ndim == 1
|
||||
assert abs(len(full) - 44100) < 2048 # aac priming / resampler delay
|
||||
assert len(trimmed) == 11025
|
||||
assert np.abs(full).max() <= 1.0 and np.abs(full).max() > 0.05
|
||||
|
||||
silent = tmp_path / "silent.mp4"
|
||||
write_test_video(silent, frames=10, fps=10)
|
||||
assert extract_source_audio(str(silent)) is None
|
||||
|
||||
|
||||
def test_output_video_round_trips_through_uint8_conversions():
|
||||
"""Every uint8 value must survive the framework writer's ``(x * 255)`` truncation,
|
||||
also when a consumer upcasts the fp16 tensor before multiplying (a plain ``k / 255``
|
||||
stored in fp16 fails that order for over half of the values)."""
|
||||
values = torch.arange(256, dtype=torch.uint8)
|
||||
video = values.view(1, 1, 8, 32).expand(3, 2, 8, 32).contiguous()
|
||||
output = to_output_video(video)
|
||||
assert output.dtype == torch.float16 and output.shape == (1, 3, 2, 8, 32)
|
||||
|
||||
frames = _sample_to_uint8_frames(output[0]) # framework writer: list of [H, W, 3]
|
||||
restored = torch.from_numpy(np.stack(frames)).permute(3, 0, 1, 2)
|
||||
assert torch.equal(restored, video)
|
||||
assert torch.equal((output.float() * 255).to(torch.uint8)[0], video)
|
||||
|
||||
naive = (video.float() / 255.0).to(torch.float16)
|
||||
assert not torch.equal((naive.float() * 255).to(torch.uint8), video)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scale,size,expected_frame,tiling_scale",
|
||||
[
|
||||
(2, (48, 64), (48, 64), 2),
|
||||
(2.25, (64, 96), (64, 112), 2),
|
||||
(4, (48, 64), (48, 64), 4),
|
||||
],
|
||||
)
|
||||
def test_input_stage_fills_the_request_from_a_video_file(
|
||||
tmp_path, scale, size, expected_frame, tiling_scale
|
||||
):
|
||||
"""A 30 fps source is thinned to 24 fps (an int), floored to 1 + 8k frames, and the
|
||||
request's output geometry follows the clip: x1.125 pre-upscale aligned to 16 for the
|
||||
2.25 scale, the tile scale for the rest. The audio is mono float32 at 44.1 kHz and
|
||||
trimmed to the processed duration."""
|
||||
path = tmp_path / "clip.mp4"
|
||||
write_test_video(path, frames=40, fps=30, size=size, audio_seconds=1.5)
|
||||
params = Kandinsky6SRSamplingParams(video_path=str(path), sr_resolution_scale=scale)
|
||||
batch = Req(sampling_params=params)
|
||||
|
||||
Kandinsky6SRInputStage().forward(batch, None)
|
||||
|
||||
video = batch.extra[SR_VIDEO_KEY]
|
||||
assert video.dtype == torch.uint8 and video.shape[1] == 3
|
||||
assert video.shape[0] == 25 and batch.num_frames == 25 # int(40 / 1.25) = 32 -> 25
|
||||
assert tuple(video.shape[2:]) == expected_frame
|
||||
assert batch.extra[SR_TILING_SCALE_KEY] == tiling_scale
|
||||
assert batch.fps == 24 and isinstance(batch.fps, int)
|
||||
assert (batch.height, batch.width) == (
|
||||
expected_frame[0] * tiling_scale,
|
||||
expected_frame[1] * tiling_scale,
|
||||
)
|
||||
assert batch.audio.dtype == torch.float32 and batch.audio.ndim == 1
|
||||
assert batch.audio.shape[0] == round(25 / 24 * 44100)
|
||||
assert batch.audio_sample_rate == 44100
|
||||
|
||||
|
||||
def test_input_stage_produces_no_audio_for_a_silent_source(tmp_path):
|
||||
path = tmp_path / "silent.mp4"
|
||||
write_test_video(path, frames=9, fps=24)
|
||||
batch = Req(sampling_params=Kandinsky6SRSamplingParams(video_path=str(path)))
|
||||
Kandinsky6SRInputStage().forward(batch, None)
|
||||
assert batch.audio is None and batch.audio_sample_rate is None
|
||||
assert batch.fps == 24 and batch.num_frames == 9
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user